Continuing file downloads instead of downloading a duplicate.

This commit is contained in:
Roland 2025-01-17 12:02:24 -07:00
parent dcbac00f6e
commit f321e9f6f3
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
5 changed files with 79 additions and 63 deletions

View file

@ -1,10 +1,12 @@
import asyncio
import contextlib
import hashlib
import inspect
import logging
import multiprocessing
import os
import pathlib
import re
import tempfile
import traceback
import xml.etree.ElementTree as ET
@ -30,7 +32,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path, timer, aiohttp_get, \
get_download_info, trim_file_name, TRACE_LEVEL
get_download_info, trim_file_name
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError
@ -402,63 +404,68 @@ class Downloader:
return None
async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \
-> pathlib.Path:
async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True,
check_for_md5: bool = True) -> pathlib.Path:
from wrolpi.files.lib import glob_shared_stem
if not ARIA2C_PATH:
raise DownloadError('Cannot find aria2c executable')
meta4_contents = None
if check_for_meta4:
meta4_contents = await self.get_meta4_contents(url)
# Log download time.
with timer('download_file', level='info'):
meta4_contents = None
if check_for_meta4:
meta4_contents = await self.get_meta4_contents(url)
info = await get_download_info(url)
info = await get_download_info(url)
# TODO verify that this output_path is exclusive.
output_path = destination / trim_file_name(info.name)
# TODO verify that this output_path is exclusive.
output_path = destination / trim_file_name(info.name)
with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path:
cmd = (
ARIA2C_PATH,
'-l', '-',
'-j3', # concurrent downloads
'-s3', # split jobs
'-d', destination,
url
)
meta4_path = pathlib.Path(meta4_path.name)
if meta4_contents:
meta4_path.write_bytes(meta4_contents)
cmd = (*cmd,
'-M', meta4_path,
)
with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path:
cmd = (
ARIA2C_PATH,
'-l', '-',
'-c', # Continue downloading a partially downloaded file.
'-j3', # concurrent downloads
'-s3', # split jobs
'-d', destination,
url
)
meta4_path = pathlib.Path(meta4_path.name)
if meta4_contents:
meta4_path.write_bytes(meta4_contents)
cmd = (*cmd,
'-M', meta4_path,
)
return_code, logs, stdout = await self.process_runner(
download_id,
url, # Log the original URL.
cmd,
destination,
)
return_code, logs, stdout = await self.process_runner(
download_id,
url, # Log the original URL.
cmd,
destination,
)
error = logs.get('stderr') or logs.get('stdout')
error = error.decode() if error else error
error = logs.get('stderr') or logs.get('stdout')
error = error.decode() if error else error
if return_code != 0:
raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}')
if return_code != 0:
raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}')
if not output_path.is_file():
raise DownloadError(f'{error}\n\nOutput file not found: {output_path}')
if not output_path.is_file():
raise DownloadError(f'{error}\n\nOutput file not found: {output_path}')
# Delete any trailing meta4 files.
matching_files = glob_shared_stem(output_path)
for file in matching_files:
if file.name.endswith('.meta4'):
if logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'Deleting meta4 file: {file}')
file.unlink()
# Delete any trailing meta4 files.
matching_files = glob_shared_stem(output_path)
for file in matching_files:
if file.name.endswith('.meta4'):
logger.warning(f'Deleting meta4 file: {file}')
file.unlink()
if file.name.endswith('.aria2'):
logger.trace(f'Deleting aria2 file: {file}')
file.unlink()
return output_path
return output_path
class DownloadManager:

View file

@ -1,15 +1,13 @@
import pathlib
import pathlib
import tempfile
import traceback
from abc import ABC
from wrolpi.cmd import which
from wrolpi.common import get_media_directory, logger, get_download_info, \
trim_file_name, TRACE_LEVEL
from wrolpi.common import get_media_directory, logger, TRACE_LEVEL
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.lib import upsert_file
from wrolpi.errors import UnrecoverableDownloadError, IgnoredDirectoryError
from wrolpi.files.lib import upsert_file, get_file_location_href
from wrolpi.vars import PYTEST
__all__ = ['FileDownloader', 'file_downloader']
@ -52,7 +50,7 @@ class FileDownloader(Downloader, ABC):
raise UnrecoverableDownloadError(f'Cannot download outside media directory!')
destination.mkdir(parents=True, exist_ok=True)
error = None
output_path = None
try:
output_path = await self.download_file(download.id, download.url, destination)
fg = await upsert_file(output_path, tag_names=download.tag_names)
@ -61,9 +59,13 @@ class FileDownloader(Downloader, ABC):
success=True,
location=location,
)
except IgnoredDirectoryError:
# `upsert_file` says this is an ignored file, that's fine.
return DownloadResult(
success=True,
location=get_file_location_href(output_path)
)
except Exception as e:
if error:
logger.error(f'FileDownloader failed with aria2c error:\n{error}')
logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e)
if PYTEST:
raise

View file

@ -9,6 +9,7 @@ import pathlib
import re
import shutil
import subprocess
import urllib.parse
from collections import OrderedDict
from itertools import zip_longest
from pathlib import Path
@ -26,7 +27,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
partition, cancelable_wrapper, \
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
get_wrolpi_config, \
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
from wrolpi.downloader import download_manager
@ -52,7 +53,7 @@ logger = logger.getChild(__name__)
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
'move', 'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href']
@optional_session
@ -1608,3 +1609,15 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
file_group.flush()
return file_group
def get_file_location_href(file: pathlib.Path) -> str:
"""Return the location a file can be viewed at on the UI."""
parent = str(get_relative_to_media_directory(file.parent))
preview = str(get_relative_to_media_directory(file))
if parent == '.':
# File is in the top of the media directory, App already shows top directory open.
query = urllib.parse.urlencode(dict(preview=str(preview)))
else:
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
return f'/files?{query}'

View file

@ -425,14 +425,8 @@ class FileGroup(ModelHelper, Base):
@property
def location(self):
"""Returns the URL that the FileGroup can be previewed."""
parent = str(get_relative_to_media_directory(self.primary_path.parent))
preview = str(get_relative_to_media_directory(self.primary_path))
if parent == '.':
# File is in the top of the media directory, App already shows top directory open.
query = urllib.parse.urlencode(dict(preview=str(preview)))
else:
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
return f'/files?{query}'
from wrolpi.files.lib import get_file_location_href
return get_file_location_href(self.primary_path)
def get_model_class(self) -> Optional[Type[ModelHelper]]:
"""Return the class that is suitable for this FileGroup (usually based on mimetype), if any."""

View file

@ -98,7 +98,7 @@ VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 80
YTDLP_CACHE_DIR = os.environ.get('YTDLP_CACHE_DIR', '/tmp/ytdlp_cache')
VIDEO_INFO_JSON_KEYS_TO_CLEAN = list_var('DELETED_VIDEO_KEYS', 'automatic_captions,formats,thumbnails')
default_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36'
default_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3'
DOWNLOAD_USER_AGENT = os.environ.get('DOWNLOAD_USER_AGENT', default_user_agent)
DEFAULT_HTTP_HEADERS = {