diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index ab70a957..4b334746 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1,10 +1,12 @@ import asyncio import contextlib +import hashlib import inspect import logging import multiprocessing import os import pathlib +import re import tempfile import traceback import xml.etree.ElementTree as ET @@ -30,7 +32,7 @@ from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.cmd import which from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \ wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path, timer, aiohttp_get, \ - get_download_info, trim_file_name, TRACE_LEVEL + get_download_info, trim_file_name from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError @@ -402,63 +404,68 @@ class Downloader: return None - async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \ - -> pathlib.Path: + async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True, + check_for_md5: bool = True) -> pathlib.Path: from wrolpi.files.lib import glob_shared_stem if not ARIA2C_PATH: raise DownloadError('Cannot find aria2c executable') - meta4_contents = None - if check_for_meta4: - meta4_contents = await self.get_meta4_contents(url) + # Log download time. + with timer('download_file', level='info'): + meta4_contents = None + if check_for_meta4: + meta4_contents = await self.get_meta4_contents(url) - info = await get_download_info(url) + info = await get_download_info(url) - # TODO verify that this output_path is exclusive. - output_path = destination / trim_file_name(info.name) + # TODO verify that this output_path is exclusive. + output_path = destination / trim_file_name(info.name) - with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path: - cmd = ( - ARIA2C_PATH, - '-l', '-', - '-j3', # concurrent downloads - '-s3', # split jobs - '-d', destination, - url - ) - meta4_path = pathlib.Path(meta4_path.name) - if meta4_contents: - meta4_path.write_bytes(meta4_contents) - cmd = (*cmd, - '-M', meta4_path, - ) + with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path: + cmd = ( + ARIA2C_PATH, + '-l', '-', + '-c', # Continue downloading a partially downloaded file. + '-j3', # concurrent downloads + '-s3', # split jobs + '-d', destination, + url + ) + meta4_path = pathlib.Path(meta4_path.name) + if meta4_contents: + meta4_path.write_bytes(meta4_contents) + cmd = (*cmd, + '-M', meta4_path, + ) - return_code, logs, stdout = await self.process_runner( - download_id, - url, # Log the original URL. - cmd, - destination, - ) + return_code, logs, stdout = await self.process_runner( + download_id, + url, # Log the original URL. + cmd, + destination, + ) - error = logs.get('stderr') or logs.get('stdout') - error = error.decode() if error else error + error = logs.get('stderr') or logs.get('stdout') + error = error.decode() if error else error - if return_code != 0: - raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}') + if return_code != 0: + raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}') - if not output_path.is_file(): - raise DownloadError(f'{error}\n\nOutput file not found: {output_path}') + if not output_path.is_file(): + raise DownloadError(f'{error}\n\nOutput file not found: {output_path}') - # Delete any trailing meta4 files. - matching_files = glob_shared_stem(output_path) - for file in matching_files: - if file.name.endswith('.meta4'): - if logger.isEnabledFor(TRACE_LEVEL): - logger.trace(f'Deleting meta4 file: {file}') - file.unlink() + # Delete any trailing meta4 files. + matching_files = glob_shared_stem(output_path) + for file in matching_files: + if file.name.endswith('.meta4'): + logger.warning(f'Deleting meta4 file: {file}') + file.unlink() + if file.name.endswith('.aria2'): + logger.trace(f'Deleting aria2 file: {file}') + file.unlink() - return output_path + return output_path class DownloadManager: diff --git a/wrolpi/files/downloader.py b/wrolpi/files/downloader.py index 2a4da67c..61243918 100644 --- a/wrolpi/files/downloader.py +++ b/wrolpi/files/downloader.py @@ -1,15 +1,13 @@ import pathlib import pathlib -import tempfile import traceback from abc import ABC from wrolpi.cmd import which -from wrolpi.common import get_media_directory, logger, get_download_info, \ - trim_file_name, TRACE_LEVEL +from wrolpi.common import get_media_directory, logger, TRACE_LEVEL from wrolpi.downloader import Downloader, Download, DownloadResult -from wrolpi.errors import UnrecoverableDownloadError -from wrolpi.files.lib import upsert_file +from wrolpi.errors import UnrecoverableDownloadError, IgnoredDirectoryError +from wrolpi.files.lib import upsert_file, get_file_location_href from wrolpi.vars import PYTEST __all__ = ['FileDownloader', 'file_downloader'] @@ -52,7 +50,7 @@ class FileDownloader(Downloader, ABC): raise UnrecoverableDownloadError(f'Cannot download outside media directory!') destination.mkdir(parents=True, exist_ok=True) - error = None + output_path = None try: output_path = await self.download_file(download.id, download.url, destination) fg = await upsert_file(output_path, tag_names=download.tag_names) @@ -61,9 +59,13 @@ class FileDownloader(Downloader, ABC): success=True, location=location, ) + except IgnoredDirectoryError: + # `upsert_file` says this is an ignored file, that's fine. + return DownloadResult( + success=True, + location=get_file_location_href(output_path) + ) except Exception as e: - if error: - logger.error(f'FileDownloader failed with aria2c error:\n{error}') logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e) if PYTEST: raise diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index a1f87e55..891d40a1 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -9,6 +9,7 @@ import pathlib import re import shutil import subprocess +import urllib.parse from collections import OrderedDict from itertools import zip_longest from pathlib import Path @@ -26,7 +27,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co partition, cancelable_wrapper, \ get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \ get_wrolpi_config, \ - timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL + timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session from wrolpi.downloader import download_manager @@ -52,7 +53,7 @@ logger = logger.getChild(__name__) __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files', 'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics', 'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem', - 'move', 'rename', 'delete_directory', 'handle_file_group_search_results'] + 'move', 'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href'] @optional_session @@ -1608,3 +1609,15 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> file_group.flush() return file_group + + +def get_file_location_href(file: pathlib.Path) -> str: + """Return the location a file can be viewed at on the UI.""" + parent = str(get_relative_to_media_directory(file.parent)) + preview = str(get_relative_to_media_directory(file)) + if parent == '.': + # File is in the top of the media directory, App already shows top directory open. + query = urllib.parse.urlencode(dict(preview=str(preview))) + else: + query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview))) + return f'/files?{query}' diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 65c33f89..b4d4ffc4 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -425,14 +425,8 @@ class FileGroup(ModelHelper, Base): @property def location(self): """Returns the URL that the FileGroup can be previewed.""" - parent = str(get_relative_to_media_directory(self.primary_path.parent)) - preview = str(get_relative_to_media_directory(self.primary_path)) - if parent == '.': - # File is in the top of the media directory, App already shows top directory open. - query = urllib.parse.urlencode(dict(preview=str(preview))) - else: - query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview))) - return f'/files?{query}' + from wrolpi.files.lib import get_file_location_href + return get_file_location_href(self.primary_path) def get_model_class(self) -> Optional[Type[ModelHelper]]: """Return the class that is suitable for this FileGroup (usually based on mimetype), if any.""" diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 7112f5de..be4ba323 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -98,7 +98,7 @@ VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 80 YTDLP_CACHE_DIR = os.environ.get('YTDLP_CACHE_DIR', '/tmp/ytdlp_cache') VIDEO_INFO_JSON_KEYS_TO_CLEAN = list_var('DELETED_VIDEO_KEYS', 'automatic_captions,formats,thumbnails') -default_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36' +default_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3' DOWNLOAD_USER_AGENT = os.environ.get('DOWNLOAD_USER_AGENT', default_user_agent) DEFAULT_HTTP_HEADERS = {