mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Continuing file downloads instead of downloading a duplicate.
This commit is contained in:
parent
dcbac00f6e
commit
f321e9f6f3
5 changed files with 79 additions and 63 deletions
|
|
@ -1,10 +1,12 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import hashlib
|
||||
import inspect
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import tempfile
|
||||
import traceback
|
||||
import xml.etree.ElementTree as ET
|
||||
|
|
@ -30,7 +32,7 @@ from wrolpi.api_utils import api_app, perpetual_signal
|
|||
from wrolpi.cmd import which
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
|
||||
wrol_mode_enabled, background_task, get_wrolpi_config, get_absolute_media_path, timer, aiohttp_get, \
|
||||
get_download_info, trim_file_name, TRACE_LEVEL
|
||||
get_download_info, trim_file_name
|
||||
from wrolpi.dates import TZDateTime, now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload, ValidationError, DownloadError
|
||||
|
|
@ -402,63 +404,68 @@ class Downloader:
|
|||
|
||||
return None
|
||||
|
||||
async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True) \
|
||||
-> pathlib.Path:
|
||||
async def download_file(self, download_id: int, url: str, destination: pathlib.Path, check_for_meta4: bool = True,
|
||||
check_for_md5: bool = True) -> pathlib.Path:
|
||||
from wrolpi.files.lib import glob_shared_stem
|
||||
|
||||
if not ARIA2C_PATH:
|
||||
raise DownloadError('Cannot find aria2c executable')
|
||||
|
||||
meta4_contents = None
|
||||
if check_for_meta4:
|
||||
meta4_contents = await self.get_meta4_contents(url)
|
||||
# Log download time.
|
||||
with timer('download_file', level='info'):
|
||||
meta4_contents = None
|
||||
if check_for_meta4:
|
||||
meta4_contents = await self.get_meta4_contents(url)
|
||||
|
||||
info = await get_download_info(url)
|
||||
info = await get_download_info(url)
|
||||
|
||||
# TODO verify that this output_path is exclusive.
|
||||
output_path = destination / trim_file_name(info.name)
|
||||
# TODO verify that this output_path is exclusive.
|
||||
output_path = destination / trim_file_name(info.name)
|
||||
|
||||
with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path:
|
||||
cmd = (
|
||||
ARIA2C_PATH,
|
||||
'-l', '-',
|
||||
'-j3', # concurrent downloads
|
||||
'-s3', # split jobs
|
||||
'-d', destination,
|
||||
url
|
||||
)
|
||||
meta4_path = pathlib.Path(meta4_path.name)
|
||||
if meta4_contents:
|
||||
meta4_path.write_bytes(meta4_contents)
|
||||
cmd = (*cmd,
|
||||
'-M', meta4_path,
|
||||
)
|
||||
with tempfile.NamedTemporaryFile(suffix='.meta4') as meta4_path:
|
||||
cmd = (
|
||||
ARIA2C_PATH,
|
||||
'-l', '-',
|
||||
'-c', # Continue downloading a partially downloaded file.
|
||||
'-j3', # concurrent downloads
|
||||
'-s3', # split jobs
|
||||
'-d', destination,
|
||||
url
|
||||
)
|
||||
meta4_path = pathlib.Path(meta4_path.name)
|
||||
if meta4_contents:
|
||||
meta4_path.write_bytes(meta4_contents)
|
||||
cmd = (*cmd,
|
||||
'-M', meta4_path,
|
||||
)
|
||||
|
||||
return_code, logs, stdout = await self.process_runner(
|
||||
download_id,
|
||||
url, # Log the original URL.
|
||||
cmd,
|
||||
destination,
|
||||
)
|
||||
return_code, logs, stdout = await self.process_runner(
|
||||
download_id,
|
||||
url, # Log the original URL.
|
||||
cmd,
|
||||
destination,
|
||||
)
|
||||
|
||||
error = logs.get('stderr') or logs.get('stdout')
|
||||
error = error.decode() if error else error
|
||||
error = logs.get('stderr') or logs.get('stdout')
|
||||
error = error.decode() if error else error
|
||||
|
||||
if return_code != 0:
|
||||
raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}')
|
||||
if return_code != 0:
|
||||
raise DownloadError(f'{error}\n\nFileDownloader failed with return code {return_code}')
|
||||
|
||||
if not output_path.is_file():
|
||||
raise DownloadError(f'{error}\n\nOutput file not found: {output_path}')
|
||||
if not output_path.is_file():
|
||||
raise DownloadError(f'{error}\n\nOutput file not found: {output_path}')
|
||||
|
||||
# Delete any trailing meta4 files.
|
||||
matching_files = glob_shared_stem(output_path)
|
||||
for file in matching_files:
|
||||
if file.name.endswith('.meta4'):
|
||||
if logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'Deleting meta4 file: {file}')
|
||||
file.unlink()
|
||||
# Delete any trailing meta4 files.
|
||||
matching_files = glob_shared_stem(output_path)
|
||||
for file in matching_files:
|
||||
if file.name.endswith('.meta4'):
|
||||
logger.warning(f'Deleting meta4 file: {file}')
|
||||
file.unlink()
|
||||
if file.name.endswith('.aria2'):
|
||||
logger.trace(f'Deleting aria2 file: {file}')
|
||||
file.unlink()
|
||||
|
||||
return output_path
|
||||
return output_path
|
||||
|
||||
|
||||
class DownloadManager:
|
||||
|
|
|
|||
|
|
@ -1,15 +1,13 @@
|
|||
import pathlib
|
||||
import pathlib
|
||||
import tempfile
|
||||
import traceback
|
||||
from abc import ABC
|
||||
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import get_media_directory, logger, get_download_info, \
|
||||
trim_file_name, TRACE_LEVEL
|
||||
from wrolpi.common import get_media_directory, logger, TRACE_LEVEL
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.lib import upsert_file
|
||||
from wrolpi.errors import UnrecoverableDownloadError, IgnoredDirectoryError
|
||||
from wrolpi.files.lib import upsert_file, get_file_location_href
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
__all__ = ['FileDownloader', 'file_downloader']
|
||||
|
|
@ -52,7 +50,7 @@ class FileDownloader(Downloader, ABC):
|
|||
raise UnrecoverableDownloadError(f'Cannot download outside media directory!')
|
||||
destination.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
error = None
|
||||
output_path = None
|
||||
try:
|
||||
output_path = await self.download_file(download.id, download.url, destination)
|
||||
fg = await upsert_file(output_path, tag_names=download.tag_names)
|
||||
|
|
@ -61,9 +59,13 @@ class FileDownloader(Downloader, ABC):
|
|||
success=True,
|
||||
location=location,
|
||||
)
|
||||
except IgnoredDirectoryError:
|
||||
# `upsert_file` says this is an ignored file, that's fine.
|
||||
return DownloadResult(
|
||||
success=True,
|
||||
location=get_file_location_href(output_path)
|
||||
)
|
||||
except Exception as e:
|
||||
if error:
|
||||
logger.error(f'FileDownloader failed with aria2c error:\n{error}')
|
||||
logger.error(f'Failed to download {repr(str(download.url))}', exc_info=e)
|
||||
if PYTEST:
|
||||
raise
|
||||
|
|
|
|||
|
|
@ -9,6 +9,7 @@ import pathlib
|
|||
import re
|
||||
import shutil
|
||||
import subprocess
|
||||
import urllib.parse
|
||||
from collections import OrderedDict
|
||||
from itertools import zip_longest
|
||||
from pathlib import Path
|
||||
|
|
@ -26,7 +27,7 @@ from wrolpi.common import get_media_directory, wrol_mode_check, logger, limit_co
|
|||
partition, cancelable_wrapper, \
|
||||
get_files_and_directories, chunks_by_stem, apply_modelers, apply_refresh_cleanup, background_task, walk, \
|
||||
get_wrolpi_config, \
|
||||
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL
|
||||
timer, chunks, unique_by_predicate, get_paths_in_media_directory, TRACE_LEVEL, get_relative_to_media_directory
|
||||
from wrolpi.dates import now, from_timestamp, months_selector_to_where, date_range_to_where
|
||||
from wrolpi.db import get_db_session, get_db_curs, mogrify, optional_session
|
||||
from wrolpi.downloader import download_manager
|
||||
|
|
@ -52,7 +53,7 @@ logger = logger.getChild(__name__)
|
|||
__all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'refresh_files', 'search_files',
|
||||
'get_mimetype', 'split_file_name_words', 'get_primary_file', 'get_file_statistics',
|
||||
'search_file_suggestion_count', 'glob_shared_stem', 'upsert_file', 'get_unique_files_by_stem',
|
||||
'move', 'rename', 'delete_directory', 'handle_file_group_search_results']
|
||||
'move', 'rename', 'delete_directory', 'handle_file_group_search_results', 'get_file_location_href']
|
||||
|
||||
|
||||
@optional_session
|
||||
|
|
@ -1608,3 +1609,15 @@ async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) ->
|
|||
|
||||
file_group.flush()
|
||||
return file_group
|
||||
|
||||
|
||||
def get_file_location_href(file: pathlib.Path) -> str:
|
||||
"""Return the location a file can be viewed at on the UI."""
|
||||
parent = str(get_relative_to_media_directory(file.parent))
|
||||
preview = str(get_relative_to_media_directory(file))
|
||||
if parent == '.':
|
||||
# File is in the top of the media directory, App already shows top directory open.
|
||||
query = urllib.parse.urlencode(dict(preview=str(preview)))
|
||||
else:
|
||||
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
|
||||
return f'/files?{query}'
|
||||
|
|
|
|||
|
|
@ -425,14 +425,8 @@ class FileGroup(ModelHelper, Base):
|
|||
@property
|
||||
def location(self):
|
||||
"""Returns the URL that the FileGroup can be previewed."""
|
||||
parent = str(get_relative_to_media_directory(self.primary_path.parent))
|
||||
preview = str(get_relative_to_media_directory(self.primary_path))
|
||||
if parent == '.':
|
||||
# File is in the top of the media directory, App already shows top directory open.
|
||||
query = urllib.parse.urlencode(dict(preview=str(preview)))
|
||||
else:
|
||||
query = urllib.parse.urlencode(dict(folders=str(parent), preview=str(preview)))
|
||||
return f'/files?{query}'
|
||||
from wrolpi.files.lib import get_file_location_href
|
||||
return get_file_location_href(self.primary_path)
|
||||
|
||||
def get_model_class(self) -> Optional[Type[ModelHelper]]:
|
||||
"""Return the class that is suitable for this FileGroup (usually based on mimetype), if any."""
|
||||
|
|
|
|||
|
|
@ -98,7 +98,7 @@ VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 80
|
|||
YTDLP_CACHE_DIR = os.environ.get('YTDLP_CACHE_DIR', '/tmp/ytdlp_cache')
|
||||
VIDEO_INFO_JSON_KEYS_TO_CLEAN = list_var('DELETED_VIDEO_KEYS', 'automatic_captions,formats,thumbnails')
|
||||
|
||||
default_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/129.0.0.0 Safari/537.36'
|
||||
default_user_agent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/131.0.0.0 Safari/537.3'
|
||||
DOWNLOAD_USER_AGENT = os.environ.get('DOWNLOAD_USER_AGENT', default_user_agent)
|
||||
|
||||
DEFAULT_HTTP_HEADERS = {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue