mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
Converting RSS and Video Downloaders to new testable strategy.
This commit is contained in:
parent
8f49f33ba2
commit
8f0bb78722
9 changed files with 851 additions and 383 deletions
|
|
@ -9,26 +9,10 @@ import pytz
|
|||
|
||||
from modules.archive.lib import archive_strftime
|
||||
from modules.archive.models import Archive
|
||||
from wrolpi.cmd import run_command
|
||||
from wrolpi.collections import Collection
|
||||
from wrolpi.downloader import DownloadContext
|
||||
|
||||
|
||||
def make_test_ctx(*, is_cancelled=None, can_download=None,
|
||||
outside_download_window=None, run_command_=None,
|
||||
download_timeout=None) -> DownloadContext:
|
||||
"""Build a DownloadContext suitable for downloader unit tests.
|
||||
|
||||
Defaults to a never-cancelled context that uses the real run_command. Tests can
|
||||
override any callable to simulate cancellation, throttling, or fake subprocess output.
|
||||
"""
|
||||
return DownloadContext(
|
||||
is_cancelled=is_cancelled or (lambda: False),
|
||||
can_download=can_download or (lambda: True),
|
||||
outside_download_window=outside_download_window or (lambda: False),
|
||||
run_command=run_command_ or run_command,
|
||||
download_timeout=download_timeout,
|
||||
)
|
||||
# Re-export make_test_ctx so existing `from modules.archive.conftest import make_test_ctx`
|
||||
# call sites keep working. The canonical implementation lives in wrolpi/conftest.py.
|
||||
from wrolpi.conftest import make_test_ctx # noqa: F401
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
|
|||
|
|
@ -213,12 +213,14 @@ async def test_channel_download_requires_refresh(
|
|||
|
||||
downloaded_urls = []
|
||||
|
||||
async def do_download(_, download):
|
||||
# Track all downloaded URLs.
|
||||
downloaded_urls.append(download.url)
|
||||
async def fake_execute(_, prepared, ctx, download=None):
|
||||
# Track all downloaded URLs and short-circuit by returning a DownloadResult.
|
||||
# Per the dispatch contract (wrolpi/downloader.py: signal_download_download),
|
||||
# if execute_download returns a DownloadResult, finalize_download is skipped.
|
||||
downloaded_urls.append(prepared.url)
|
||||
return DownloadResult(success=True)
|
||||
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download):
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download', fake_execute):
|
||||
entries = [
|
||||
dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded.
|
||||
dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'),
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
#! /usr/bin/env python3
|
||||
import copy
|
||||
import dataclasses
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
|
|
@ -8,6 +9,7 @@ import re
|
|||
import time
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta
|
||||
from typing import Tuple, List, Dict, Optional
|
||||
from urllib.parse import urlparse, parse_qs
|
||||
|
|
@ -24,7 +26,7 @@ from wrolpi.common import logger, get_media_directory, escape_file_name, resolve
|
|||
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult, \
|
||||
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
|
||||
clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config
|
||||
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
|
||||
|
|
@ -399,6 +401,40 @@ def is_youtube_rss_feed_url(url: str) -> bool:
|
|||
return bool(YOUTUBE_RSS_FEED_URL_RE.match(url or ''))
|
||||
|
||||
|
||||
@dataclass
|
||||
class PreparedChannel:
|
||||
"""Plan produced by ChannelDownloader.prepare_download.
|
||||
|
||||
Cheap pre-flight: validates the URL is not an RSS feed, snapshots the settings
|
||||
fields execute and finalize need so they don't have to read back through the
|
||||
Download row. All real work — yt-dlp metadata fetch, channel resolution, file
|
||||
indexing — happens in execute_download.
|
||||
"""
|
||||
url: str
|
||||
settings: dict
|
||||
destination: Optional[str]
|
||||
tag_names: Optional[List[str]]
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutedChannel:
|
||||
"""Output of ChannelDownloader.execute_download.
|
||||
|
||||
`info_json` is the resolved yt-dlp metadata; `channel_id` and `collection_id` are
|
||||
the IDs assigned by the channel-resolution helper. `child_settings` is the dict
|
||||
that will be passed to every child VideoDownloader. `error` is set when the
|
||||
catalog update raised — finalize converts that into a failure DownloadResult
|
||||
while still mutating the parent Download with sub_downloader/info_json so the
|
||||
next attempt has them.
|
||||
"""
|
||||
info_json: Optional[dict]
|
||||
channel_id: Optional[int]
|
||||
collection_id: Optional[int]
|
||||
location: Optional[str]
|
||||
child_settings: dict
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class ChannelDownloader(Downloader, ABC):
|
||||
"""Handles downloading of videos in a Channel or Playlist."""
|
||||
name = 'video_channel'
|
||||
|
|
@ -413,9 +449,7 @@ class ChannelDownloader(Downloader, ABC):
|
|||
# A playlist may have an id different from its channel.
|
||||
return info['id'] != info.get('channel_id')
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
"""Update a Channel's catalog, then schedule downloads of every missing video."""
|
||||
_log_ytdlp_versions()
|
||||
def prepare_download(self, session: Session, download: Download) -> PreparedChannel:
|
||||
if is_youtube_rss_feed_url(download.url):
|
||||
# yt-dlp returns no `uploader` or `channel_id` for an RSS feed URL, so falling through
|
||||
# would create a junk channel named after `webpage_url_basename` ("videos.xml").
|
||||
|
|
@ -423,8 +457,23 @@ class ChannelDownloader(Downloader, ABC):
|
|||
f'Refusing to handle RSS feed URL with ChannelDownloader: {download.url}.'
|
||||
f' Use the RSS Downloader (sub_downloader=video) instead.'
|
||||
)
|
||||
return PreparedChannel(
|
||||
url=download.url,
|
||||
settings=download.settings or dict(),
|
||||
destination=str(download.destination) if download.destination else None,
|
||||
tag_names=download.tag_names,
|
||||
)
|
||||
|
||||
async def execute_download(self, prepared: PreparedChannel, ctx: DownloadContext,
|
||||
download: Download = None) -> ExecutedChannel:
|
||||
"""Fetch the channel/playlist metadata via yt-dlp, get/create the local Channel
|
||||
record, and update its catalog. Channel creation + catalog update use a
|
||||
short-lived self-managed session so the manager's outer session isn't held
|
||||
during the file_worker await.
|
||||
"""
|
||||
_log_ytdlp_versions()
|
||||
try:
|
||||
info = extract_info(download.url, process=False)
|
||||
info = extract_info(prepared.url, process=False)
|
||||
except Exception as e:
|
||||
if _bot_blocked(e):
|
||||
raise BotBlockedDownloadError(str(e)) from e
|
||||
|
|
@ -434,86 +483,103 @@ class ChannelDownloader(Downloader, ABC):
|
|||
entries = info.get('entries')
|
||||
if entries and isinstance(entries[0], Dict) and entries[0].get('entries'):
|
||||
# Entries contains more channels/playlist, tell the Maintainer to specify the URL.
|
||||
raise UnrecoverableDownloadError(f'Unable to download {download.url} because the URL is ambiguous.'
|
||||
raise UnrecoverableDownloadError(f'Unable to download {prepared.url} because the URL is ambiguous.'
|
||||
f' Try requesting the videos only:'
|
||||
f' https://example.com -> https://example.com/videos')
|
||||
|
||||
download_settings = download.settings or dict()
|
||||
channel_tag_name = i[0] if (i := download_settings.get('channel_tag_name')) else None
|
||||
|
||||
download.sub_downloader = video_downloader.name
|
||||
download.info_json = info
|
||||
if session := Session.object_session(download):
|
||||
# May not have a session during testing.
|
||||
session.commit()
|
||||
|
||||
channel_tag_name = i[0] if (i := prepared.settings.get('channel_tag_name')) else None
|
||||
name = info.get('uploader') or info.get('webpage_url_basename')
|
||||
if not name:
|
||||
raise ValueError(f'Could not find name')
|
||||
channel_source_id = info.get('channel_id') or info.get('id')
|
||||
channel_url = info.get('channel_url') or info.get('uploader_url') or download.url
|
||||
channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name,
|
||||
destination=download.destination)
|
||||
channel.dict() # get all attributes while we have the session.
|
||||
channel_url = info.get('channel_url') or info.get('uploader_url') or prepared.url
|
||||
|
||||
# Link this channel download to the channel's collection.
|
||||
if channel and channel.collection_id and not download.collection_id:
|
||||
download.collection_id = channel.collection_id
|
||||
if session:
|
||||
session.commit()
|
||||
channel_id = None
|
||||
collection_id = None
|
||||
location = None
|
||||
with get_db_session(commit=True) as session:
|
||||
channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name,
|
||||
destination=prepared.destination)
|
||||
channel.dict() # get all attributes while we have the session.
|
||||
if channel and channel.id:
|
||||
channel_id = channel.id
|
||||
collection_id = channel.collection_id
|
||||
location = f'/videos/channel/{channel.id}/video'
|
||||
|
||||
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
|
||||
|
||||
# The settings to send to the VideoDownloader.
|
||||
settings = dict()
|
||||
if channel:
|
||||
settings.update(dict(collection_id=channel.collection_id, channel_url=download.url))
|
||||
if download.destination:
|
||||
destination = get_absolute_media_path(download.destination)
|
||||
settings['destination'] = str(destination) # Need str for JSON conversion
|
||||
if download.tag_names:
|
||||
settings['tag_names'] = download.tag_names
|
||||
# The settings to send to the VideoDownloader children.
|
||||
child_settings = dict()
|
||||
if channel_id:
|
||||
child_settings.update(dict(collection_id=collection_id, channel_url=prepared.url))
|
||||
if prepared.destination:
|
||||
destination_abs = get_absolute_media_path(prepared.destination)
|
||||
child_settings['destination'] = str(destination_abs) # Need str for JSON conversion
|
||||
if prepared.tag_names:
|
||||
child_settings['tag_names'] = prepared.tag_names
|
||||
# Pass all channel-level inheritable settings to child video downloads.
|
||||
for key in CHANNEL_INHERITABLE_SETTINGS:
|
||||
if key in download_settings:
|
||||
settings[key] = download_settings[key]
|
||||
if key in prepared.settings:
|
||||
child_settings[key] = prepared.settings[key]
|
||||
|
||||
is_a_playlist = self.is_a_playlist(info)
|
||||
try:
|
||||
if not is_a_playlist:
|
||||
await self.prepare_channel_for_downloads(download, channel)
|
||||
else:
|
||||
if not is_a_playlist and channel_id is not None:
|
||||
await self.prepare_channel_for_downloads(info, channel_id)
|
||||
elif is_a_playlist:
|
||||
logger.debug('Not updating channel because this is a playlist')
|
||||
|
||||
downloads = self.get_missing_videos(session, download)
|
||||
return DownloadResult(
|
||||
success=True,
|
||||
location=location,
|
||||
downloads=downloads,
|
||||
settings=settings,
|
||||
)
|
||||
except Exception as e:
|
||||
if PYTEST:
|
||||
raise
|
||||
kind = 'playlist' if is_a_playlist else 'channel'
|
||||
logger.error(f'Failed to update catalog of {kind} {download.url}', exc_info=e)
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
location=location,
|
||||
logger.error(f'Failed to update catalog of {kind} {prepared.url}', exc_info=e)
|
||||
return ExecutedChannel(
|
||||
info_json=info, channel_id=channel_id, collection_id=collection_id,
|
||||
location=location, child_settings=child_settings,
|
||||
error=str(traceback.format_exc()),
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
return ExecutedChannel(
|
||||
info_json=info, channel_id=channel_id, collection_id=collection_id,
|
||||
location=location, child_settings=child_settings,
|
||||
)
|
||||
|
||||
def finalize_download(self, session: Session, download: Download,
|
||||
executed: ExecutedChannel) -> DownloadResult:
|
||||
"""Mutate the parent Download (sub_downloader / info_json / collection_id) and
|
||||
return the list of missing video URLs as child downloads."""
|
||||
download.sub_downloader = video_downloader.name
|
||||
download.info_json = executed.info_json
|
||||
if executed.collection_id and not download.collection_id:
|
||||
download.collection_id = executed.collection_id
|
||||
|
||||
if executed.error:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
location=executed.location,
|
||||
error=executed.error,
|
||||
settings=executed.child_settings,
|
||||
)
|
||||
|
||||
downloads = self.get_missing_videos(session, download)
|
||||
return DownloadResult(
|
||||
success=True,
|
||||
location=executed.location,
|
||||
downloads=downloads,
|
||||
settings=executed.child_settings,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
|
||||
"""Update the Channel's video catalog. Refresh the Channel's files if necessary."""
|
||||
logger.debug(f'Preparing {channel} for downloads')
|
||||
channel_id = channel.id
|
||||
|
||||
update_channel_catalog(channel, download.info_json)
|
||||
async def prepare_channel_for_downloads(cls, info_json: dict, channel_id: int):
|
||||
"""Update the Channel's video catalog. Refresh the Channel's files if necessary.
|
||||
|
||||
Takes channel_id (not Channel) and info_json (not Download) so the caller doesn't
|
||||
have to hold the manager's session open while we await the file_worker job.
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
channel = get_channel(session, channel_id=channel_id, return_dict=False)
|
||||
logger.debug(f'Preparing {channel} for downloads')
|
||||
|
||||
update_channel_catalog(channel, info_json)
|
||||
|
||||
if not channel.videos or not channel.refreshed:
|
||||
logger.warning(f'Refreshing videos in {channel.directory} for channel {channel}')
|
||||
job_id = channel.refresh_files(channel.id, send_events=False)
|
||||
|
|
@ -589,6 +655,42 @@ class ChannelDownloader(Downloader, ABC):
|
|||
return downloads
|
||||
|
||||
|
||||
@dataclass
|
||||
class PreparedVideo:
|
||||
"""Plan produced by VideoDownloader.prepare_download.
|
||||
|
||||
`info_json` is None unless the upstream download row already has it (e.g. set by
|
||||
ChannelDownloader before queuing this child). `location` is the existing
|
||||
Video.location if any, used for error attribution if the new download fails.
|
||||
"""
|
||||
url: str # normalized
|
||||
settings: dict
|
||||
tag_names: List[str] = field(default_factory=list)
|
||||
destination: Optional[pathlib.Path] = None
|
||||
info_json: Optional[dict] = None
|
||||
location: Optional[str] = None # existing video, for error reports
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutedVideo:
|
||||
"""Output of VideoDownloader.execute_download once the file is on disk and validated.
|
||||
|
||||
Carries everything finalize_download needs to persist the Video record without
|
||||
re-running any I/O. When the metadata-only flow runs, execute returns a
|
||||
DownloadResult directly and the dispatch skips finalize_download entirely.
|
||||
"""
|
||||
url: str # normalized
|
||||
video_path: pathlib.Path
|
||||
video_paths: List[pathlib.Path] # all files in the FileGroup, post-glob+normalize
|
||||
info_json: dict
|
||||
entry: dict
|
||||
channel_id: Optional[int]
|
||||
audio_only: bool
|
||||
tag_names: List[str]
|
||||
settings: dict
|
||||
location: Optional[str] # for error reporting
|
||||
|
||||
|
||||
class VideoDownloader(Downloader, ABC):
|
||||
"""Downloads a single video.
|
||||
|
||||
|
|
@ -608,84 +710,126 @@ class VideoDownloader(Downloader, ABC):
|
|||
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
|
||||
return file_groups
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
def prepare_download(self, session: Session, download: Download) -> PreparedVideo:
|
||||
"""Sync prep: validate attempts cap, normalize URL, copy upstream-provided settings
|
||||
onto the Download row, and look up any pre-existing Video for error attribution.
|
||||
|
||||
Holding the session here is brief: no network or subprocess work runs in this
|
||||
phase. extract_info and channel resolution move to execute_download.
|
||||
"""
|
||||
if (download.attempts or 0) >= 10:
|
||||
raise UnrecoverableDownloadError('Max download attempts reached')
|
||||
|
||||
url = normalize_video_url(download.url)
|
||||
|
||||
# ChannelDownloader passes destination/tag_names via settings; copy onto the row
|
||||
# so the rest of the flow can read them off the Download object as before.
|
||||
settings = download.settings or dict()
|
||||
download.destination = download.destination or settings.get('destination')
|
||||
tag_names = download.tag_names = download.tag_names or settings.get('tag_names') or []
|
||||
|
||||
# If a Video already exists at this URL, capture its location so a failure later
|
||||
# can still tell the user where the previous attempt landed.
|
||||
existing = Video.get_by_url(session, url)
|
||||
location = existing.location if existing else None
|
||||
|
||||
return PreparedVideo(
|
||||
url=url,
|
||||
settings=settings,
|
||||
tag_names=list(tag_names),
|
||||
destination=download.destination,
|
||||
info_json=download.info_json,
|
||||
location=location,
|
||||
)
|
||||
|
||||
async def execute_download(self, prepared: PreparedVideo, ctx: DownloadContext,
|
||||
download: Download = None):
|
||||
"""Async I/O phase: extract_info, channel resolution (brief session),
|
||||
yt-dlp subprocess, file resolution, ffmpeg validation, sibling file globbing.
|
||||
|
||||
Returns either an ExecutedVideo (file is on disk; finalize persists) or a
|
||||
DownloadResult (an early-exit case the dispatch will surface verbatim — used
|
||||
for the metadata-only refresh flow and recoverable failures like a missing
|
||||
file or incomplete stream).
|
||||
"""
|
||||
_log_ytdlp_versions()
|
||||
with get_db_session() as session:
|
||||
if download.attempts >= 10:
|
||||
raise UnrecoverableDownloadError('Max download attempts reached')
|
||||
|
||||
url = normalize_video_url(download.url)
|
||||
|
||||
# Copy settings into Download (they may be from the ChannelDownloader)
|
||||
settings = download.settings or dict()
|
||||
download.destination = download.destination or settings.get('destination')
|
||||
tag_names = download.tag_names = download.tag_names or settings.get('tag_names')
|
||||
|
||||
# Video may have been downloaded previously, get its location for error reporting.
|
||||
video_ = Video.get_by_url(session, url)
|
||||
location = video_.location if video_ else None
|
||||
url = prepared.url
|
||||
settings = prepared.settings
|
||||
location = prepared.location
|
||||
|
||||
# extract_info is sync but expensive; legacy held a session across it. We don't.
|
||||
info_json = prepared.info_json
|
||||
if not info_json:
|
||||
try:
|
||||
download.info_json = download.info_json or extract_info(url)
|
||||
info_json = extract_info(url)
|
||||
except yt_dlp.utils.DownloadError as e:
|
||||
if _bot_blocked(e):
|
||||
raise BotBlockedDownloadError(str(e)) from e
|
||||
# Video may be private.
|
||||
# Video may be private — surface as a recoverable failure.
|
||||
try:
|
||||
raise RuntimeError('Failed to extract_info') from e
|
||||
except Exception as e:
|
||||
except Exception as e2:
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
location=location,
|
||||
error='\n'.join(traceback.format_exception(e)),
|
||||
error='\n'.join(traceback.format_exception(e2)),
|
||||
)
|
||||
|
||||
if not download.info_json:
|
||||
raise ValueError(f'Cannot download video with no info_json.')
|
||||
if not info_json:
|
||||
raise ValueError(f'Cannot download video with no info_json.')
|
||||
|
||||
channel = await self._get_channel(session, download)
|
||||
# Brief session for channel resolution. _get_channel reads info_json (already
|
||||
# populated above) and may create a Channel record.
|
||||
download_for_channel = download if download is not None else Download(
|
||||
url=url, settings=settings, destination=prepared.destination,
|
||||
)
|
||||
download_for_channel.info_json = info_json
|
||||
with get_db_session(commit=True) as session:
|
||||
channel = await self._get_channel(session, download_for_channel)
|
||||
channel_id = channel.id if channel else None
|
||||
destination = download.destination
|
||||
channel_directory = channel.directory if channel else None
|
||||
|
||||
if destination:
|
||||
# Download to the directory specified in the settings.
|
||||
out_dir = destination
|
||||
logger.debug(f'Downloading {url} to destination {destination} from settings')
|
||||
elif channel:
|
||||
out_dir = channel.directory
|
||||
logger.debug(f'Downloading {url} to channel directory: {channel.directory}')
|
||||
else:
|
||||
# Download to the default directory if this video has no channel.
|
||||
out_dir = get_no_channel_directory()
|
||||
logger.debug(f'Downloading {url} to default directory')
|
||||
if prepared.destination:
|
||||
out_dir = prepared.destination
|
||||
logger.debug(f'Downloading {url} to destination {prepared.destination} from settings')
|
||||
elif channel_directory:
|
||||
out_dir = channel_directory
|
||||
logger.debug(f'Downloading {url} to channel directory: {channel_directory}')
|
||||
else:
|
||||
out_dir = get_no_channel_directory()
|
||||
logger.debug(f'Downloading {url} to default directory')
|
||||
out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir)
|
||||
out_dir = get_absolute_media_path(out_dir)
|
||||
out_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
# Make output directory. (Maybe string from settings)
|
||||
out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir)
|
||||
out_dir = get_absolute_media_path(out_dir)
|
||||
out_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
if tag_names and logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f'Downloading {url} with {tag_names=}')
|
||||
if prepared.tag_names and logger.isEnabledFor(logging.DEBUG):
|
||||
logger.debug(f'Downloading {url} with tag_names={prepared.tag_names}')
|
||||
|
||||
effective = get_effective_video_settings(settings)
|
||||
audio_only = effective.get('audio_only', False)
|
||||
audio_format = effective.get('audio_format', 'mp3')
|
||||
|
||||
logs = None # noqa
|
||||
# download_for_log carries .id (for progress) and .url (for log messages).
|
||||
# Falling back to a stub keeps unit tests viable.
|
||||
download_for_log = download if download is not None else Download(url=url)
|
||||
|
||||
try:
|
||||
video_resolutions = effective['video_resolutions']
|
||||
# yt-dlp expects a string like so: 299+140,298+140,bestvideo*+bestaudio/best
|
||||
video_resolutions_str = ','.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i])
|
||||
|
||||
video_format = effective['video_format']
|
||||
video_path, entry = self.prepare_filename(url, out_dir, video_resolutions_str, video_format,
|
||||
audio_only=audio_only, audio_format=audio_format)
|
||||
|
||||
if settings.get('download_metadata_only'):
|
||||
# User has requested refresh of video metadata, skip the rest of the video downloader.
|
||||
return await self.download_info_json(download, video_path)
|
||||
# download_info_json manages its own DB sessions; we surface its DownloadResult
|
||||
# directly and the dispatch skips finalize_download.
|
||||
download_for_metadata = download if download is not None else Download(
|
||||
url=url, settings=settings,
|
||||
)
|
||||
download_for_metadata.info_json = info_json
|
||||
return await self.download_info_json(download_for_metadata, video_path)
|
||||
|
||||
# Common yt-dlp flags shared by both video and audio-only downloads.
|
||||
cmd = (
|
||||
str(YT_DLP_BIN),
|
||||
'--match-filter', '!is_live',
|
||||
|
|
@ -719,27 +863,26 @@ class VideoDownloader(Downloader, ABC):
|
|||
if effective['sleep_requests'] > 0:
|
||||
cmd = (*cmd, '--sleep-requests', str(effective['sleep_requests']))
|
||||
|
||||
on_stdout = make_progress_callback(download.id, parse_ytdlp_progress)
|
||||
on_stdout = make_progress_callback(download_for_log.id, parse_ytdlp_progress)
|
||||
|
||||
try:
|
||||
if cookies_unlocked():
|
||||
# Use encrypted cookies file
|
||||
logger.info(f'Using encrypted cookies for video download: {url}')
|
||||
with cookies_for_download() as cookies_path:
|
||||
cmd = (*cmd, '--cookies', str(cookies_path))
|
||||
if effective['user_agent']:
|
||||
cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
|
||||
cmd = (*cmd, '-o', video_path, url)
|
||||
result = await self.process_runner(download, cmd, out_dir, debug=True,
|
||||
stdout_callback=on_stdout)
|
||||
result = await self.process_runner(download_for_log, cmd, out_dir, debug=True,
|
||||
stdout_callback=on_stdout, ctx=ctx)
|
||||
else:
|
||||
# No cookies available
|
||||
logger.info(f'No cookies available for video download: {url}')
|
||||
cmd = (*cmd, '-o', video_path, url)
|
||||
result = await self.process_runner(download, cmd, out_dir, debug=True,
|
||||
stdout_callback=on_stdout)
|
||||
result = await self.process_runner(download_for_log, cmd, out_dir, debug=True,
|
||||
stdout_callback=on_stdout, ctx=ctx)
|
||||
finally:
|
||||
clear_download_progress(download.id)
|
||||
clear_download_progress(download_for_log.id)
|
||||
|
||||
stdout = result.stdout.decode()
|
||||
stderr = result.stderr.decode()
|
||||
|
||||
|
|
@ -747,36 +890,27 @@ class VideoDownloader(Downloader, ABC):
|
|||
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}'
|
||||
if _bot_blocked(error):
|
||||
raise BotBlockedDownloadError(f'Bot detection or invalid cookies: {error}')
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=error,
|
||||
location=location,
|
||||
)
|
||||
return DownloadResult(success=False, error=error, location=location)
|
||||
|
||||
output_format = audio_output_extension(audio_format) if audio_only else video_format
|
||||
preferred_path = video_path.with_suffix(f'.{output_format}')
|
||||
if not video_path.is_file() and preferred_path.is_file():
|
||||
# Prepared filename does not exist, but file with preferred extension does, it was probably
|
||||
# remuxed/converted by yt-dlp.
|
||||
video_path = preferred_path
|
||||
logger.info(f'Using preferred file which exists: {preferred_path}')
|
||||
|
||||
if not video_path.is_file():
|
||||
# yt-dlp may have appended the real extension to a bogus one (e.g. .NA.mp4 from a playlist entry).
|
||||
# yt-dlp may have appended the real extension to a bogus one.
|
||||
appended_path = video_path.parent / f'{video_path.name}.{output_format}'
|
||||
if appended_path.is_file():
|
||||
video_path = appended_path
|
||||
logger.info(f'Using file with appended extension: {appended_path}')
|
||||
|
||||
if video_path.suffix == '.part':
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=f'Video file that completed was a .part file: {video_path}',
|
||||
)
|
||||
return DownloadResult(success=False,
|
||||
error=f'Video file that completed was a .part file: {video_path}')
|
||||
|
||||
if not video_path.is_file():
|
||||
error = f'{stdout}\n\n\n{stderr}\n\n' \
|
||||
f'Video file could not be found! {video_path}'
|
||||
error = f'{stdout}\n\n\n{stderr}\n\nVideo file could not be found! {video_path}'
|
||||
retry_seconds = None
|
||||
if '!is_live' in stdout:
|
||||
error = f'{stdout}\n\nVideo was live and did not finish downloading.'
|
||||
|
|
@ -784,82 +918,29 @@ class VideoDownloader(Downloader, ABC):
|
|||
if ' live event ' in stdout:
|
||||
error = f'{stdout}\n\nVideo will be live and did not finish downloading.'
|
||||
retry_seconds = 3600
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error=error,
|
||||
location=location,
|
||||
retry_seconds=retry_seconds,
|
||||
)
|
||||
return DownloadResult(success=False, error=error, location=location,
|
||||
retry_seconds=retry_seconds)
|
||||
|
||||
if not audio_only and not ffmpeg_video_complete(video_path):
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Video was incomplete',
|
||||
location=location,
|
||||
)
|
||||
return DownloadResult(success=False, error='Video was incomplete', location=location)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Find any files downloaded with the video (poster, caption, etc.).
|
||||
video_paths = glob_shared_stem(video_path)
|
||||
# Delete any leftover part files now that we have verified the video is complete.
|
||||
video_paths = self._delete_part_files(video_path, video_paths)
|
||||
# Rename any special files that do not match the stem.
|
||||
video_paths = self.normalize_video_file_names(video_path, video_paths)
|
||||
# Create the Video record.
|
||||
video = Video.from_paths(session, *video_paths)
|
||||
video.source_id = entry['id']
|
||||
video.channel_id = channel_id
|
||||
video_id, video_info_json_path = video.id, video.info_json_path
|
||||
if video_info_json_path and (new_info_json := video.clean_info_json()):
|
||||
video.replace_info_json(new_info_json, clean=False)
|
||||
# File is on disk and complete. Glob siblings, drop part files, normalize names.
|
||||
video_paths = glob_shared_stem(video_path)
|
||||
video_paths = self._delete_part_files(video_path, video_paths)
|
||||
video_paths = self.normalize_video_file_names(video_path, video_paths)
|
||||
|
||||
if parent_download_url := settings.get('parent_download_url'):
|
||||
try:
|
||||
video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
|
||||
except ValueError as e:
|
||||
logger.warning(f'Failed to update wrolpi json for {video}: {e}')
|
||||
|
||||
# Update channel's collection's file_format to track the format used
|
||||
if channel_id:
|
||||
from modules.videos.models import Channel
|
||||
from modules.videos.lib import save_channels_config
|
||||
channel_obj = session.query(Channel).get(channel_id)
|
||||
if channel_obj and channel_obj.collection and not channel_obj.collection.file_format:
|
||||
video_config = get_videos_downloader_config()
|
||||
channel_obj.collection.file_format = video_config.file_name_format
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
session.commit()
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Second session is started because SQLAlchemy will forget what we have done.
|
||||
video = Video.get_by_id(session, video_id)
|
||||
if tag_names:
|
||||
existing_names = video.file_group.tag_names
|
||||
for name in tag_names:
|
||||
if name not in existing_names:
|
||||
video.add_tag(session, name)
|
||||
location = video.location
|
||||
|
||||
await video.get_ffprobe_json()
|
||||
|
||||
# Check that the download has the expected streams.
|
||||
if not audio_only and not video.get_streams_by_codec_type('video'):
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Video was downloaded but did not contain video stream',
|
||||
location=location,
|
||||
)
|
||||
if not video.get_streams_by_codec_type('audio'):
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Download did not contain audio stream',
|
||||
location=location,
|
||||
)
|
||||
logger.info(f'Successfully downloaded video {url} {video}')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path)
|
||||
return ExecutedVideo(
|
||||
url=url,
|
||||
video_path=video_path,
|
||||
video_paths=video_paths,
|
||||
info_json=info_json,
|
||||
entry=entry,
|
||||
channel_id=channel_id,
|
||||
audio_only=audio_only,
|
||||
tag_names=prepared.tag_names,
|
||||
settings=settings,
|
||||
location=location,
|
||||
)
|
||||
|
||||
except BotBlockedDownloadError:
|
||||
raise
|
||||
|
|
@ -874,30 +955,73 @@ class VideoDownloader(Downloader, ABC):
|
|||
if _skip_download(e):
|
||||
# The video failed to download, and the error will never be fixed. Skip it forever.
|
||||
try:
|
||||
source_id = download.info_json.get('id')
|
||||
source_id = (info_json or {}).get('id')
|
||||
logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
|
||||
f'attempt to download it again.')
|
||||
download.add_to_skip_list()
|
||||
if download is not None:
|
||||
download.add_to_skip_list()
|
||||
except Exception:
|
||||
# Could not skip this video, it may not have a channel.
|
||||
logger.warning(f'Could not skip video {url}')
|
||||
|
||||
# Skipped downloads should not be tried again.
|
||||
raise UnrecoverableDownloadError() from e
|
||||
# Download did not succeed, try again later.
|
||||
if logs and (stderr := logs.get('stderr')):
|
||||
error = f'{stderr}\n\n{traceback.format_exc()}'
|
||||
else:
|
||||
error = str(traceback.format_exc())
|
||||
return DownloadResult(success=False, error=error, location=location)
|
||||
return DownloadResult(success=False, error=str(traceback.format_exc()), location=location)
|
||||
|
||||
logger.debug(f'Downloaded video {location=}')
|
||||
async def finalize_download(self, session: Session, download: Download,
|
||||
executed: ExecutedVideo) -> DownloadResult:
|
||||
"""Async DB-bound persistence: create the Video record, normalize info_json,
|
||||
update the channel's collection format, apply tags, run ffprobe, validate
|
||||
streams, and dedupe. Caller commits.
|
||||
"""
|
||||
# Create the Video record.
|
||||
video = Video.from_paths(session, *executed.video_paths)
|
||||
video.source_id = executed.entry['id']
|
||||
video.channel_id = executed.channel_id
|
||||
if video.info_json_path and (new_info_json := video.clean_info_json()):
|
||||
video.replace_info_json(new_info_json, clean=False)
|
||||
|
||||
result = DownloadResult(
|
||||
success=True,
|
||||
location=location,
|
||||
)
|
||||
return result
|
||||
if parent_download_url := executed.settings.get('parent_download_url'):
|
||||
try:
|
||||
video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
|
||||
except ValueError as e:
|
||||
logger.warning(f'Failed to update wrolpi json for {video}: {e}')
|
||||
|
||||
# Update channel's collection's file_format to track the format used.
|
||||
if executed.channel_id:
|
||||
from modules.videos.lib import save_channels_config
|
||||
channel_obj = session.query(Channel).get(executed.channel_id)
|
||||
if channel_obj and channel_obj.collection and not channel_obj.collection.file_format:
|
||||
video_config = get_videos_downloader_config()
|
||||
channel_obj.collection.file_format = video_config.file_name_format
|
||||
save_channels_config.activate_switch()
|
||||
|
||||
# Apply tags now that the file_group is bound to the session.
|
||||
if executed.tag_names:
|
||||
existing_names = video.file_group.tag_names
|
||||
for name in executed.tag_names:
|
||||
if name not in existing_names:
|
||||
video.add_tag(session, name)
|
||||
|
||||
location = video.location
|
||||
|
||||
# ffprobe runs the ffprobe subprocess (may persist .ffprobe.json on disk).
|
||||
await video.get_ffprobe_json()
|
||||
|
||||
if not executed.audio_only and not video.get_streams_by_codec_type('video'):
|
||||
return DownloadResult(success=False,
|
||||
error='Video was downloaded but did not contain video stream',
|
||||
location=location)
|
||||
if not video.get_streams_by_codec_type('audio'):
|
||||
return DownloadResult(success=False,
|
||||
error='Download did not contain audio stream',
|
||||
location=location)
|
||||
|
||||
logger.info(f'Successfully downloaded video {executed.url} {video}')
|
||||
|
||||
# Delete duplicate Videos that share the URL or source_id. Runs against the same
|
||||
# session as the new Video record so deletion is atomic with creation.
|
||||
await Video.delete_duplicate_videos(session, download.url, executed.entry['id'],
|
||||
executed.video_path)
|
||||
|
||||
return DownloadResult(success=True, location=location)
|
||||
|
||||
@staticmethod
|
||||
async def _get_channel(session: Session, download: Download) -> Optional[Channel]:
|
||||
|
|
|
|||
|
|
@ -482,7 +482,9 @@ async def test_channel_downloader_rejects_rss_feed_url(test_session, mock_video_
|
|||
}
|
||||
|
||||
with pytest.raises(UnrecoverableDownloadError):
|
||||
await channel_downloader.do_download(download)
|
||||
# RSS-feed rejection happens in the prepare phase (sync), so we don't even
|
||||
# need to drive execute_download here.
|
||||
channel_downloader.prepare_download(test_session, download)
|
||||
|
||||
# No Channel should have been created from the RSS URL.
|
||||
assert test_session.query(Channel).count() == 0
|
||||
|
|
@ -668,9 +670,13 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
|
|||
|
||||
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.
|
||||
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download:
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download:
|
||||
mock_video_do_download.return_value = DownloadResult(success=True) # Don't download the videos.
|
||||
result = await channel_downloader.do_download(download)
|
||||
# Drive the phase-split API directly: prepare → execute → finalize.
|
||||
from modules.archive.conftest import make_test_ctx
|
||||
prepared = channel_downloader.prepare_download(test_session, download)
|
||||
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
|
||||
result = channel_downloader.finalize_download(test_session, download, executed)
|
||||
assert result.success is True, 'Download was not successful'
|
||||
assert set(result.downloads) == {
|
||||
'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL.
|
||||
|
|
@ -899,11 +905,14 @@ async def test_channel_downloader_sets_collection_id(test_session, channel_facto
|
|||
mock_video_extract_info.return_value = mock_info
|
||||
|
||||
# Mock Channel.refresh_files and file_worker.wait_for_job to avoid background tasks
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \
|
||||
mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
|
||||
mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
|
||||
mock_video_do_download.return_value = DownloadResult(success=True)
|
||||
await channel_downloader.do_download(download)
|
||||
from modules.archive.conftest import make_test_ctx
|
||||
prepared = channel_downloader.prepare_download(test_session, download)
|
||||
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
|
||||
channel_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
await await_switches()
|
||||
|
||||
|
|
@ -947,11 +956,14 @@ async def test_channel_passes_all_inheritable_settings_to_video(test_session, ch
|
|||
}
|
||||
mock_video_extract_info.return_value = mock_info
|
||||
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \
|
||||
mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
|
||||
mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
|
||||
mock_video_do_download.return_value = DownloadResult(success=True)
|
||||
result = await channel_downloader.do_download(download)
|
||||
from modules.archive.conftest import make_test_ctx
|
||||
prepared = channel_downloader.prepare_download(test_session, download)
|
||||
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
|
||||
result = channel_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
# Verify all channel-level inheritable settings are passed through
|
||||
for key in ('video_resolutions', 'video_format', 'writesubtitles', 'sleep_requests', 'user_agent'):
|
||||
|
|
@ -991,11 +1003,14 @@ async def test_channel_does_not_pass_continue_dl_nooverwrites(test_session, chan
|
|||
}
|
||||
mock_video_extract_info.return_value = mock_info
|
||||
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \
|
||||
mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
|
||||
mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
|
||||
mock_video_do_download.return_value = DownloadResult(success=True)
|
||||
result = await channel_downloader.do_download(download)
|
||||
from modules.archive.conftest import make_test_ctx
|
||||
prepared = channel_downloader.prepare_download(test_session, download)
|
||||
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
|
||||
result = channel_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
assert 'continue_dl' not in result.settings, 'continue_dl should not be passed to child video downloads'
|
||||
assert 'nooverwrites' not in result.settings, 'nooverwrites should not be passed to child video downloads'
|
||||
|
|
@ -1063,7 +1078,12 @@ async def test_live_video_retry_seconds(test_session, test_directory, mock_video
|
|||
mock_prepare_filename.return_value = (simple_channel.directory / 'live video.mp4', {'id': 'foo'})
|
||||
|
||||
download = Download(url=url, downloader='video', attempts=0)
|
||||
result = await video_downloader.do_download(download)
|
||||
# Drive the phase-split API directly: when execute_download detects a live video
|
||||
# it short-circuits with a DownloadResult (the dispatch surfaces it verbatim and
|
||||
# skips finalize_download).
|
||||
from modules.archive.conftest import make_test_ctx
|
||||
prepared = video_downloader.prepare_download(test_session, download)
|
||||
result = await video_downloader.execute_download(prepared, make_test_ctx(), download=download)
|
||||
|
||||
assert result.success is False
|
||||
assert result.retry_seconds == 3600
|
||||
|
|
@ -1252,3 +1272,70 @@ async def test_channel_download_preserves_skip_list(test_session, test_directory
|
|||
once_downloads = video_download_manager.get_once_downloads(test_session)
|
||||
video_downloads = [i for i in once_downloads if 'watch' in i.url]
|
||||
assert {i.url for i in video_downloads} == {'https://youtube.com/watch?v=video_2_url'}
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase-split unit tests for VideoDownloader.prepare_download.
|
||||
#
|
||||
# These exercise the sync prep invariants in isolation — no async_client, no
|
||||
# extract_info, no subprocess. Heavier execute/finalize behavior is covered by
|
||||
# the integration tests above (which drive through video_download_manager).
|
||||
# ---------------------------------------------------------------------------
|
||||
from modules.videos.downloader import PreparedVideo # noqa: E402
|
||||
|
||||
|
||||
def test_prepare_max_attempts_raises(test_session):
|
||||
"""prepare_download enforces the 10-attempt cap (videos retry more than other
|
||||
downloaders because YouTube transients are common)."""
|
||||
download = Download(url='https://youtube.com/watch?v=foo', downloader='video', attempts=10)
|
||||
|
||||
with pytest.raises(UnrecoverableDownloadError, match='Max download attempts'):
|
||||
video_downloader.prepare_download(test_session, download)
|
||||
|
||||
|
||||
def test_prepare_normalizes_url(test_session):
|
||||
"""Shorts URLs become watch URLs in the PreparedVideo (matches DB-stored format)."""
|
||||
download = Download(
|
||||
url='https://www.youtube.com/shorts/ABC123',
|
||||
downloader='video',
|
||||
settings={},
|
||||
)
|
||||
|
||||
prepared = video_downloader.prepare_download(test_session, download)
|
||||
|
||||
assert isinstance(prepared, PreparedVideo)
|
||||
assert prepared.url == 'https://www.youtube.com/watch?v=ABC123'
|
||||
|
||||
|
||||
def test_prepare_copies_upstream_settings_onto_download(test_session):
|
||||
"""ChannelDownloader passes destination + tag_names via settings; prep copies them
|
||||
onto the Download row so downstream code sees them as columns."""
|
||||
download = Download(
|
||||
url='https://youtube.com/watch?v=foo',
|
||||
downloader='video',
|
||||
destination=None,
|
||||
tag_names=None,
|
||||
settings={'destination': '/media/wrolpi/videos/foo', 'tag_names': ['news', 'tech']},
|
||||
)
|
||||
|
||||
prepared = video_downloader.prepare_download(test_session, download)
|
||||
|
||||
assert download.destination == '/media/wrolpi/videos/foo'
|
||||
assert download.tag_names == ['news', 'tech']
|
||||
assert prepared.tag_names == ['news', 'tech']
|
||||
|
||||
|
||||
def test_prepare_uses_existing_video_location(test_session, video_factory):
|
||||
"""If a Video already exists at the URL, prep captures its location for error
|
||||
attribution (so a failed re-download still tells the user where the previous file is)."""
|
||||
url = 'https://youtube.com/watch?v=existing'
|
||||
existing = video_factory(with_video_file=True)
|
||||
existing.file_group.url = url
|
||||
test_session.commit()
|
||||
|
||||
download = Download(url=url, downloader='video', settings={})
|
||||
|
||||
prepared = video_downloader.prepare_download(test_session, download)
|
||||
|
||||
assert prepared.location == existing.location
|
||||
assert prepared.location is not None
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
|||
import wrolpi.root_api # noqa
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.cmd import CommandResult
|
||||
from wrolpi.cmd import CommandResult, run_command
|
||||
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL, \
|
||||
get_wrolpi_config
|
||||
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
|
||||
|
|
@ -43,7 +43,7 @@ from wrolpi.common import set_test_media_directory, Base, set_test_config
|
|||
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
|
||||
from wrolpi.dates import set_test_now
|
||||
from wrolpi.db import postgres_engine, get_db_args
|
||||
from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \
|
||||
from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \
|
||||
downloads_manager_config_context, download_cache_config_context
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from wrolpi.files.models import Directory, FileGroup
|
||||
|
|
@ -378,16 +378,52 @@ def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
|
|||
return asserter
|
||||
|
||||
|
||||
def make_test_ctx(*, is_cancelled=None, can_download=None,
|
||||
outside_download_window=None, run_command_=None,
|
||||
download_timeout=None) -> DownloadContext:
|
||||
"""Build a DownloadContext suitable for downloader unit tests.
|
||||
|
||||
Defaults: never-cancelled, can_download True, no window restriction, real run_command,
|
||||
no global timeout. Override any callable to simulate cancellation, throttling, or
|
||||
fake subprocess output.
|
||||
|
||||
Lives in wrolpi/conftest.py so any test (including primitive tests of process_runner /
|
||||
cancel_wrapper) can construct a ctx without crossing module boundaries.
|
||||
"""
|
||||
return DownloadContext(
|
||||
is_cancelled=is_cancelled or (lambda: False),
|
||||
can_download=can_download or (lambda: True),
|
||||
outside_download_window=outside_download_window or (lambda: False),
|
||||
run_command=run_command_ or run_command,
|
||||
download_timeout=download_timeout,
|
||||
)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_downloader(test_download_manager) -> Downloader:
|
||||
class TestDownloader(Downloader, ABC):
|
||||
"""A testing Downloader"""
|
||||
"""A testing Downloader.
|
||||
|
||||
prepare_download returns a non-None sentinel so the manager's phase-split dispatch
|
||||
routes through this subclass. execute_download is an AsyncMock — tests wire its
|
||||
side_effect via the set_test_* methods. Returning a DownloadResult from
|
||||
execute_download tells the dispatch to skip finalize (see signal_download_download).
|
||||
"""
|
||||
name = 'test_downloader'
|
||||
|
||||
def __repr__(self):
|
||||
return '<TESTING Downloader>'
|
||||
|
||||
do_download = AsyncMock()
|
||||
execute_download = AsyncMock()
|
||||
|
||||
def prepare_download(self, session, download):
|
||||
return object() # sentinel; activates the phase-split path
|
||||
|
||||
def finalize_download(self, session, download, executed):
|
||||
# execute_download returns a DownloadResult directly, so the dispatch
|
||||
# short-circuits before reaching finalize. Provided here only to satisfy
|
||||
# the abstract contract.
|
||||
return DownloadResult(success=True)
|
||||
|
||||
def set_test_success(self):
|
||||
async def _(*a, **kwargs):
|
||||
|
|
@ -395,7 +431,7 @@ def test_downloader(test_download_manager) -> Downloader:
|
|||
await asyncio.sleep(1)
|
||||
return DownloadResult(success=True)
|
||||
|
||||
self.do_download.side_effect = _
|
||||
self.execute_download.side_effect = _
|
||||
|
||||
def set_test_failure(self):
|
||||
async def _(*a, **kwargs):
|
||||
|
|
@ -403,7 +439,7 @@ def test_downloader(test_download_manager) -> Downloader:
|
|||
await asyncio.sleep(1)
|
||||
return DownloadResult(success=False)
|
||||
|
||||
self.do_download.side_effect = _
|
||||
self.execute_download.side_effect = _
|
||||
|
||||
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
|
||||
async def _(*a, **kwargs):
|
||||
|
|
@ -411,7 +447,7 @@ def test_downloader(test_download_manager) -> Downloader:
|
|||
await asyncio.sleep(1)
|
||||
raise exception
|
||||
|
||||
self.do_download.side_effect = _
|
||||
self.execute_download.side_effect = _
|
||||
|
||||
def set_test_unrecoverable_exception(self):
|
||||
async def _(*a, **kwargs):
|
||||
|
|
@ -419,7 +455,7 @@ def test_downloader(test_download_manager) -> Downloader:
|
|||
await asyncio.sleep(1)
|
||||
raise UnrecoverableDownloadError()
|
||||
|
||||
self.do_download.side_effect = _
|
||||
self.execute_download.side_effect = _
|
||||
|
||||
test_downloader = TestDownloader()
|
||||
# Default to successful download.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,5 @@
|
|||
import asyncio
|
||||
import contextlib
|
||||
import inspect
|
||||
import logging
|
||||
import multiprocessing
|
||||
import os
|
||||
|
|
@ -419,23 +418,19 @@ class Downloader:
|
|||
def __repr__(self):
|
||||
return f'<Downloader name={self.name}>'
|
||||
|
||||
async def do_download(self, download: Download, ctx: 'DownloadContext' = None) -> DownloadResult:
|
||||
"""Legacy single-phase entry point.
|
||||
def prepare_download(self, session: Session, download: Download) -> Any:
|
||||
"""DB-bound prep. Caller (DownloadManager) opens and commits the session.
|
||||
|
||||
Subclasses opting in to the phase-split API override prepare_download /
|
||||
execute_download / finalize_download instead and leave this unimplemented.
|
||||
Sync by default; subclasses may override with `async def prepare_download` when
|
||||
they need network or subprocess work during prep (e.g. yt-dlp extract_info).
|
||||
The dispatch awaits the coroutine. Holding the session across an await means
|
||||
the connection is in use — keep async prep work brief.
|
||||
|
||||
Subclasses must override and return a value (typically a dataclass) that is
|
||||
passed unchanged to execute_download.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def prepare_download(self, session: Session, download: Download) -> Optional[Any]:
|
||||
"""Synchronous DB-bound prep. Caller (DownloadManager) opens and commits the session.
|
||||
|
||||
Returning None signals the subclass has not opted in to the phase-split API and
|
||||
DownloadManager should fall back to do_download. Returning any other value
|
||||
(typically a dataclass) is passed unchanged to execute_download.
|
||||
"""
|
||||
return None
|
||||
|
||||
async def execute_download(self, prepared: Any, ctx: 'DownloadContext',
|
||||
download: Download = None) -> Any:
|
||||
"""Async I/O-bound work. No DB session — does network and subprocess work,
|
||||
|
|
@ -448,7 +443,12 @@ class Downloader:
|
|||
raise NotImplementedError()
|
||||
|
||||
def finalize_download(self, session: Session, download: Download, executed: Any) -> DownloadResult:
|
||||
"""Synchronous DB-bound persistence. Caller opens and commits the session."""
|
||||
"""DB-bound persistence. Caller opens and commits the session.
|
||||
|
||||
Sync by default; subclasses may override with `async def finalize_download` when
|
||||
post-download persistence requires async work (e.g. ffprobe metadata, dedupe
|
||||
queries). The dispatch awaits the coroutine.
|
||||
"""
|
||||
raise NotImplementedError()
|
||||
|
||||
def already_downloaded(self, session: Session, *urls: List[str]):
|
||||
|
|
@ -465,25 +465,22 @@ class Downloader:
|
|||
|
||||
Global timeout takes precedence over the timeout argument, unless it is 0. (Smaller global timeout wins)
|
||||
|
||||
When ctx is provided, ctx.run_command and ctx.download_timeout are used and cancellation
|
||||
is polled through ctx. When ctx is None (legacy callers), behavior is unchanged.
|
||||
ctx is required from the dispatch path; the kwarg default of None is a convenience
|
||||
for unit tests that drive process_runner directly with a stub ctx.
|
||||
"""
|
||||
if cwd and not cwd.is_dir():
|
||||
raise RuntimeError('cwd directory does not exist')
|
||||
if ctx is None:
|
||||
raise RuntimeError('process_runner requires a DownloadContext')
|
||||
|
||||
if ctx is not None:
|
||||
config_timeout = ctx.download_timeout
|
||||
runner = ctx.run_command
|
||||
else:
|
||||
config_timeout = get_wrolpi_config().download_timeout
|
||||
runner = run_command
|
||||
config_timeout = ctx.download_timeout
|
||||
# 0 means no timeout. Prefer config timeout, then argument, then class default.
|
||||
if config_timeout:
|
||||
timeout = config_timeout
|
||||
elif timeout is None:
|
||||
timeout = self.timeout
|
||||
coro = runner(cmd, cwd=cwd, timeout=timeout, log_command=debug,
|
||||
stdout_callback=stdout_callback, env=env)
|
||||
coro = ctx.run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
|
||||
stdout_callback=stdout_callback, env=env)
|
||||
result = await self.cancel_wrapper(coro, download, ctx=ctx)
|
||||
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
|
|
@ -497,23 +494,17 @@ class Downloader:
|
|||
@staticmethod
|
||||
async def cancel_wrapper(coro: Coroutine, download: Download, ctx: 'DownloadContext' = None):
|
||||
"""
|
||||
Converts an async coroutine to a task. If DownloadManager receives a kill request, this method will cancel
|
||||
the task.
|
||||
|
||||
When ctx is provided, cancellation is polled through ctx.is_cancelled / ctx.can_download /
|
||||
ctx.outside_download_window. When ctx is None, the global download_manager singleton is
|
||||
used (legacy path for unmigrated subclasses).
|
||||
Converts an async coroutine to a task. If DownloadManager receives a kill request,
|
||||
this method will cancel the task. Cancellation state is read through ctx.
|
||||
"""
|
||||
if ctx is None:
|
||||
raise RuntimeError('cancel_wrapper requires a DownloadContext')
|
||||
download_id = download.id
|
||||
task = asyncio.create_task(coro)
|
||||
try:
|
||||
while not task.done():
|
||||
if ctx is not None:
|
||||
cancelled = ctx.is_cancelled() or not ctx.can_download()
|
||||
outside_window = ctx.outside_download_window()
|
||||
else:
|
||||
cancelled = download_manager.download_is_killed(download_id) or not download_manager.can_download
|
||||
outside_window = download_manager.outside_download_window
|
||||
cancelled = ctx.is_cancelled() or not ctx.can_download()
|
||||
outside_window = ctx.outside_download_window()
|
||||
if cancelled:
|
||||
logger.warning(f'Cancel download of {download.url}')
|
||||
task.cancel()
|
||||
|
|
@ -528,16 +519,13 @@ class Downloader:
|
|||
error=error,
|
||||
)
|
||||
finally:
|
||||
if ctx is None:
|
||||
# The killed-downloads list lives on the manager; clear via the
|
||||
# singleton. Tests that bypass the dispatch may not have a manager
|
||||
# singleton wired up — silently ignore in that case.
|
||||
try:
|
||||
download_manager.unkill_download(download_id)
|
||||
else:
|
||||
# Manager still owns the killed-downloads list; clear via singleton.
|
||||
# The ctx is read-only with respect to cancellation state.
|
||||
try:
|
||||
download_manager.unkill_download(download_id)
|
||||
except Exception:
|
||||
# In tests without a manager, silently ignore.
|
||||
pass
|
||||
except Exception:
|
||||
pass
|
||||
else:
|
||||
# Wait for the download to complete. Cancel if requested.
|
||||
await asyncio.sleep(0.1)
|
||||
|
|
@ -1539,30 +1527,33 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
|
||||
try_again = True
|
||||
try:
|
||||
# Phase-split path: subclass opts in by overriding prepare_download to return non-None.
|
||||
# Legacy path: do_download remains the single entry point.
|
||||
# Phase-split dispatch. Caller (us) owns transaction boundaries: a session
|
||||
# is opened for prepare_download (committing any pre-flight DB work like
|
||||
# destination resolution) and a fresh session for finalize_download (so we
|
||||
# don't hold a connection across the long-running execute_download).
|
||||
# cancel_wrapper may short-circuit with a DownloadResult instead of an
|
||||
# ExecutedX dataclass; subclasses also use that protocol to skip finalize
|
||||
# for early-exit cases (e.g. metadata-only video downloads).
|
||||
ctx = DownloadContext.production(download_manager, download_id)
|
||||
with get_db_session(commit=True) as s:
|
||||
prepared = downloader.prepare_download(s, download)
|
||||
# Subclasses with unavoidable async prep work (e.g. VideoDownloader's
|
||||
# extract_info / channel resolution) may declare prepare_download as
|
||||
# `async def`; await the coroutine here.
|
||||
if asyncio.iscoroutine(prepared):
|
||||
prepared = await prepared
|
||||
|
||||
if prepared is not None:
|
||||
# New phase-split path: caller (us) owns transaction boundaries.
|
||||
coro = downloader.execute_download(prepared, ctx, download=download)
|
||||
executed = await downloader.cancel_wrapper(coro, download, ctx=ctx)
|
||||
if isinstance(executed, DownloadResult):
|
||||
# cancel_wrapper short-circuited with a cancel result.
|
||||
result = executed
|
||||
else:
|
||||
with get_db_session(commit=True) as s:
|
||||
result = downloader.finalize_download(s, download, executed)
|
||||
coro = downloader.execute_download(prepared, ctx, download=download)
|
||||
executed = await downloader.cancel_wrapper(coro, download, ctx=ctx)
|
||||
if isinstance(executed, DownloadResult):
|
||||
# cancel_wrapper or execute_download short-circuited with a result.
|
||||
result = executed
|
||||
else:
|
||||
# Legacy path. Subclasses on this path have signature do_download(self, download)
|
||||
# without ctx, so we must NOT pass ctx as a positional arg. cancel_wrapper continues
|
||||
# to use the global download_manager (ctx=None).
|
||||
if not inspect.iscoroutinefunction(downloader.do_download):
|
||||
raise RuntimeError(f'Coroutine expected from {downloader} do_download method.')
|
||||
coro = downloader.do_download(download)
|
||||
result = await downloader.cancel_wrapper(coro, download)
|
||||
with get_db_session(commit=True) as s:
|
||||
result = downloader.finalize_download(s, download, executed)
|
||||
# finalize_download may also be async (same rationale as prepare).
|
||||
if asyncio.iscoroutine(result):
|
||||
result = await result
|
||||
except BotBlockedDownloadError as e:
|
||||
# Bot detection or invalid cookies - stop all pending downloads from this domain.
|
||||
worker_logger.warning(f'BotBlockedDownloadError for {url}', exc_info=e)
|
||||
|
|
@ -2048,6 +2039,34 @@ def parse_feed(url: str) -> FeedParserDict:
|
|||
return feedparser.parse(url)
|
||||
|
||||
|
||||
@dataclass
|
||||
class PreparedRSS:
|
||||
"""Plan produced by RSSDownloader.prepare_download.
|
||||
|
||||
sub_downloader is the resolved Downloader instance used both for already_downloaded
|
||||
DB queries and to detect the 'video' branch (where URL normalization + duration
|
||||
filtering applies). Carries the original Download stand-in fields execute and
|
||||
finalize need so neither phase has to reach back to the DB row.
|
||||
"""
|
||||
url: str
|
||||
sub_downloader_name: str
|
||||
sub_downloader: 'Downloader'
|
||||
settings: dict
|
||||
|
||||
|
||||
@dataclass
|
||||
class ExecutedRSS:
|
||||
"""Output of RSSDownloader.execute_download.
|
||||
|
||||
candidate_urls is the post-filter, post-duration-check URL list ready for DB dedupe
|
||||
in finalize. yt_channel_id (when present) lets finalize apply Channel info to the
|
||||
Download row. error covers feed-parse failures.
|
||||
"""
|
||||
yt_channel_id: Optional[str]
|
||||
candidate_urls: List[str]
|
||||
error: Optional[str] = None
|
||||
|
||||
|
||||
class RSSDownloader(Downloader):
|
||||
"""Downloads an RSS feed and creates new downloads for every unique link in the feed."""
|
||||
name = 'rss'
|
||||
|
|
@ -2057,27 +2076,34 @@ class RSSDownloader(Downloader):
|
|||
def __repr__(self):
|
||||
return '<RSSDownloader>'
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
def prepare_download(self, session: Session, download: Download) -> PreparedRSS:
|
||||
sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader)
|
||||
if not sub_downloader:
|
||||
raise ValueError(f'Unable to find sub_downloader for {download.url}')
|
||||
return PreparedRSS(
|
||||
url=download.url,
|
||||
sub_downloader_name=download.sub_downloader,
|
||||
sub_downloader=sub_downloader,
|
||||
settings=download.settings or dict(),
|
||||
)
|
||||
|
||||
feed: FeedParserDict = parse_feed(download.url)
|
||||
async def execute_download(self, prepared: PreparedRSS, ctx: 'DownloadContext',
|
||||
download: Download = None) -> ExecutedRSS:
|
||||
feed: FeedParserDict = parse_feed(prepared.url)
|
||||
if feed['bozo'] and not self.acceptable_bozo_errors(feed):
|
||||
# Feed did not parse
|
||||
return DownloadResult(success=False, error='Failed to parse RSS feed')
|
||||
return ExecutedRSS(yt_channel_id=None, candidate_urls=[],
|
||||
error='Failed to parse RSS feed')
|
||||
|
||||
if not isinstance(feed, dict) or not feed.get('entries'):
|
||||
# RSS parsed but does not have entries.
|
||||
return DownloadResult(success=False, error='RSS feed did not have any entries')
|
||||
return ExecutedRSS(yt_channel_id=None, candidate_urls=[],
|
||||
error='RSS feed did not have any entries')
|
||||
|
||||
# Apply YT channel to the Download, if not already applied.
|
||||
if yt_channel_id := feed.get('feed', dict()).get('yt_channelid'):
|
||||
if not (download.location or download.collection_id):
|
||||
self.apply_yt_channel(download.id, yt_channel_id)
|
||||
yt_channel_id = feed.get('feed', dict()).get('yt_channelid')
|
||||
|
||||
# Filter entries using Download.settings.
|
||||
entries = self.filter_entries(download, feed['entries'])
|
||||
# Filter entries using settings; preserve filter_entries' Download-keyed signature
|
||||
# by passing a stand-in (test_rss.py calls filter_entries directly with a Download).
|
||||
log_stub = Download(url=prepared.url, settings=prepared.settings)
|
||||
entries = self.filter_entries(log_stub, feed['entries'])
|
||||
|
||||
# Filter URL links.
|
||||
urls = []
|
||||
|
|
@ -2087,24 +2113,43 @@ class RSSDownloader(Downloader):
|
|||
else:
|
||||
logger.warning(f'RSS entry {idx} did not have a link!')
|
||||
|
||||
if download.sub_downloader == 'video':
|
||||
if prepared.sub_downloader_name == 'video':
|
||||
# Normalize video URLs before checking if already downloaded.
|
||||
# YouTube Shorts URLs (e.g., youtube.com/shorts/ABC123) must be converted to watch URLs
|
||||
# (e.g., youtube.com/watch?v=ABC123) to match what is stored in the database.
|
||||
from modules.videos.normalize_video_url import normalize_video_url
|
||||
urls = [normalize_video_url(i) for i in urls]
|
||||
# Duration filtering performs network I/O (yt-dlp duration probe) — must run
|
||||
# in execute, not finalize. This may probe URLs that DB-dedupe would have
|
||||
# eliminated; fetch_video_duration is cached, so duplicate probes are cheap.
|
||||
urls = await self.filter_videos(log_stub, urls)
|
||||
|
||||
# Only download new URLs.
|
||||
with get_db_session() as session:
|
||||
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(session, *urls)]]
|
||||
# Filter out URLs already in the Download table (pending downloads).
|
||||
return ExecutedRSS(yt_channel_id=yt_channel_id, candidate_urls=urls)
|
||||
|
||||
def finalize_download(self, session: Session, download: Download,
|
||||
executed: ExecutedRSS) -> DownloadResult:
|
||||
if executed.error:
|
||||
return DownloadResult(success=False, error=executed.error)
|
||||
|
||||
# Apply YT channel info to the Download, if not already applied.
|
||||
if executed.yt_channel_id and not (download.location or download.collection_id):
|
||||
self.apply_yt_channel(session, download, executed.yt_channel_id)
|
||||
|
||||
# Resolve sub_downloader via the manager (matches prepare; cheap dict lookup).
|
||||
sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader)
|
||||
urls = executed.candidate_urls
|
||||
|
||||
# DB dedupe: drop URLs already downloaded under this sub_downloader, plus URLs
|
||||
# already queued in the Download table.
|
||||
if sub_downloader and urls:
|
||||
already = sub_downloader.already_downloaded(session, *urls)
|
||||
already_urls = {a.url for a in already}
|
||||
urls = [u for u in urls if u not in already_urls]
|
||||
existing_download_urls = {d[0] for d in session.query(Download.url).filter(Download.url.in_(urls))}
|
||||
urls = [url for url in urls if url not in existing_download_urls]
|
||||
# Remove skipped URLs before duration checks. (Typically done after this by the download worker).
|
||||
urls = [i for i in urls if not download_manager.is_skipped(i)]
|
||||
urls = [u for u in urls if u not in existing_download_urls]
|
||||
|
||||
if download.sub_downloader == 'video': # VideoDownloader
|
||||
urls = await self.filter_videos(download, urls)
|
||||
# Skip-list filter (manager-owned shared_ctx state, not DB).
|
||||
urls = [u for u in urls if not download_manager.is_skipped(u)]
|
||||
|
||||
logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}')
|
||||
|
||||
|
|
@ -2123,12 +2168,7 @@ class RSSDownloader(Downloader):
|
|||
# Fallback to legacy settings for backwards compatibility
|
||||
next_download_settings['destination'] = i
|
||||
|
||||
result = DownloadResult(
|
||||
success=True,
|
||||
downloads=urls,
|
||||
settings=next_download_settings,
|
||||
)
|
||||
return result
|
||||
return DownloadResult(success=True, downloads=urls, settings=next_download_settings)
|
||||
|
||||
@staticmethod
|
||||
async def filter_videos(download: Download, urls: list[str]) -> list[str]:
|
||||
|
|
@ -2190,17 +2230,17 @@ class RSSDownloader(Downloader):
|
|||
return entries
|
||||
|
||||
@staticmethod
|
||||
def apply_yt_channel(download_id: int, yt_channel_id: str):
|
||||
"""Get Channel that matches this Download, apply Channel/Collection information to the Download."""
|
||||
with get_db_session() as session:
|
||||
from modules.videos.models import Channel
|
||||
channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}')
|
||||
if channel:
|
||||
download_ = Download.get_by_id(session, download_id)
|
||||
download_.collection = channel.collection
|
||||
download_.collection_id = channel.collection_id
|
||||
download_.location = download_.location or channel.location
|
||||
session.commit()
|
||||
def apply_yt_channel(session: Session, download: Download, yt_channel_id: str):
|
||||
"""Apply Channel/Collection information to the given Download row.
|
||||
|
||||
Caller (finalize_download) owns the session and commits.
|
||||
"""
|
||||
from modules.videos.models import Channel
|
||||
channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}')
|
||||
if channel:
|
||||
download.collection = channel.collection
|
||||
download.collection_id = channel.collection_id
|
||||
download.location = download.location or channel.location
|
||||
|
||||
@staticmethod
|
||||
def acceptable_bozo_errors(feed):
|
||||
|
|
|
|||
|
|
@ -53,7 +53,7 @@ async def test_scrape_html_downloader(test_directory, test_session, test_downloa
|
|||
assert_download_urls(['https://example.com/dir', ])
|
||||
|
||||
with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \
|
||||
mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download):
|
||||
mock.patch('wrolpi.files.downloader.FileDownloader.execute_download', fake_file_do_download):
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
await await_switches()
|
||||
|
||||
|
|
@ -77,7 +77,7 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do
|
|||
assert_download_urls(['https://example.com/dir', ])
|
||||
|
||||
with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \
|
||||
mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download):
|
||||
mock.patch('wrolpi.files.downloader.FileDownloader.execute_download', fake_file_do_download):
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
await await_switches()
|
||||
|
||||
|
|
|
|||
|
|
@ -115,6 +115,7 @@ async def test_make_progress_callback(async_client):
|
|||
@pytest.mark.asyncio
|
||||
async def test_cancel_wrapper_outer_cancellation(test_session, test_download_manager, test_downloader):
|
||||
"""cancel_wrapper propagates outer CancelledError and cancels the inner task."""
|
||||
from wrolpi.conftest import make_test_ctx
|
||||
download = test_download_manager.create_download(test_session, 'https://example.com/cancel', test_downloader.name)
|
||||
test_session.commit()
|
||||
|
||||
|
|
@ -127,7 +128,7 @@ async def test_cancel_wrapper_outer_cancellation(test_session, test_download_man
|
|||
inner_cancelled.set()
|
||||
raise
|
||||
|
||||
wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download))
|
||||
wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download, ctx=make_test_ctx()))
|
||||
# Give the wrapper time to start polling.
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
|
|
@ -461,7 +462,7 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
|
||||
# Download is processed and successful, no longer "new".
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
test_downloader.do_download.assert_called_once()
|
||||
test_downloader.execute_download.assert_called_once()
|
||||
assert list(test_download_manager.get_new_downloads(test_session)) == []
|
||||
downloads = list(test_download_manager.get_recurring_downloads(test_session))
|
||||
assert len(downloads) == 1
|
||||
|
|
@ -487,10 +488,10 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
assert download.last_successful_download == now_
|
||||
|
||||
# Try the download, but it fails.
|
||||
test_downloader.do_download.reset_mock()
|
||||
test_downloader.execute_download.reset_mock()
|
||||
test_downloader.set_test_failure()
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
test_downloader.do_download.assert_called_once()
|
||||
test_downloader.execute_download.assert_called_once()
|
||||
download = test_session.query(Download).one()
|
||||
# Download is deferred, last successful download remains the same.
|
||||
assert download.is_deferred, download.status_code
|
||||
|
|
@ -500,12 +501,12 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
|
|||
assert download.next_download == expected
|
||||
|
||||
# Try the download again, it finally succeeds.
|
||||
test_downloader.do_download.reset_mock()
|
||||
test_downloader.execute_download.reset_mock()
|
||||
now_ = fake_now(datetime(2020, 1, 1, 4, 0, 1, tzinfo=pytz.UTC))
|
||||
test_downloader.set_test_success()
|
||||
test_download_manager.renew_recurring_downloads()
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
test_downloader.do_download.assert_called_once()
|
||||
test_downloader.execute_download.assert_called_once()
|
||||
download = test_session.query(Download).one()
|
||||
assert download.is_complete, download.status_code
|
||||
assert download.last_successful_download == now_
|
||||
|
|
@ -664,31 +665,36 @@ async def test_skip_already_downloaded_disabled(test_session, test_download_mana
|
|||
@pytest.mark.asyncio
|
||||
async def test_process_runner_timeout(async_client, test_session, test_directory):
|
||||
"""A Downloader can cancel its download using a timeout."""
|
||||
from wrolpi.conftest import make_test_ctx
|
||||
# Default timeout of 3 seconds.
|
||||
downloader = Downloader('downloader', timeout=3)
|
||||
|
||||
# Sleep for 8 seconds really takes 8 seconds.
|
||||
start = datetime.now()
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=10)
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory,
|
||||
timeout=10, ctx=make_test_ctx())
|
||||
elapsed = datetime.now() - start
|
||||
assert elapsed.total_seconds() >= 8
|
||||
|
||||
# Download.timeout is obeyed.
|
||||
# Downloader.timeout (class default) is obeyed.
|
||||
start = datetime.now()
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory)
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, ctx=make_test_ctx())
|
||||
elapsed = datetime.now() - start
|
||||
assert 3 < elapsed.total_seconds() < 5
|
||||
|
||||
# One-off timeout is obeyed.
|
||||
start = datetime.now()
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=1)
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory,
|
||||
timeout=1, ctx=make_test_ctx())
|
||||
elapsed = datetime.now() - start
|
||||
assert 1 < elapsed.total_seconds() < 3
|
||||
|
||||
# Global timeout is obeyed.
|
||||
get_wrolpi_config().download_timeout = 3
|
||||
# ctx.download_timeout takes precedence over class/per-call timeouts.
|
||||
# Production builds ctx.download_timeout from get_wrolpi_config().download_timeout
|
||||
# (see DownloadContext.production); we bake the value directly here.
|
||||
start = datetime.now()
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory)
|
||||
await downloader.process_runner(Download(), ('sleep', '8'), test_directory,
|
||||
ctx=make_test_ctx(download_timeout=3))
|
||||
elapsed = datetime.now() - start
|
||||
assert 2 < elapsed.total_seconds() < 5
|
||||
|
||||
|
|
@ -707,6 +713,7 @@ exit 123
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_process_runner_output(async_client, test_directory, test_downloader):
|
||||
from wrolpi.conftest import make_test_ctx
|
||||
script = test_directory / 'echo_script.sh'
|
||||
script.write_text(GOOD_SCRIPT)
|
||||
cmd = ('/bin/bash', script)
|
||||
|
|
@ -714,7 +721,8 @@ async def test_process_runner_output(async_client, test_directory, test_download
|
|||
Download(),
|
||||
cmd,
|
||||
test_directory,
|
||||
timeout=1
|
||||
timeout=1,
|
||||
ctx=make_test_ctx(),
|
||||
)
|
||||
assert result.return_code == 0
|
||||
assert result.stdout == b'standard output\n'
|
||||
|
|
@ -726,7 +734,8 @@ async def test_process_runner_output(async_client, test_directory, test_download
|
|||
Download(),
|
||||
cmd,
|
||||
test_directory,
|
||||
timeout=1
|
||||
timeout=1,
|
||||
ctx=make_test_ctx(),
|
||||
)
|
||||
assert result.return_code == 123
|
||||
assert result.stdout == b'bad standard output\n'
|
||||
|
|
@ -1123,16 +1132,17 @@ async def test_child_downloads_have_parent_download_url(test_session, test_downl
|
|||
# Track which URLs have been processed to avoid returning children for child downloads
|
||||
processed_urls = set()
|
||||
|
||||
# Configure the test_downloader to return child downloads only for the parent URL
|
||||
async def download_with_children(download, *a, **kwargs):
|
||||
# Configure the test_downloader to return child downloads only for the parent URL.
|
||||
# New signature matches Downloader.execute_download(prepared, ctx, download=...).
|
||||
async def download_with_children(prepared, ctx, *, download=None, **kwargs):
|
||||
await asyncio.sleep(0.1)
|
||||
if download.url == parent_url and parent_url not in processed_urls:
|
||||
if download is not None and download.url == parent_url and parent_url not in processed_urls:
|
||||
processed_urls.add(parent_url)
|
||||
return DownloadResult(success=True, downloads=child_urls)
|
||||
# Return success without children for child downloads
|
||||
return DownloadResult(success=True)
|
||||
|
||||
test_downloader.do_download.side_effect = download_with_children
|
||||
test_downloader.execute_download.side_effect = download_with_children
|
||||
|
||||
# Create the parent download with a sub_downloader specified
|
||||
test_download_manager.create_download(test_session, parent_url, test_downloader.name,
|
||||
|
|
|
|||
|
|
@ -9,9 +9,17 @@ from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadMana
|
|||
|
||||
|
||||
class RSSHTTPDownloader(Downloader):
|
||||
"""Test fixture: a no-op sub-downloader RSS hands child URLs to."""
|
||||
name = 'rss_http'
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
def prepare_download(self, session: Session, download: Download):
|
||||
# Return a non-None sentinel so the dispatch routes to the phase-split path.
|
||||
return object()
|
||||
|
||||
async def execute_download(self, prepared, ctx, download: Download = None):
|
||||
return None
|
||||
|
||||
def finalize_download(self, session: Session, download: Download, executed) -> DownloadResult:
|
||||
return DownloadResult(success=True)
|
||||
|
||||
def already_downloaded(self, session: Session, *urls: List[str]):
|
||||
|
|
@ -295,3 +303,180 @@ async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_sessio
|
|||
# The new video should be added for download with the normalized URL.
|
||||
assert 'https://www.youtube.com/watch?v=NEW456' in download_urls, \
|
||||
'New video should be added to downloads with normalized URL'
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Phase-split unit tests.
|
||||
#
|
||||
# These exercise prepare_download / execute_download / finalize_download in
|
||||
# isolation. They drive the new code paths directly rather than through the
|
||||
# DownloadManager + wait_for_all_downloads polling loop the integration tests
|
||||
# above use.
|
||||
# ---------------------------------------------------------------------------
|
||||
from wrolpi.downloader import ExecutedRSS, PreparedRSS # noqa: E402
|
||||
|
||||
from modules.archive.conftest import make_test_ctx # noqa: E402
|
||||
|
||||
|
||||
def test_prepare_resolves_sub_downloader(test_session, test_download_manager):
|
||||
"""prepare_download finds the sub_downloader instance and surfaces settings."""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
http_downloader = RSSHTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
download = Download(
|
||||
url='https://example.com/feed',
|
||||
downloader='rss',
|
||||
sub_downloader='rss_http',
|
||||
settings={'title_include': 'foo'},
|
||||
)
|
||||
|
||||
prepared = rss_downloader.prepare_download(test_session, download)
|
||||
|
||||
assert isinstance(prepared, PreparedRSS)
|
||||
assert prepared.url == 'https://example.com/feed'
|
||||
assert prepared.sub_downloader_name == 'rss_http'
|
||||
assert prepared.sub_downloader is http_downloader
|
||||
assert prepared.settings == {'title_include': 'foo'}
|
||||
|
||||
|
||||
def test_prepare_unknown_sub_downloader_raises(test_session, test_download_manager):
|
||||
"""No matching sub_downloader → manager raises InvalidDownload (manager will record
|
||||
the failure as a deferred download). The unreachable ValueError branch in the legacy
|
||||
code was dead because find_downloader_by_name raises rather than returning None."""
|
||||
from wrolpi.errors import InvalidDownload
|
||||
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
|
||||
download = Download(
|
||||
url='https://example.com/feed',
|
||||
downloader='rss',
|
||||
sub_downloader='no_such_downloader',
|
||||
)
|
||||
|
||||
with pytest.raises(InvalidDownload, match='Cannot find downloader'):
|
||||
rss_downloader.prepare_download(test_session, download)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_failed_parse_returns_error(test_session, test_download_manager):
|
||||
"""A bozo feed without acceptable errors short-circuits with executed.error."""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
http_downloader = RSSHTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
prepared = PreparedRSS(
|
||||
url='https://example.com/feed',
|
||||
sub_downloader_name='rss_http',
|
||||
sub_downloader=http_downloader,
|
||||
settings={},
|
||||
)
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(bozo=1, bozo_exception=Exception('nope'), entries=[])
|
||||
executed = await rss_downloader.execute_download(prepared, make_test_ctx())
|
||||
|
||||
assert executed.candidate_urls == []
|
||||
assert executed.error == 'Failed to parse RSS feed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_collects_links_and_yt_channel(test_session, test_download_manager):
|
||||
"""A well-formed feed yields candidate_urls plus the yt_channelid for finalize."""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
http_downloader = RSSHTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
prepared = PreparedRSS(
|
||||
url='https://example.com/feed',
|
||||
sub_downloader_name='rss_http',
|
||||
sub_downloader=http_downloader,
|
||||
settings={},
|
||||
)
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(
|
||||
bozo=0,
|
||||
feed=dict(yt_channelid='abc123'),
|
||||
entries=[
|
||||
dict(link='https://example.com/a'),
|
||||
dict(link='https://example.com/b'),
|
||||
],
|
||||
)
|
||||
executed = await rss_downloader.execute_download(prepared, make_test_ctx())
|
||||
|
||||
assert executed.error is None
|
||||
assert executed.yt_channel_id == 'abc123'
|
||||
assert executed.candidate_urls == ['https://example.com/a', 'https://example.com/b']
|
||||
|
||||
|
||||
def test_finalize_propagates_executed_error(test_session, test_download_manager):
|
||||
"""An executed-phase error becomes a failure DownloadResult."""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
|
||||
download = Download(url='https://example.com/feed', downloader='rss', sub_downloader='rss_http')
|
||||
executed = ExecutedRSS(yt_channel_id=None, candidate_urls=[], error='Failed to parse RSS feed')
|
||||
|
||||
result = rss_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
assert result.success is False
|
||||
assert result.error == 'Failed to parse RSS feed'
|
||||
|
||||
|
||||
def test_finalize_dedupes_existing_downloads(test_session, test_download_manager):
|
||||
"""URLs already present in the Download table are filtered out before being returned
|
||||
as child downloads."""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
http_downloader = RSSHTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
# Pre-existing pending download for /a.
|
||||
test_session.add(Download(url='https://example.com/a', downloader='rss_http'))
|
||||
test_session.commit()
|
||||
|
||||
download = Download(
|
||||
url='https://example.com/feed',
|
||||
downloader='rss',
|
||||
sub_downloader='rss_http',
|
||||
)
|
||||
executed = ExecutedRSS(
|
||||
yt_channel_id=None,
|
||||
candidate_urls=['https://example.com/a', 'https://example.com/b', 'https://example.com/c'],
|
||||
)
|
||||
|
||||
result = rss_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
assert result.success is True
|
||||
assert sorted(result.downloads) == ['https://example.com/b', 'https://example.com/c']
|
||||
|
||||
|
||||
def test_finalize_passes_destination_from_column(test_session, test_download_manager, test_directory):
|
||||
"""Child downloads inherit destination from the RSS row's destination column."""
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
http_downloader = RSSHTTPDownloader()
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
dest_path = test_directory / 'archive/custom/path'
|
||||
download = Download(
|
||||
url='https://example.com/feed',
|
||||
downloader='rss',
|
||||
sub_downloader='rss_http',
|
||||
destination=str(dest_path),
|
||||
settings={},
|
||||
)
|
||||
executed = ExecutedRSS(
|
||||
yt_channel_id=None,
|
||||
candidate_urls=['https://example.com/x'],
|
||||
)
|
||||
|
||||
result = rss_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
assert result.success is True
|
||||
assert result.settings['destination'].endswith('archive/custom/path')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue