From 8f0bb78722b7f9167afa1ac77e66b8d621239983 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 7 May 2026 15:55:38 -0600 Subject: [PATCH] Converting RSS and Video Downloaders to new testable strategy. --- modules/archive/conftest.py | 22 +- modules/videos/channel/test/test_api.py | 10 +- modules/videos/downloader.py | 556 ++++++++++++-------- modules/videos/test/test_downloader.py | 107 +++- wrolpi/conftest.py | 52 +- wrolpi/downloader.py | 248 +++++---- wrolpi/files/test/test_scrape_downloader.py | 4 +- wrolpi/test/test_downloader.py | 48 +- wrolpi/test/test_rss.py | 187 ++++++- 9 files changed, 851 insertions(+), 383 deletions(-) diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index 07490064..73ba5f78 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -9,26 +9,10 @@ import pytz from modules.archive.lib import archive_strftime from modules.archive.models import Archive -from wrolpi.cmd import run_command from wrolpi.collections import Collection -from wrolpi.downloader import DownloadContext - - -def make_test_ctx(*, is_cancelled=None, can_download=None, - outside_download_window=None, run_command_=None, - download_timeout=None) -> DownloadContext: - """Build a DownloadContext suitable for downloader unit tests. - - Defaults to a never-cancelled context that uses the real run_command. Tests can - override any callable to simulate cancellation, throttling, or fake subprocess output. - """ - return DownloadContext( - is_cancelled=is_cancelled or (lambda: False), - can_download=can_download or (lambda: True), - outside_download_window=outside_download_window or (lambda: False), - run_command=run_command_ or run_command, - download_timeout=download_timeout, - ) +# Re-export make_test_ctx so existing `from modules.archive.conftest import make_test_ctx` +# call sites keep working. The canonical implementation lives in wrolpi/conftest.py. +from wrolpi.conftest import make_test_ctx # noqa: F401 @pytest.fixture diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index ed088ba9..e594d6b3 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -213,12 +213,14 @@ async def test_channel_download_requires_refresh( downloaded_urls = [] - async def do_download(_, download): - # Track all downloaded URLs. - downloaded_urls.append(download.url) + async def fake_execute(_, prepared, ctx, download=None): + # Track all downloaded URLs and short-circuit by returning a DownloadResult. + # Per the dispatch contract (wrolpi/downloader.py: signal_download_download), + # if execute_download returns a DownloadResult, finalize_download is skipped. + downloaded_urls.append(prepared.url) return DownloadResult(success=True) - with mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download): + with mock.patch('modules.videos.downloader.VideoDownloader.execute_download', fake_execute): entries = [ dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded. dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'), diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 880779ed..d20c7437 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -1,5 +1,6 @@ #! /usr/bin/env python3 import copy +import dataclasses import json import logging import os.path @@ -8,6 +9,7 @@ import re import time import traceback from abc import ABC +from dataclasses import dataclass, field from datetime import timedelta from typing import Tuple, List, Dict, Optional from urllib.parse import urlparse, parse_qs @@ -24,7 +26,7 @@ from wrolpi.common import logger, get_media_directory, escape_file_name, resolve trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post from wrolpi.dates import now from wrolpi.db import get_db_session -from wrolpi.downloader import Downloader, Download, DownloadResult, \ +from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \ clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix @@ -399,6 +401,40 @@ def is_youtube_rss_feed_url(url: str) -> bool: return bool(YOUTUBE_RSS_FEED_URL_RE.match(url or '')) +@dataclass +class PreparedChannel: + """Plan produced by ChannelDownloader.prepare_download. + + Cheap pre-flight: validates the URL is not an RSS feed, snapshots the settings + fields execute and finalize need so they don't have to read back through the + Download row. All real work — yt-dlp metadata fetch, channel resolution, file + indexing — happens in execute_download. + """ + url: str + settings: dict + destination: Optional[str] + tag_names: Optional[List[str]] + + +@dataclass +class ExecutedChannel: + """Output of ChannelDownloader.execute_download. + + `info_json` is the resolved yt-dlp metadata; `channel_id` and `collection_id` are + the IDs assigned by the channel-resolution helper. `child_settings` is the dict + that will be passed to every child VideoDownloader. `error` is set when the + catalog update raised — finalize converts that into a failure DownloadResult + while still mutating the parent Download with sub_downloader/info_json so the + next attempt has them. + """ + info_json: Optional[dict] + channel_id: Optional[int] + collection_id: Optional[int] + location: Optional[str] + child_settings: dict + error: Optional[str] = None + + class ChannelDownloader(Downloader, ABC): """Handles downloading of videos in a Channel or Playlist.""" name = 'video_channel' @@ -413,9 +449,7 @@ class ChannelDownloader(Downloader, ABC): # A playlist may have an id different from its channel. return info['id'] != info.get('channel_id') - async def do_download(self, download: Download) -> DownloadResult: - """Update a Channel's catalog, then schedule downloads of every missing video.""" - _log_ytdlp_versions() + def prepare_download(self, session: Session, download: Download) -> PreparedChannel: if is_youtube_rss_feed_url(download.url): # yt-dlp returns no `uploader` or `channel_id` for an RSS feed URL, so falling through # would create a junk channel named after `webpage_url_basename` ("videos.xml"). @@ -423,8 +457,23 @@ class ChannelDownloader(Downloader, ABC): f'Refusing to handle RSS feed URL with ChannelDownloader: {download.url}.' f' Use the RSS Downloader (sub_downloader=video) instead.' ) + return PreparedChannel( + url=download.url, + settings=download.settings or dict(), + destination=str(download.destination) if download.destination else None, + tag_names=download.tag_names, + ) + + async def execute_download(self, prepared: PreparedChannel, ctx: DownloadContext, + download: Download = None) -> ExecutedChannel: + """Fetch the channel/playlist metadata via yt-dlp, get/create the local Channel + record, and update its catalog. Channel creation + catalog update use a + short-lived self-managed session so the manager's outer session isn't held + during the file_worker await. + """ + _log_ytdlp_versions() try: - info = extract_info(download.url, process=False) + info = extract_info(prepared.url, process=False) except Exception as e: if _bot_blocked(e): raise BotBlockedDownloadError(str(e)) from e @@ -434,86 +483,103 @@ class ChannelDownloader(Downloader, ABC): entries = info.get('entries') if entries and isinstance(entries[0], Dict) and entries[0].get('entries'): # Entries contains more channels/playlist, tell the Maintainer to specify the URL. - raise UnrecoverableDownloadError(f'Unable to download {download.url} because the URL is ambiguous.' + raise UnrecoverableDownloadError(f'Unable to download {prepared.url} because the URL is ambiguous.' f' Try requesting the videos only:' f' https://example.com -> https://example.com/videos') - download_settings = download.settings or dict() - channel_tag_name = i[0] if (i := download_settings.get('channel_tag_name')) else None - - download.sub_downloader = video_downloader.name - download.info_json = info - if session := Session.object_session(download): - # May not have a session during testing. - session.commit() - + channel_tag_name = i[0] if (i := prepared.settings.get('channel_tag_name')) else None name = info.get('uploader') or info.get('webpage_url_basename') if not name: raise ValueError(f'Could not find name') channel_source_id = info.get('channel_id') or info.get('id') - channel_url = info.get('channel_url') or info.get('uploader_url') or download.url - channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name, - destination=download.destination) - channel.dict() # get all attributes while we have the session. + channel_url = info.get('channel_url') or info.get('uploader_url') or prepared.url - # Link this channel download to the channel's collection. - if channel and channel.collection_id and not download.collection_id: - download.collection_id = channel.collection_id - if session: - session.commit() + channel_id = None + collection_id = None + location = None + with get_db_session(commit=True) as session: + channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name, + destination=prepared.destination) + channel.dict() # get all attributes while we have the session. + if channel and channel.id: + channel_id = channel.id + collection_id = channel.collection_id + location = f'/videos/channel/{channel.id}/video' - location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None - - # The settings to send to the VideoDownloader. - settings = dict() - if channel: - settings.update(dict(collection_id=channel.collection_id, channel_url=download.url)) - if download.destination: - destination = get_absolute_media_path(download.destination) - settings['destination'] = str(destination) # Need str for JSON conversion - if download.tag_names: - settings['tag_names'] = download.tag_names + # The settings to send to the VideoDownloader children. + child_settings = dict() + if channel_id: + child_settings.update(dict(collection_id=collection_id, channel_url=prepared.url)) + if prepared.destination: + destination_abs = get_absolute_media_path(prepared.destination) + child_settings['destination'] = str(destination_abs) # Need str for JSON conversion + if prepared.tag_names: + child_settings['tag_names'] = prepared.tag_names # Pass all channel-level inheritable settings to child video downloads. for key in CHANNEL_INHERITABLE_SETTINGS: - if key in download_settings: - settings[key] = download_settings[key] + if key in prepared.settings: + child_settings[key] = prepared.settings[key] is_a_playlist = self.is_a_playlist(info) try: - if not is_a_playlist: - await self.prepare_channel_for_downloads(download, channel) - else: + if not is_a_playlist and channel_id is not None: + await self.prepare_channel_for_downloads(info, channel_id) + elif is_a_playlist: logger.debug('Not updating channel because this is a playlist') - - downloads = self.get_missing_videos(session, download) - return DownloadResult( - success=True, - location=location, - downloads=downloads, - settings=settings, - ) except Exception as e: if PYTEST: raise kind = 'playlist' if is_a_playlist else 'channel' - logger.error(f'Failed to update catalog of {kind} {download.url}', exc_info=e) - return DownloadResult( - success=False, - location=location, + logger.error(f'Failed to update catalog of {kind} {prepared.url}', exc_info=e) + return ExecutedChannel( + info_json=info, channel_id=channel_id, collection_id=collection_id, + location=location, child_settings=child_settings, error=str(traceback.format_exc()), - settings=settings, ) + return ExecutedChannel( + info_json=info, channel_id=channel_id, collection_id=collection_id, + location=location, child_settings=child_settings, + ) + + def finalize_download(self, session: Session, download: Download, + executed: ExecutedChannel) -> DownloadResult: + """Mutate the parent Download (sub_downloader / info_json / collection_id) and + return the list of missing video URLs as child downloads.""" + download.sub_downloader = video_downloader.name + download.info_json = executed.info_json + if executed.collection_id and not download.collection_id: + download.collection_id = executed.collection_id + + if executed.error: + return DownloadResult( + success=False, + location=executed.location, + error=executed.error, + settings=executed.child_settings, + ) + + downloads = self.get_missing_videos(session, download) + return DownloadResult( + success=True, + location=executed.location, + downloads=downloads, + settings=executed.child_settings, + ) + @classmethod - async def prepare_channel_for_downloads(cls, download: Download, channel: Channel): - """Update the Channel's video catalog. Refresh the Channel's files if necessary.""" - logger.debug(f'Preparing {channel} for downloads') - channel_id = channel.id - - update_channel_catalog(channel, download.info_json) + async def prepare_channel_for_downloads(cls, info_json: dict, channel_id: int): + """Update the Channel's video catalog. Refresh the Channel's files if necessary. + Takes channel_id (not Channel) and info_json (not Download) so the caller doesn't + have to hold the manager's session open while we await the file_worker job. + """ with get_db_session() as session: channel = get_channel(session, channel_id=channel_id, return_dict=False) + logger.debug(f'Preparing {channel} for downloads') + + update_channel_catalog(channel, info_json) + if not channel.videos or not channel.refreshed: logger.warning(f'Refreshing videos in {channel.directory} for channel {channel}') job_id = channel.refresh_files(channel.id, send_events=False) @@ -589,6 +655,42 @@ class ChannelDownloader(Downloader, ABC): return downloads +@dataclass +class PreparedVideo: + """Plan produced by VideoDownloader.prepare_download. + + `info_json` is None unless the upstream download row already has it (e.g. set by + ChannelDownloader before queuing this child). `location` is the existing + Video.location if any, used for error attribution if the new download fails. + """ + url: str # normalized + settings: dict + tag_names: List[str] = field(default_factory=list) + destination: Optional[pathlib.Path] = None + info_json: Optional[dict] = None + location: Optional[str] = None # existing video, for error reports + + +@dataclass +class ExecutedVideo: + """Output of VideoDownloader.execute_download once the file is on disk and validated. + + Carries everything finalize_download needs to persist the Video record without + re-running any I/O. When the metadata-only flow runs, execute returns a + DownloadResult directly and the dispatch skips finalize_download entirely. + """ + url: str # normalized + video_path: pathlib.Path + video_paths: List[pathlib.Path] # all files in the FileGroup, post-glob+normalize + info_json: dict + entry: dict + channel_id: Optional[int] + audio_only: bool + tag_names: List[str] + settings: dict + location: Optional[str] # for error reporting + + class VideoDownloader(Downloader, ABC): """Downloads a single video. @@ -608,84 +710,126 @@ class VideoDownloader(Downloader, ABC): file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video')) return file_groups - async def do_download(self, download: Download) -> DownloadResult: + def prepare_download(self, session: Session, download: Download) -> PreparedVideo: + """Sync prep: validate attempts cap, normalize URL, copy upstream-provided settings + onto the Download row, and look up any pre-existing Video for error attribution. + + Holding the session here is brief: no network or subprocess work runs in this + phase. extract_info and channel resolution move to execute_download. + """ + if (download.attempts or 0) >= 10: + raise UnrecoverableDownloadError('Max download attempts reached') + + url = normalize_video_url(download.url) + + # ChannelDownloader passes destination/tag_names via settings; copy onto the row + # so the rest of the flow can read them off the Download object as before. + settings = download.settings or dict() + download.destination = download.destination or settings.get('destination') + tag_names = download.tag_names = download.tag_names or settings.get('tag_names') or [] + + # If a Video already exists at this URL, capture its location so a failure later + # can still tell the user where the previous attempt landed. + existing = Video.get_by_url(session, url) + location = existing.location if existing else None + + return PreparedVideo( + url=url, + settings=settings, + tag_names=list(tag_names), + destination=download.destination, + info_json=download.info_json, + location=location, + ) + + async def execute_download(self, prepared: PreparedVideo, ctx: DownloadContext, + download: Download = None): + """Async I/O phase: extract_info, channel resolution (brief session), + yt-dlp subprocess, file resolution, ffmpeg validation, sibling file globbing. + + Returns either an ExecutedVideo (file is on disk; finalize persists) or a + DownloadResult (an early-exit case the dispatch will surface verbatim — used + for the metadata-only refresh flow and recoverable failures like a missing + file or incomplete stream). + """ _log_ytdlp_versions() - with get_db_session() as session: - if download.attempts >= 10: - raise UnrecoverableDownloadError('Max download attempts reached') - - url = normalize_video_url(download.url) - - # Copy settings into Download (they may be from the ChannelDownloader) - settings = download.settings or dict() - download.destination = download.destination or settings.get('destination') - tag_names = download.tag_names = download.tag_names or settings.get('tag_names') - - # Video may have been downloaded previously, get its location for error reporting. - video_ = Video.get_by_url(session, url) - location = video_.location if video_ else None + url = prepared.url + settings = prepared.settings + location = prepared.location + # extract_info is sync but expensive; legacy held a session across it. We don't. + info_json = prepared.info_json + if not info_json: try: - download.info_json = download.info_json or extract_info(url) + info_json = extract_info(url) except yt_dlp.utils.DownloadError as e: if _bot_blocked(e): raise BotBlockedDownloadError(str(e)) from e - # Video may be private. + # Video may be private — surface as a recoverable failure. try: raise RuntimeError('Failed to extract_info') from e - except Exception as e: + except Exception as e2: return DownloadResult( success=False, location=location, - error='\n'.join(traceback.format_exception(e)), + error='\n'.join(traceback.format_exception(e2)), ) - if not download.info_json: - raise ValueError(f'Cannot download video with no info_json.') + if not info_json: + raise ValueError(f'Cannot download video with no info_json.') - channel = await self._get_channel(session, download) + # Brief session for channel resolution. _get_channel reads info_json (already + # populated above) and may create a Channel record. + download_for_channel = download if download is not None else Download( + url=url, settings=settings, destination=prepared.destination, + ) + download_for_channel.info_json = info_json + with get_db_session(commit=True) as session: + channel = await self._get_channel(session, download_for_channel) channel_id = channel.id if channel else None - destination = download.destination + channel_directory = channel.directory if channel else None - if destination: - # Download to the directory specified in the settings. - out_dir = destination - logger.debug(f'Downloading {url} to destination {destination} from settings') - elif channel: - out_dir = channel.directory - logger.debug(f'Downloading {url} to channel directory: {channel.directory}') - else: - # Download to the default directory if this video has no channel. - out_dir = get_no_channel_directory() - logger.debug(f'Downloading {url} to default directory') + if prepared.destination: + out_dir = prepared.destination + logger.debug(f'Downloading {url} to destination {prepared.destination} from settings') + elif channel_directory: + out_dir = channel_directory + logger.debug(f'Downloading {url} to channel directory: {channel_directory}') + else: + out_dir = get_no_channel_directory() + logger.debug(f'Downloading {url} to default directory') + out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir) + out_dir = get_absolute_media_path(out_dir) + out_dir.mkdir(exist_ok=True, parents=True) - # Make output directory. (Maybe string from settings) - out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir) - out_dir = get_absolute_media_path(out_dir) - out_dir.mkdir(exist_ok=True, parents=True) - - if tag_names and logger.isEnabledFor(logging.DEBUG): - logger.debug(f'Downloading {url} with {tag_names=}') + if prepared.tag_names and logger.isEnabledFor(logging.DEBUG): + logger.debug(f'Downloading {url} with tag_names={prepared.tag_names}') effective = get_effective_video_settings(settings) audio_only = effective.get('audio_only', False) audio_format = effective.get('audio_format', 'mp3') - logs = None # noqa + # download_for_log carries .id (for progress) and .url (for log messages). + # Falling back to a stub keeps unit tests viable. + download_for_log = download if download is not None else Download(url=url) + try: video_resolutions = effective['video_resolutions'] - # yt-dlp expects a string like so: 299+140,298+140,bestvideo*+bestaudio/best video_resolutions_str = ','.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i]) - video_format = effective['video_format'] video_path, entry = self.prepare_filename(url, out_dir, video_resolutions_str, video_format, audio_only=audio_only, audio_format=audio_format) if settings.get('download_metadata_only'): # User has requested refresh of video metadata, skip the rest of the video downloader. - return await self.download_info_json(download, video_path) + # download_info_json manages its own DB sessions; we surface its DownloadResult + # directly and the dispatch skips finalize_download. + download_for_metadata = download if download is not None else Download( + url=url, settings=settings, + ) + download_for_metadata.info_json = info_json + return await self.download_info_json(download_for_metadata, video_path) - # Common yt-dlp flags shared by both video and audio-only downloads. cmd = ( str(YT_DLP_BIN), '--match-filter', '!is_live', @@ -719,27 +863,26 @@ class VideoDownloader(Downloader, ABC): if effective['sleep_requests'] > 0: cmd = (*cmd, '--sleep-requests', str(effective['sleep_requests'])) - on_stdout = make_progress_callback(download.id, parse_ytdlp_progress) + on_stdout = make_progress_callback(download_for_log.id, parse_ytdlp_progress) try: if cookies_unlocked(): - # Use encrypted cookies file logger.info(f'Using encrypted cookies for video download: {url}') with cookies_for_download() as cookies_path: cmd = (*cmd, '--cookies', str(cookies_path)) if effective['user_agent']: cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}') cmd = (*cmd, '-o', video_path, url) - result = await self.process_runner(download, cmd, out_dir, debug=True, - stdout_callback=on_stdout) + result = await self.process_runner(download_for_log, cmd, out_dir, debug=True, + stdout_callback=on_stdout, ctx=ctx) else: - # No cookies available logger.info(f'No cookies available for video download: {url}') cmd = (*cmd, '-o', video_path, url) - result = await self.process_runner(download, cmd, out_dir, debug=True, - stdout_callback=on_stdout) + result = await self.process_runner(download_for_log, cmd, out_dir, debug=True, + stdout_callback=on_stdout, ctx=ctx) finally: - clear_download_progress(download.id) + clear_download_progress(download_for_log.id) + stdout = result.stdout.decode() stderr = result.stderr.decode() @@ -747,36 +890,27 @@ class VideoDownloader(Downloader, ABC): error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}' if _bot_blocked(error): raise BotBlockedDownloadError(f'Bot detection or invalid cookies: {error}') - return DownloadResult( - success=False, - error=error, - location=location, - ) + return DownloadResult(success=False, error=error, location=location) output_format = audio_output_extension(audio_format) if audio_only else video_format preferred_path = video_path.with_suffix(f'.{output_format}') if not video_path.is_file() and preferred_path.is_file(): - # Prepared filename does not exist, but file with preferred extension does, it was probably - # remuxed/converted by yt-dlp. video_path = preferred_path logger.info(f'Using preferred file which exists: {preferred_path}') if not video_path.is_file(): - # yt-dlp may have appended the real extension to a bogus one (e.g. .NA.mp4 from a playlist entry). + # yt-dlp may have appended the real extension to a bogus one. appended_path = video_path.parent / f'{video_path.name}.{output_format}' if appended_path.is_file(): video_path = appended_path logger.info(f'Using file with appended extension: {appended_path}') if video_path.suffix == '.part': - return DownloadResult( - success=False, - error=f'Video file that completed was a .part file: {video_path}', - ) + return DownloadResult(success=False, + error=f'Video file that completed was a .part file: {video_path}') if not video_path.is_file(): - error = f'{stdout}\n\n\n{stderr}\n\n' \ - f'Video file could not be found! {video_path}' + error = f'{stdout}\n\n\n{stderr}\n\nVideo file could not be found! {video_path}' retry_seconds = None if '!is_live' in stdout: error = f'{stdout}\n\nVideo was live and did not finish downloading.' @@ -784,82 +918,29 @@ class VideoDownloader(Downloader, ABC): if ' live event ' in stdout: error = f'{stdout}\n\nVideo will be live and did not finish downloading.' retry_seconds = 3600 - return DownloadResult( - success=False, - error=error, - location=location, - retry_seconds=retry_seconds, - ) + return DownloadResult(success=False, error=error, location=location, + retry_seconds=retry_seconds) if not audio_only and not ffmpeg_video_complete(video_path): - return DownloadResult( - success=False, - error='Video was incomplete', - location=location, - ) + return DownloadResult(success=False, error='Video was incomplete', location=location) - with get_db_session(commit=True) as session: - # Find any files downloaded with the video (poster, caption, etc.). - video_paths = glob_shared_stem(video_path) - # Delete any leftover part files now that we have verified the video is complete. - video_paths = self._delete_part_files(video_path, video_paths) - # Rename any special files that do not match the stem. - video_paths = self.normalize_video_file_names(video_path, video_paths) - # Create the Video record. - video = Video.from_paths(session, *video_paths) - video.source_id = entry['id'] - video.channel_id = channel_id - video_id, video_info_json_path = video.id, video.info_json_path - if video_info_json_path and (new_info_json := video.clean_info_json()): - video.replace_info_json(new_info_json, clean=False) + # File is on disk and complete. Glob siblings, drop part files, normalize names. + video_paths = glob_shared_stem(video_path) + video_paths = self._delete_part_files(video_path, video_paths) + video_paths = self.normalize_video_file_names(video_path, video_paths) - if parent_download_url := settings.get('parent_download_url'): - try: - video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url}) - except ValueError as e: - logger.warning(f'Failed to update wrolpi json for {video}: {e}') - - # Update channel's collection's file_format to track the format used - if channel_id: - from modules.videos.models import Channel - from modules.videos.lib import save_channels_config - channel_obj = session.query(Channel).get(channel_id) - if channel_obj and channel_obj.collection and not channel_obj.collection.file_format: - video_config = get_videos_downloader_config() - channel_obj.collection.file_format = video_config.file_name_format - save_channels_config.activate_switch() - - session.commit() - - with get_db_session(commit=True) as session: - # Second session is started because SQLAlchemy will forget what we have done. - video = Video.get_by_id(session, video_id) - if tag_names: - existing_names = video.file_group.tag_names - for name in tag_names: - if name not in existing_names: - video.add_tag(session, name) - location = video.location - - await video.get_ffprobe_json() - - # Check that the download has the expected streams. - if not audio_only and not video.get_streams_by_codec_type('video'): - return DownloadResult( - success=False, - error='Video was downloaded but did not contain video stream', - location=location, - ) - if not video.get_streams_by_codec_type('audio'): - return DownloadResult( - success=False, - error='Download did not contain audio stream', - location=location, - ) - logger.info(f'Successfully downloaded video {url} {video}') - - with get_db_session(commit=True) as session: - await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path) + return ExecutedVideo( + url=url, + video_path=video_path, + video_paths=video_paths, + info_json=info_json, + entry=entry, + channel_id=channel_id, + audio_only=audio_only, + tag_names=prepared.tag_names, + settings=settings, + location=location, + ) except BotBlockedDownloadError: raise @@ -874,30 +955,73 @@ class VideoDownloader(Downloader, ABC): if _skip_download(e): # The video failed to download, and the error will never be fixed. Skip it forever. try: - source_id = download.info_json.get('id') + source_id = (info_json or {}).get('id') logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not ' f'attempt to download it again.') - download.add_to_skip_list() + if download is not None: + download.add_to_skip_list() except Exception: - # Could not skip this video, it may not have a channel. logger.warning(f'Could not skip video {url}') - - # Skipped downloads should not be tried again. raise UnrecoverableDownloadError() from e - # Download did not succeed, try again later. - if logs and (stderr := logs.get('stderr')): - error = f'{stderr}\n\n{traceback.format_exc()}' - else: - error = str(traceback.format_exc()) - return DownloadResult(success=False, error=error, location=location) + return DownloadResult(success=False, error=str(traceback.format_exc()), location=location) - logger.debug(f'Downloaded video {location=}') + async def finalize_download(self, session: Session, download: Download, + executed: ExecutedVideo) -> DownloadResult: + """Async DB-bound persistence: create the Video record, normalize info_json, + update the channel's collection format, apply tags, run ffprobe, validate + streams, and dedupe. Caller commits. + """ + # Create the Video record. + video = Video.from_paths(session, *executed.video_paths) + video.source_id = executed.entry['id'] + video.channel_id = executed.channel_id + if video.info_json_path and (new_info_json := video.clean_info_json()): + video.replace_info_json(new_info_json, clean=False) - result = DownloadResult( - success=True, - location=location, - ) - return result + if parent_download_url := executed.settings.get('parent_download_url'): + try: + video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url}) + except ValueError as e: + logger.warning(f'Failed to update wrolpi json for {video}: {e}') + + # Update channel's collection's file_format to track the format used. + if executed.channel_id: + from modules.videos.lib import save_channels_config + channel_obj = session.query(Channel).get(executed.channel_id) + if channel_obj and channel_obj.collection and not channel_obj.collection.file_format: + video_config = get_videos_downloader_config() + channel_obj.collection.file_format = video_config.file_name_format + save_channels_config.activate_switch() + + # Apply tags now that the file_group is bound to the session. + if executed.tag_names: + existing_names = video.file_group.tag_names + for name in executed.tag_names: + if name not in existing_names: + video.add_tag(session, name) + + location = video.location + + # ffprobe runs the ffprobe subprocess (may persist .ffprobe.json on disk). + await video.get_ffprobe_json() + + if not executed.audio_only and not video.get_streams_by_codec_type('video'): + return DownloadResult(success=False, + error='Video was downloaded but did not contain video stream', + location=location) + if not video.get_streams_by_codec_type('audio'): + return DownloadResult(success=False, + error='Download did not contain audio stream', + location=location) + + logger.info(f'Successfully downloaded video {executed.url} {video}') + + # Delete duplicate Videos that share the URL or source_id. Runs against the same + # session as the new Video record so deletion is atomic with creation. + await Video.delete_duplicate_videos(session, download.url, executed.entry['id'], + executed.video_path) + + return DownloadResult(success=True, location=location) @staticmethod async def _get_channel(session: Session, download: Download) -> Optional[Channel]: diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 3b68f6e0..08c40d5d 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -482,7 +482,9 @@ async def test_channel_downloader_rejects_rss_feed_url(test_session, mock_video_ } with pytest.raises(UnrecoverableDownloadError): - await channel_downloader.do_download(download) + # RSS-feed rejection happens in the prepare phase (sync), so we don't even + # need to drive execute_download here. + channel_downloader.prepare_download(test_session, download) # No Channel should have been created from the RSS URL. assert test_session.query(Channel).count() == 0 @@ -668,9 +670,13 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first. - with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download: + with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download: mock_video_do_download.return_value = DownloadResult(success=True) # Don't download the videos. - result = await channel_downloader.do_download(download) + # Drive the phase-split API directly: prepare → execute → finalize. + from modules.archive.conftest import make_test_ctx + prepared = channel_downloader.prepare_download(test_session, download) + executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download) + result = channel_downloader.finalize_download(test_session, download, executed) assert result.success is True, 'Download was not successful' assert set(result.downloads) == { 'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL. @@ -899,11 +905,14 @@ async def test_channel_downloader_sets_collection_id(test_session, channel_facto mock_video_extract_info.return_value = mock_info # Mock Channel.refresh_files and file_worker.wait_for_job to avoid background tasks - with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \ + with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \ mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \ mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock): mock_video_do_download.return_value = DownloadResult(success=True) - await channel_downloader.do_download(download) + from modules.archive.conftest import make_test_ctx + prepared = channel_downloader.prepare_download(test_session, download) + executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download) + channel_downloader.finalize_download(test_session, download, executed) await await_switches() @@ -947,11 +956,14 @@ async def test_channel_passes_all_inheritable_settings_to_video(test_session, ch } mock_video_extract_info.return_value = mock_info - with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \ + with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \ mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \ mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock): mock_video_do_download.return_value = DownloadResult(success=True) - result = await channel_downloader.do_download(download) + from modules.archive.conftest import make_test_ctx + prepared = channel_downloader.prepare_download(test_session, download) + executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download) + result = channel_downloader.finalize_download(test_session, download, executed) # Verify all channel-level inheritable settings are passed through for key in ('video_resolutions', 'video_format', 'writesubtitles', 'sleep_requests', 'user_agent'): @@ -991,11 +1003,14 @@ async def test_channel_does_not_pass_continue_dl_nooverwrites(test_session, chan } mock_video_extract_info.return_value = mock_info - with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \ + with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \ mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \ mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock): mock_video_do_download.return_value = DownloadResult(success=True) - result = await channel_downloader.do_download(download) + from modules.archive.conftest import make_test_ctx + prepared = channel_downloader.prepare_download(test_session, download) + executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download) + result = channel_downloader.finalize_download(test_session, download, executed) assert 'continue_dl' not in result.settings, 'continue_dl should not be passed to child video downloads' assert 'nooverwrites' not in result.settings, 'nooverwrites should not be passed to child video downloads' @@ -1063,7 +1078,12 @@ async def test_live_video_retry_seconds(test_session, test_directory, mock_video mock_prepare_filename.return_value = (simple_channel.directory / 'live video.mp4', {'id': 'foo'}) download = Download(url=url, downloader='video', attempts=0) - result = await video_downloader.do_download(download) + # Drive the phase-split API directly: when execute_download detects a live video + # it short-circuits with a DownloadResult (the dispatch surfaces it verbatim and + # skips finalize_download). + from modules.archive.conftest import make_test_ctx + prepared = video_downloader.prepare_download(test_session, download) + result = await video_downloader.execute_download(prepared, make_test_ctx(), download=download) assert result.success is False assert result.retry_seconds == 3600 @@ -1252,3 +1272,70 @@ async def test_channel_download_preserves_skip_list(test_session, test_directory once_downloads = video_download_manager.get_once_downloads(test_session) video_downloads = [i for i in once_downloads if 'watch' in i.url] assert {i.url for i in video_downloads} == {'https://youtube.com/watch?v=video_2_url'} + + +# --------------------------------------------------------------------------- +# Phase-split unit tests for VideoDownloader.prepare_download. +# +# These exercise the sync prep invariants in isolation — no async_client, no +# extract_info, no subprocess. Heavier execute/finalize behavior is covered by +# the integration tests above (which drive through video_download_manager). +# --------------------------------------------------------------------------- +from modules.videos.downloader import PreparedVideo # noqa: E402 + + +def test_prepare_max_attempts_raises(test_session): + """prepare_download enforces the 10-attempt cap (videos retry more than other + downloaders because YouTube transients are common).""" + download = Download(url='https://youtube.com/watch?v=foo', downloader='video', attempts=10) + + with pytest.raises(UnrecoverableDownloadError, match='Max download attempts'): + video_downloader.prepare_download(test_session, download) + + +def test_prepare_normalizes_url(test_session): + """Shorts URLs become watch URLs in the PreparedVideo (matches DB-stored format).""" + download = Download( + url='https://www.youtube.com/shorts/ABC123', + downloader='video', + settings={}, + ) + + prepared = video_downloader.prepare_download(test_session, download) + + assert isinstance(prepared, PreparedVideo) + assert prepared.url == 'https://www.youtube.com/watch?v=ABC123' + + +def test_prepare_copies_upstream_settings_onto_download(test_session): + """ChannelDownloader passes destination + tag_names via settings; prep copies them + onto the Download row so downstream code sees them as columns.""" + download = Download( + url='https://youtube.com/watch?v=foo', + downloader='video', + destination=None, + tag_names=None, + settings={'destination': '/media/wrolpi/videos/foo', 'tag_names': ['news', 'tech']}, + ) + + prepared = video_downloader.prepare_download(test_session, download) + + assert download.destination == '/media/wrolpi/videos/foo' + assert download.tag_names == ['news', 'tech'] + assert prepared.tag_names == ['news', 'tech'] + + +def test_prepare_uses_existing_video_location(test_session, video_factory): + """If a Video already exists at the URL, prep captures its location for error + attribution (so a failed re-download still tells the user where the previous file is).""" + url = 'https://youtube.com/watch?v=existing' + existing = video_factory(with_video_file=True) + existing.file_group.url = url + test_session.commit() + + download = Download(url=url, downloader='video', settings={}) + + prepared = video_downloader.prepare_download(test_session, download) + + assert prepared.location == existing.location + assert prepared.location is not None diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 8e292872..dbab1f5f 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -35,7 +35,7 @@ from sqlalchemy.orm import Session, sessionmaker import wrolpi.root_api # noqa from wrolpi import flags from wrolpi.api_utils import api_app -from wrolpi.cmd import CommandResult +from wrolpi.cmd import CommandResult, run_command from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL, \ get_wrolpi_config from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS @@ -43,7 +43,7 @@ from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args -from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \ +from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \ downloads_manager_config_context, download_cache_config_context from wrolpi.errors import UnrecoverableDownloadError from wrolpi.files.models import Directory, FileGroup @@ -378,16 +378,52 @@ def assert_downloads(test_session) -> Callable[[List[Dict]], None]: return asserter +def make_test_ctx(*, is_cancelled=None, can_download=None, + outside_download_window=None, run_command_=None, + download_timeout=None) -> DownloadContext: + """Build a DownloadContext suitable for downloader unit tests. + + Defaults: never-cancelled, can_download True, no window restriction, real run_command, + no global timeout. Override any callable to simulate cancellation, throttling, or + fake subprocess output. + + Lives in wrolpi/conftest.py so any test (including primitive tests of process_runner / + cancel_wrapper) can construct a ctx without crossing module boundaries. + """ + return DownloadContext( + is_cancelled=is_cancelled or (lambda: False), + can_download=can_download or (lambda: True), + outside_download_window=outside_download_window or (lambda: False), + run_command=run_command_ or run_command, + download_timeout=download_timeout, + ) + + @pytest.fixture def test_downloader(test_download_manager) -> Downloader: class TestDownloader(Downloader, ABC): - """A testing Downloader""" + """A testing Downloader. + + prepare_download returns a non-None sentinel so the manager's phase-split dispatch + routes through this subclass. execute_download is an AsyncMock — tests wire its + side_effect via the set_test_* methods. Returning a DownloadResult from + execute_download tells the dispatch to skip finalize (see signal_download_download). + """ name = 'test_downloader' def __repr__(self): return '' - do_download = AsyncMock() + execute_download = AsyncMock() + + def prepare_download(self, session, download): + return object() # sentinel; activates the phase-split path + + def finalize_download(self, session, download, executed): + # execute_download returns a DownloadResult directly, so the dispatch + # short-circuits before reaching finalize. Provided here only to satisfy + # the abstract contract. + return DownloadResult(success=True) def set_test_success(self): async def _(*a, **kwargs): @@ -395,7 +431,7 @@ def test_downloader(test_download_manager) -> Downloader: await asyncio.sleep(1) return DownloadResult(success=True) - self.do_download.side_effect = _ + self.execute_download.side_effect = _ def set_test_failure(self): async def _(*a, **kwargs): @@ -403,7 +439,7 @@ def test_downloader(test_download_manager) -> Downloader: await asyncio.sleep(1) return DownloadResult(success=False) - self.do_download.side_effect = _ + self.execute_download.side_effect = _ def set_test_exception(self, exception: Exception = Exception('Test downloader exception')): async def _(*a, **kwargs): @@ -411,7 +447,7 @@ def test_downloader(test_download_manager) -> Downloader: await asyncio.sleep(1) raise exception - self.do_download.side_effect = _ + self.execute_download.side_effect = _ def set_test_unrecoverable_exception(self): async def _(*a, **kwargs): @@ -419,7 +455,7 @@ def test_downloader(test_download_manager) -> Downloader: await asyncio.sleep(1) raise UnrecoverableDownloadError() - self.do_download.side_effect = _ + self.execute_download.side_effect = _ test_downloader = TestDownloader() # Default to successful download. diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 8f623c51..71ad3e25 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1,6 +1,5 @@ import asyncio import contextlib -import inspect import logging import multiprocessing import os @@ -419,23 +418,19 @@ class Downloader: def __repr__(self): return f'' - async def do_download(self, download: Download, ctx: 'DownloadContext' = None) -> DownloadResult: - """Legacy single-phase entry point. + def prepare_download(self, session: Session, download: Download) -> Any: + """DB-bound prep. Caller (DownloadManager) opens and commits the session. - Subclasses opting in to the phase-split API override prepare_download / - execute_download / finalize_download instead and leave this unimplemented. + Sync by default; subclasses may override with `async def prepare_download` when + they need network or subprocess work during prep (e.g. yt-dlp extract_info). + The dispatch awaits the coroutine. Holding the session across an await means + the connection is in use — keep async prep work brief. + + Subclasses must override and return a value (typically a dataclass) that is + passed unchanged to execute_download. """ raise NotImplementedError() - def prepare_download(self, session: Session, download: Download) -> Optional[Any]: - """Synchronous DB-bound prep. Caller (DownloadManager) opens and commits the session. - - Returning None signals the subclass has not opted in to the phase-split API and - DownloadManager should fall back to do_download. Returning any other value - (typically a dataclass) is passed unchanged to execute_download. - """ - return None - async def execute_download(self, prepared: Any, ctx: 'DownloadContext', download: Download = None) -> Any: """Async I/O-bound work. No DB session — does network and subprocess work, @@ -448,7 +443,12 @@ class Downloader: raise NotImplementedError() def finalize_download(self, session: Session, download: Download, executed: Any) -> DownloadResult: - """Synchronous DB-bound persistence. Caller opens and commits the session.""" + """DB-bound persistence. Caller opens and commits the session. + + Sync by default; subclasses may override with `async def finalize_download` when + post-download persistence requires async work (e.g. ffprobe metadata, dedupe + queries). The dispatch awaits the coroutine. + """ raise NotImplementedError() def already_downloaded(self, session: Session, *urls: List[str]): @@ -465,25 +465,22 @@ class Downloader: Global timeout takes precedence over the timeout argument, unless it is 0. (Smaller global timeout wins) - When ctx is provided, ctx.run_command and ctx.download_timeout are used and cancellation - is polled through ctx. When ctx is None (legacy callers), behavior is unchanged. + ctx is required from the dispatch path; the kwarg default of None is a convenience + for unit tests that drive process_runner directly with a stub ctx. """ if cwd and not cwd.is_dir(): raise RuntimeError('cwd directory does not exist') + if ctx is None: + raise RuntimeError('process_runner requires a DownloadContext') - if ctx is not None: - config_timeout = ctx.download_timeout - runner = ctx.run_command - else: - config_timeout = get_wrolpi_config().download_timeout - runner = run_command + config_timeout = ctx.download_timeout # 0 means no timeout. Prefer config timeout, then argument, then class default. if config_timeout: timeout = config_timeout elif timeout is None: timeout = self.timeout - coro = runner(cmd, cwd=cwd, timeout=timeout, log_command=debug, - stdout_callback=stdout_callback, env=env) + coro = ctx.run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug, + stdout_callback=stdout_callback, env=env) result = await self.cancel_wrapper(coro, download, ctx=ctx) if __debug__ and logger.isEnabledFor(TRACE_LEVEL): @@ -497,23 +494,17 @@ class Downloader: @staticmethod async def cancel_wrapper(coro: Coroutine, download: Download, ctx: 'DownloadContext' = None): """ - Converts an async coroutine to a task. If DownloadManager receives a kill request, this method will cancel - the task. - - When ctx is provided, cancellation is polled through ctx.is_cancelled / ctx.can_download / - ctx.outside_download_window. When ctx is None, the global download_manager singleton is - used (legacy path for unmigrated subclasses). + Converts an async coroutine to a task. If DownloadManager receives a kill request, + this method will cancel the task. Cancellation state is read through ctx. """ + if ctx is None: + raise RuntimeError('cancel_wrapper requires a DownloadContext') download_id = download.id task = asyncio.create_task(coro) try: while not task.done(): - if ctx is not None: - cancelled = ctx.is_cancelled() or not ctx.can_download() - outside_window = ctx.outside_download_window() - else: - cancelled = download_manager.download_is_killed(download_id) or not download_manager.can_download - outside_window = download_manager.outside_download_window + cancelled = ctx.is_cancelled() or not ctx.can_download() + outside_window = ctx.outside_download_window() if cancelled: logger.warning(f'Cancel download of {download.url}') task.cancel() @@ -528,16 +519,13 @@ class Downloader: error=error, ) finally: - if ctx is None: + # The killed-downloads list lives on the manager; clear via the + # singleton. Tests that bypass the dispatch may not have a manager + # singleton wired up — silently ignore in that case. + try: download_manager.unkill_download(download_id) - else: - # Manager still owns the killed-downloads list; clear via singleton. - # The ctx is read-only with respect to cancellation state. - try: - download_manager.unkill_download(download_id) - except Exception: - # In tests without a manager, silently ignore. - pass + except Exception: + pass else: # Wait for the download to complete. Cancel if requested. await asyncio.sleep(0.1) @@ -1539,30 +1527,33 @@ async def signal_download_download(download_id: int, download_url: str): try_again = True try: - # Phase-split path: subclass opts in by overriding prepare_download to return non-None. - # Legacy path: do_download remains the single entry point. + # Phase-split dispatch. Caller (us) owns transaction boundaries: a session + # is opened for prepare_download (committing any pre-flight DB work like + # destination resolution) and a fresh session for finalize_download (so we + # don't hold a connection across the long-running execute_download). + # cancel_wrapper may short-circuit with a DownloadResult instead of an + # ExecutedX dataclass; subclasses also use that protocol to skip finalize + # for early-exit cases (e.g. metadata-only video downloads). ctx = DownloadContext.production(download_manager, download_id) with get_db_session(commit=True) as s: prepared = downloader.prepare_download(s, download) + # Subclasses with unavoidable async prep work (e.g. VideoDownloader's + # extract_info / channel resolution) may declare prepare_download as + # `async def`; await the coroutine here. + if asyncio.iscoroutine(prepared): + prepared = await prepared - if prepared is not None: - # New phase-split path: caller (us) owns transaction boundaries. - coro = downloader.execute_download(prepared, ctx, download=download) - executed = await downloader.cancel_wrapper(coro, download, ctx=ctx) - if isinstance(executed, DownloadResult): - # cancel_wrapper short-circuited with a cancel result. - result = executed - else: - with get_db_session(commit=True) as s: - result = downloader.finalize_download(s, download, executed) + coro = downloader.execute_download(prepared, ctx, download=download) + executed = await downloader.cancel_wrapper(coro, download, ctx=ctx) + if isinstance(executed, DownloadResult): + # cancel_wrapper or execute_download short-circuited with a result. + result = executed else: - # Legacy path. Subclasses on this path have signature do_download(self, download) - # without ctx, so we must NOT pass ctx as a positional arg. cancel_wrapper continues - # to use the global download_manager (ctx=None). - if not inspect.iscoroutinefunction(downloader.do_download): - raise RuntimeError(f'Coroutine expected from {downloader} do_download method.') - coro = downloader.do_download(download) - result = await downloader.cancel_wrapper(coro, download) + with get_db_session(commit=True) as s: + result = downloader.finalize_download(s, download, executed) + # finalize_download may also be async (same rationale as prepare). + if asyncio.iscoroutine(result): + result = await result except BotBlockedDownloadError as e: # Bot detection or invalid cookies - stop all pending downloads from this domain. worker_logger.warning(f'BotBlockedDownloadError for {url}', exc_info=e) @@ -2048,6 +2039,34 @@ def parse_feed(url: str) -> FeedParserDict: return feedparser.parse(url) +@dataclass +class PreparedRSS: + """Plan produced by RSSDownloader.prepare_download. + + sub_downloader is the resolved Downloader instance used both for already_downloaded + DB queries and to detect the 'video' branch (where URL normalization + duration + filtering applies). Carries the original Download stand-in fields execute and + finalize need so neither phase has to reach back to the DB row. + """ + url: str + sub_downloader_name: str + sub_downloader: 'Downloader' + settings: dict + + +@dataclass +class ExecutedRSS: + """Output of RSSDownloader.execute_download. + + candidate_urls is the post-filter, post-duration-check URL list ready for DB dedupe + in finalize. yt_channel_id (when present) lets finalize apply Channel info to the + Download row. error covers feed-parse failures. + """ + yt_channel_id: Optional[str] + candidate_urls: List[str] + error: Optional[str] = None + + class RSSDownloader(Downloader): """Downloads an RSS feed and creates new downloads for every unique link in the feed.""" name = 'rss' @@ -2057,27 +2076,34 @@ class RSSDownloader(Downloader): def __repr__(self): return '' - async def do_download(self, download: Download) -> DownloadResult: + def prepare_download(self, session: Session, download: Download) -> PreparedRSS: sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader) if not sub_downloader: raise ValueError(f'Unable to find sub_downloader for {download.url}') + return PreparedRSS( + url=download.url, + sub_downloader_name=download.sub_downloader, + sub_downloader=sub_downloader, + settings=download.settings or dict(), + ) - feed: FeedParserDict = parse_feed(download.url) + async def execute_download(self, prepared: PreparedRSS, ctx: 'DownloadContext', + download: Download = None) -> ExecutedRSS: + feed: FeedParserDict = parse_feed(prepared.url) if feed['bozo'] and not self.acceptable_bozo_errors(feed): - # Feed did not parse - return DownloadResult(success=False, error='Failed to parse RSS feed') + return ExecutedRSS(yt_channel_id=None, candidate_urls=[], + error='Failed to parse RSS feed') if not isinstance(feed, dict) or not feed.get('entries'): - # RSS parsed but does not have entries. - return DownloadResult(success=False, error='RSS feed did not have any entries') + return ExecutedRSS(yt_channel_id=None, candidate_urls=[], + error='RSS feed did not have any entries') - # Apply YT channel to the Download, if not already applied. - if yt_channel_id := feed.get('feed', dict()).get('yt_channelid'): - if not (download.location or download.collection_id): - self.apply_yt_channel(download.id, yt_channel_id) + yt_channel_id = feed.get('feed', dict()).get('yt_channelid') - # Filter entries using Download.settings. - entries = self.filter_entries(download, feed['entries']) + # Filter entries using settings; preserve filter_entries' Download-keyed signature + # by passing a stand-in (test_rss.py calls filter_entries directly with a Download). + log_stub = Download(url=prepared.url, settings=prepared.settings) + entries = self.filter_entries(log_stub, feed['entries']) # Filter URL links. urls = [] @@ -2087,24 +2113,43 @@ class RSSDownloader(Downloader): else: logger.warning(f'RSS entry {idx} did not have a link!') - if download.sub_downloader == 'video': + if prepared.sub_downloader_name == 'video': # Normalize video URLs before checking if already downloaded. # YouTube Shorts URLs (e.g., youtube.com/shorts/ABC123) must be converted to watch URLs # (e.g., youtube.com/watch?v=ABC123) to match what is stored in the database. from modules.videos.normalize_video_url import normalize_video_url urls = [normalize_video_url(i) for i in urls] + # Duration filtering performs network I/O (yt-dlp duration probe) — must run + # in execute, not finalize. This may probe URLs that DB-dedupe would have + # eliminated; fetch_video_duration is cached, so duplicate probes are cheap. + urls = await self.filter_videos(log_stub, urls) - # Only download new URLs. - with get_db_session() as session: - urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(session, *urls)]] - # Filter out URLs already in the Download table (pending downloads). + return ExecutedRSS(yt_channel_id=yt_channel_id, candidate_urls=urls) + + def finalize_download(self, session: Session, download: Download, + executed: ExecutedRSS) -> DownloadResult: + if executed.error: + return DownloadResult(success=False, error=executed.error) + + # Apply YT channel info to the Download, if not already applied. + if executed.yt_channel_id and not (download.location or download.collection_id): + self.apply_yt_channel(session, download, executed.yt_channel_id) + + # Resolve sub_downloader via the manager (matches prepare; cheap dict lookup). + sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader) + urls = executed.candidate_urls + + # DB dedupe: drop URLs already downloaded under this sub_downloader, plus URLs + # already queued in the Download table. + if sub_downloader and urls: + already = sub_downloader.already_downloaded(session, *urls) + already_urls = {a.url for a in already} + urls = [u for u in urls if u not in already_urls] existing_download_urls = {d[0] for d in session.query(Download.url).filter(Download.url.in_(urls))} - urls = [url for url in urls if url not in existing_download_urls] - # Remove skipped URLs before duration checks. (Typically done after this by the download worker). - urls = [i for i in urls if not download_manager.is_skipped(i)] + urls = [u for u in urls if u not in existing_download_urls] - if download.sub_downloader == 'video': # VideoDownloader - urls = await self.filter_videos(download, urls) + # Skip-list filter (manager-owned shared_ctx state, not DB). + urls = [u for u in urls if not download_manager.is_skipped(u)] logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}') @@ -2123,12 +2168,7 @@ class RSSDownloader(Downloader): # Fallback to legacy settings for backwards compatibility next_download_settings['destination'] = i - result = DownloadResult( - success=True, - downloads=urls, - settings=next_download_settings, - ) - return result + return DownloadResult(success=True, downloads=urls, settings=next_download_settings) @staticmethod async def filter_videos(download: Download, urls: list[str]) -> list[str]: @@ -2190,17 +2230,17 @@ class RSSDownloader(Downloader): return entries @staticmethod - def apply_yt_channel(download_id: int, yt_channel_id: str): - """Get Channel that matches this Download, apply Channel/Collection information to the Download.""" - with get_db_session() as session: - from modules.videos.models import Channel - channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}') - if channel: - download_ = Download.get_by_id(session, download_id) - download_.collection = channel.collection - download_.collection_id = channel.collection_id - download_.location = download_.location or channel.location - session.commit() + def apply_yt_channel(session: Session, download: Download, yt_channel_id: str): + """Apply Channel/Collection information to the given Download row. + + Caller (finalize_download) owns the session and commits. + """ + from modules.videos.models import Channel + channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}') + if channel: + download.collection = channel.collection + download.collection_id = channel.collection_id + download.location = download.location or channel.location @staticmethod def acceptable_bozo_errors(feed): diff --git a/wrolpi/files/test/test_scrape_downloader.py b/wrolpi/files/test/test_scrape_downloader.py index 4e389215..402583d3 100644 --- a/wrolpi/files/test/test_scrape_downloader.py +++ b/wrolpi/files/test/test_scrape_downloader.py @@ -53,7 +53,7 @@ async def test_scrape_html_downloader(test_directory, test_session, test_downloa assert_download_urls(['https://example.com/dir', ]) with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \ - mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download): + mock.patch('wrolpi.files.downloader.FileDownloader.execute_download', fake_file_do_download): await test_download_manager.wait_for_all_downloads() await await_switches() @@ -77,7 +77,7 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do assert_download_urls(['https://example.com/dir', ]) with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \ - mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download): + mock.patch('wrolpi.files.downloader.FileDownloader.execute_download', fake_file_do_download): await test_download_manager.wait_for_all_downloads() await await_switches() diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index e5f2109e..c98e5628 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -115,6 +115,7 @@ async def test_make_progress_callback(async_client): @pytest.mark.asyncio async def test_cancel_wrapper_outer_cancellation(test_session, test_download_manager, test_downloader): """cancel_wrapper propagates outer CancelledError and cancels the inner task.""" + from wrolpi.conftest import make_test_ctx download = test_download_manager.create_download(test_session, 'https://example.com/cancel', test_downloader.name) test_session.commit() @@ -127,7 +128,7 @@ async def test_cancel_wrapper_outer_cancellation(test_session, test_download_man inner_cancelled.set() raise - wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download)) + wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download, ctx=make_test_ctx())) # Give the wrapper time to start polling. await asyncio.sleep(0.3) @@ -461,7 +462,7 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now # Download is processed and successful, no longer "new". await test_download_manager.wait_for_all_downloads() - test_downloader.do_download.assert_called_once() + test_downloader.execute_download.assert_called_once() assert list(test_download_manager.get_new_downloads(test_session)) == [] downloads = list(test_download_manager.get_recurring_downloads(test_session)) assert len(downloads) == 1 @@ -487,10 +488,10 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now assert download.last_successful_download == now_ # Try the download, but it fails. - test_downloader.do_download.reset_mock() + test_downloader.execute_download.reset_mock() test_downloader.set_test_failure() await test_download_manager.wait_for_all_downloads() - test_downloader.do_download.assert_called_once() + test_downloader.execute_download.assert_called_once() download = test_session.query(Download).one() # Download is deferred, last successful download remains the same. assert download.is_deferred, download.status_code @@ -500,12 +501,12 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now assert download.next_download == expected # Try the download again, it finally succeeds. - test_downloader.do_download.reset_mock() + test_downloader.execute_download.reset_mock() now_ = fake_now(datetime(2020, 1, 1, 4, 0, 1, tzinfo=pytz.UTC)) test_downloader.set_test_success() test_download_manager.renew_recurring_downloads() await test_download_manager.wait_for_all_downloads() - test_downloader.do_download.assert_called_once() + test_downloader.execute_download.assert_called_once() download = test_session.query(Download).one() assert download.is_complete, download.status_code assert download.last_successful_download == now_ @@ -664,31 +665,36 @@ async def test_skip_already_downloaded_disabled(test_session, test_download_mana @pytest.mark.asyncio async def test_process_runner_timeout(async_client, test_session, test_directory): """A Downloader can cancel its download using a timeout.""" + from wrolpi.conftest import make_test_ctx # Default timeout of 3 seconds. downloader = Downloader('downloader', timeout=3) # Sleep for 8 seconds really takes 8 seconds. start = datetime.now() - await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=10) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory, + timeout=10, ctx=make_test_ctx()) elapsed = datetime.now() - start assert elapsed.total_seconds() >= 8 - # Download.timeout is obeyed. + # Downloader.timeout (class default) is obeyed. start = datetime.now() - await downloader.process_runner(Download(), ('sleep', '8'), test_directory) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory, ctx=make_test_ctx()) elapsed = datetime.now() - start assert 3 < elapsed.total_seconds() < 5 # One-off timeout is obeyed. start = datetime.now() - await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=1) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory, + timeout=1, ctx=make_test_ctx()) elapsed = datetime.now() - start assert 1 < elapsed.total_seconds() < 3 - # Global timeout is obeyed. - get_wrolpi_config().download_timeout = 3 + # ctx.download_timeout takes precedence over class/per-call timeouts. + # Production builds ctx.download_timeout from get_wrolpi_config().download_timeout + # (see DownloadContext.production); we bake the value directly here. start = datetime.now() - await downloader.process_runner(Download(), ('sleep', '8'), test_directory) + await downloader.process_runner(Download(), ('sleep', '8'), test_directory, + ctx=make_test_ctx(download_timeout=3)) elapsed = datetime.now() - start assert 2 < elapsed.total_seconds() < 5 @@ -707,6 +713,7 @@ exit 123 @pytest.mark.asyncio async def test_process_runner_output(async_client, test_directory, test_downloader): + from wrolpi.conftest import make_test_ctx script = test_directory / 'echo_script.sh' script.write_text(GOOD_SCRIPT) cmd = ('/bin/bash', script) @@ -714,7 +721,8 @@ async def test_process_runner_output(async_client, test_directory, test_download Download(), cmd, test_directory, - timeout=1 + timeout=1, + ctx=make_test_ctx(), ) assert result.return_code == 0 assert result.stdout == b'standard output\n' @@ -726,7 +734,8 @@ async def test_process_runner_output(async_client, test_directory, test_download Download(), cmd, test_directory, - timeout=1 + timeout=1, + ctx=make_test_ctx(), ) assert result.return_code == 123 assert result.stdout == b'bad standard output\n' @@ -1123,16 +1132,17 @@ async def test_child_downloads_have_parent_download_url(test_session, test_downl # Track which URLs have been processed to avoid returning children for child downloads processed_urls = set() - # Configure the test_downloader to return child downloads only for the parent URL - async def download_with_children(download, *a, **kwargs): + # Configure the test_downloader to return child downloads only for the parent URL. + # New signature matches Downloader.execute_download(prepared, ctx, download=...). + async def download_with_children(prepared, ctx, *, download=None, **kwargs): await asyncio.sleep(0.1) - if download.url == parent_url and parent_url not in processed_urls: + if download is not None and download.url == parent_url and parent_url not in processed_urls: processed_urls.add(parent_url) return DownloadResult(success=True, downloads=child_urls) # Return success without children for child downloads return DownloadResult(success=True) - test_downloader.do_download.side_effect = download_with_children + test_downloader.execute_download.side_effect = download_with_children # Create the parent download with a sub_downloader specified test_download_manager.create_download(test_session, parent_url, test_downloader.name, diff --git a/wrolpi/test/test_rss.py b/wrolpi/test/test_rss.py index a8d778b9..df6eea2e 100644 --- a/wrolpi/test/test_rss.py +++ b/wrolpi/test/test_rss.py @@ -9,9 +9,17 @@ from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadMana class RSSHTTPDownloader(Downloader): + """Test fixture: a no-op sub-downloader RSS hands child URLs to.""" name = 'rss_http' - async def do_download(self, download: Download) -> DownloadResult: + def prepare_download(self, session: Session, download: Download): + # Return a non-None sentinel so the dispatch routes to the phase-split path. + return object() + + async def execute_download(self, prepared, ctx, download: Download = None): + return None + + def finalize_download(self, session: Session, download: Download, executed) -> DownloadResult: return DownloadResult(success=True) def already_downloaded(self, session: Session, *urls: List[str]): @@ -295,3 +303,180 @@ async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_sessio # The new video should be added for download with the normalized URL. assert 'https://www.youtube.com/watch?v=NEW456' in download_urls, \ 'New video should be added to downloads with normalized URL' + + +# --------------------------------------------------------------------------- +# Phase-split unit tests. +# +# These exercise prepare_download / execute_download / finalize_download in +# isolation. They drive the new code paths directly rather than through the +# DownloadManager + wait_for_all_downloads polling loop the integration tests +# above use. +# --------------------------------------------------------------------------- +from wrolpi.downloader import ExecutedRSS, PreparedRSS # noqa: E402 + +from modules.archive.conftest import make_test_ctx # noqa: E402 + + +def test_prepare_resolves_sub_downloader(test_session, test_download_manager): + """prepare_download finds the sub_downloader instance and surfaces settings.""" + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + http_downloader = RSSHTTPDownloader() + test_download_manager.register_downloader(http_downloader) + + download = Download( + url='https://example.com/feed', + downloader='rss', + sub_downloader='rss_http', + settings={'title_include': 'foo'}, + ) + + prepared = rss_downloader.prepare_download(test_session, download) + + assert isinstance(prepared, PreparedRSS) + assert prepared.url == 'https://example.com/feed' + assert prepared.sub_downloader_name == 'rss_http' + assert prepared.sub_downloader is http_downloader + assert prepared.settings == {'title_include': 'foo'} + + +def test_prepare_unknown_sub_downloader_raises(test_session, test_download_manager): + """No matching sub_downloader → manager raises InvalidDownload (manager will record + the failure as a deferred download). The unreachable ValueError branch in the legacy + code was dead because find_downloader_by_name raises rather than returning None.""" + from wrolpi.errors import InvalidDownload + + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + + download = Download( + url='https://example.com/feed', + downloader='rss', + sub_downloader='no_such_downloader', + ) + + with pytest.raises(InvalidDownload, match='Cannot find downloader'): + rss_downloader.prepare_download(test_session, download) + + +@pytest.mark.asyncio +async def test_execute_failed_parse_returns_error(test_session, test_download_manager): + """A bozo feed without acceptable errors short-circuits with executed.error.""" + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + http_downloader = RSSHTTPDownloader() + test_download_manager.register_downloader(http_downloader) + + prepared = PreparedRSS( + url='https://example.com/feed', + sub_downloader_name='rss_http', + sub_downloader=http_downloader, + settings={}, + ) + + with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: + mock_parse_feed.return_value = dict(bozo=1, bozo_exception=Exception('nope'), entries=[]) + executed = await rss_downloader.execute_download(prepared, make_test_ctx()) + + assert executed.candidate_urls == [] + assert executed.error == 'Failed to parse RSS feed' + + +@pytest.mark.asyncio +async def test_execute_collects_links_and_yt_channel(test_session, test_download_manager): + """A well-formed feed yields candidate_urls plus the yt_channelid for finalize.""" + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + http_downloader = RSSHTTPDownloader() + test_download_manager.register_downloader(http_downloader) + + prepared = PreparedRSS( + url='https://example.com/feed', + sub_downloader_name='rss_http', + sub_downloader=http_downloader, + settings={}, + ) + + with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed: + mock_parse_feed.return_value = dict( + bozo=0, + feed=dict(yt_channelid='abc123'), + entries=[ + dict(link='https://example.com/a'), + dict(link='https://example.com/b'), + ], + ) + executed = await rss_downloader.execute_download(prepared, make_test_ctx()) + + assert executed.error is None + assert executed.yt_channel_id == 'abc123' + assert executed.candidate_urls == ['https://example.com/a', 'https://example.com/b'] + + +def test_finalize_propagates_executed_error(test_session, test_download_manager): + """An executed-phase error becomes a failure DownloadResult.""" + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + + download = Download(url='https://example.com/feed', downloader='rss', sub_downloader='rss_http') + executed = ExecutedRSS(yt_channel_id=None, candidate_urls=[], error='Failed to parse RSS feed') + + result = rss_downloader.finalize_download(test_session, download, executed) + + assert result.success is False + assert result.error == 'Failed to parse RSS feed' + + +def test_finalize_dedupes_existing_downloads(test_session, test_download_manager): + """URLs already present in the Download table are filtered out before being returned + as child downloads.""" + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + http_downloader = RSSHTTPDownloader() + test_download_manager.register_downloader(http_downloader) + + # Pre-existing pending download for /a. + test_session.add(Download(url='https://example.com/a', downloader='rss_http')) + test_session.commit() + + download = Download( + url='https://example.com/feed', + downloader='rss', + sub_downloader='rss_http', + ) + executed = ExecutedRSS( + yt_channel_id=None, + candidate_urls=['https://example.com/a', 'https://example.com/b', 'https://example.com/c'], + ) + + result = rss_downloader.finalize_download(test_session, download, executed) + + assert result.success is True + assert sorted(result.downloads) == ['https://example.com/b', 'https://example.com/c'] + + +def test_finalize_passes_destination_from_column(test_session, test_download_manager, test_directory): + """Child downloads inherit destination from the RSS row's destination column.""" + rss_downloader = RSSDownloader() + test_download_manager.register_downloader(rss_downloader) + http_downloader = RSSHTTPDownloader() + test_download_manager.register_downloader(http_downloader) + + dest_path = test_directory / 'archive/custom/path' + download = Download( + url='https://example.com/feed', + downloader='rss', + sub_downloader='rss_http', + destination=str(dest_path), + settings={}, + ) + executed = ExecutedRSS( + yt_channel_id=None, + candidate_urls=['https://example.com/x'], + ) + + result = rss_downloader.finalize_download(test_session, download, executed) + + assert result.success is True + assert result.settings['destination'].endswith('archive/custom/path')