Converting RSS and Video Downloaders to new testable strategy.

This commit is contained in:
Roland 2026-05-07 15:55:38 -06:00 • committed by lrnselfreliance
parent 8f49f33ba2
commit 8f0bb78722
9 changed files with 851 additions and 383 deletions

View file

@ -9,26 +9,10 @@ import pytz
from modules.archive.lib import archive_strftime
from modules.archive.models import Archive
from wrolpi.cmd import run_command
from wrolpi.collections import Collection
from wrolpi.downloader import DownloadContext
def make_test_ctx(*, is_cancelled=None, can_download=None,
outside_download_window=None, run_command_=None,
download_timeout=None) -> DownloadContext:
"""Build a DownloadContext suitable for downloader unit tests.
Defaults to a never-cancelled context that uses the real run_command. Tests can
override any callable to simulate cancellation, throttling, or fake subprocess output.
"""
return DownloadContext(
is_cancelled=is_cancelled or (lambda: False),
can_download=can_download or (lambda: True),
outside_download_window=outside_download_window or (lambda: False),
run_command=run_command_ or run_command,
download_timeout=download_timeout,
)
# Re-export make_test_ctx so existing `from modules.archive.conftest import make_test_ctx`
# call sites keep working. The canonical implementation lives in wrolpi/conftest.py.
from wrolpi.conftest import make_test_ctx # noqa: F401
@pytest.fixture

View file

@ -213,12 +213,14 @@ async def test_channel_download_requires_refresh(
downloaded_urls = []
async def do_download(_, download):
# Track all downloaded URLs.
downloaded_urls.append(download.url)
async def fake_execute(_, prepared, ctx, download=None):
# Track all downloaded URLs and short-circuit by returning a DownloadResult.
# Per the dispatch contract (wrolpi/downloader.py: signal_download_download),
# if execute_download returns a DownloadResult, finalize_download is skipped.
downloaded_urls.append(prepared.url)
return DownloadResult(success=True)
with mock.patch('modules.videos.downloader.VideoDownloader.do_download', do_download):
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download', fake_execute):
entries = [
dict(id=vid.source_id, view_count=0, webpage_url='https://example.com/1'), # Already downloaded.
dict(id='not downloaded', view_count=0, webpage_url='https://example.com/2'),

View file

@ -1,5 +1,6 @@
#! /usr/bin/env python3
import copy
import dataclasses
import json
import logging
import os.path
@ -8,6 +9,7 @@ import re
import time
import traceback
from abc import ABC
from dataclasses import dataclass, field
from datetime import timedelta
from typing import Tuple, List, Dict, Optional
from urllib.parse import urlparse, parse_qs
@ -24,7 +26,7 @@ from wrolpi.common import logger, get_media_directory, escape_file_name, resolve
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult, \
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, \
clear_download_progress, _parse_size, make_progress_callback, get_download_cache_config
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError, ValidationError
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
@ -399,6 +401,40 @@ def is_youtube_rss_feed_url(url: str) -> bool:
return bool(YOUTUBE_RSS_FEED_URL_RE.match(url or ''))
@dataclass
class PreparedChannel:
"""Plan produced by ChannelDownloader.prepare_download.
Cheap pre-flight: validates the URL is not an RSS feed, snapshots the settings
fields execute and finalize need so they don't have to read back through the
Download row. All real work — yt-dlp metadata fetch, channel resolution, file
indexing — happens in execute_download.
"""
url: str
settings: dict
destination: Optional[str]
tag_names: Optional[List[str]]
@dataclass
class ExecutedChannel:
"""Output of ChannelDownloader.execute_download.
`info_json` is the resolved yt-dlp metadata; `channel_id` and `collection_id` are
the IDs assigned by the channel-resolution helper. `child_settings` is the dict
that will be passed to every child VideoDownloader. `error` is set when the
catalog update raised — finalize converts that into a failure DownloadResult
while still mutating the parent Download with sub_downloader/info_json so the
next attempt has them.
"""
info_json: Optional[dict]
channel_id: Optional[int]
collection_id: Optional[int]
location: Optional[str]
child_settings: dict
error: Optional[str] = None
class ChannelDownloader(Downloader, ABC):
"""Handles downloading of videos in a Channel or Playlist."""
name = 'video_channel'
@ -413,9 +449,7 @@ class ChannelDownloader(Downloader, ABC):
# A playlist may have an id different from its channel.
return info['id'] != info.get('channel_id')
async def do_download(self, download: Download) -> DownloadResult:
"""Update a Channel's catalog, then schedule downloads of every missing video."""
_log_ytdlp_versions()
def prepare_download(self, session: Session, download: Download) -> PreparedChannel:
if is_youtube_rss_feed_url(download.url):
# yt-dlp returns no `uploader` or `channel_id` for an RSS feed URL, so falling through
# would create a junk channel named after `webpage_url_basename` ("videos.xml").
@ -423,8 +457,23 @@ class ChannelDownloader(Downloader, ABC):
f'Refusing to handle RSS feed URL with ChannelDownloader: {download.url}.'
f' Use the RSS Downloader (sub_downloader=video) instead.'
)
return PreparedChannel(
url=download.url,
settings=download.settings or dict(),
destination=str(download.destination) if download.destination else None,
tag_names=download.tag_names,
)
async def execute_download(self, prepared: PreparedChannel, ctx: DownloadContext,
download: Download = None) -> ExecutedChannel:
"""Fetch the channel/playlist metadata via yt-dlp, get/create the local Channel
record, and update its catalog. Channel creation + catalog update use a
short-lived self-managed session so the manager's outer session isn't held
during the file_worker await.
"""
_log_ytdlp_versions()
try:
info = extract_info(download.url, process=False)
info = extract_info(prepared.url, process=False)
except Exception as e:
if _bot_blocked(e):
raise BotBlockedDownloadError(str(e)) from e
@ -434,86 +483,103 @@ class ChannelDownloader(Downloader, ABC):
entries = info.get('entries')
if entries and isinstance(entries[0], Dict) and entries[0].get('entries'):
# Entries contains more channels/playlist, tell the Maintainer to specify the URL.
raise UnrecoverableDownloadError(f'Unable to download {download.url} because the URL is ambiguous.'
raise UnrecoverableDownloadError(f'Unable to download {prepared.url} because the URL is ambiguous.'
f' Try requesting the videos only:'
f' https://example.com -> https://example.com/videos')
download_settings = download.settings or dict()
channel_tag_name = i[0] if (i := download_settings.get('channel_tag_name')) else None
download.sub_downloader = video_downloader.name
download.info_json = info
if session := Session.object_session(download):
# May not have a session during testing.
session.commit()
channel_tag_name = i[0] if (i := prepared.settings.get('channel_tag_name')) else None
name = info.get('uploader') or info.get('webpage_url_basename')
if not name:
raise ValueError(f'Could not find name')
channel_source_id = info.get('channel_id') or info.get('id')
channel_url = info.get('channel_url') or info.get('uploader_url') or download.url
channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name,
destination=download.destination)
channel.dict() # get all attributes while we have the session.
channel_url = info.get('channel_url') or info.get('uploader_url') or prepared.url
# Link this channel download to the channel's collection.
if channel and channel.collection_id and not download.collection_id:
download.collection_id = channel.collection_id
if session:
session.commit()
channel_id = None
collection_id = None
location = None
with get_db_session(commit=True) as session:
channel = get_or_create_channel(session, channel_source_id, channel_url, name, channel_tag_name,
destination=prepared.destination)
channel.dict() # get all attributes while we have the session.
if channel and channel.id:
channel_id = channel.id
collection_id = channel.collection_id
location = f'/videos/channel/{channel.id}/video'
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
# The settings to send to the VideoDownloader.
settings = dict()
if channel:
settings.update(dict(collection_id=channel.collection_id, channel_url=download.url))
if download.destination:
destination = get_absolute_media_path(download.destination)
settings['destination'] = str(destination) # Need str for JSON conversion
if download.tag_names:
settings['tag_names'] = download.tag_names
# The settings to send to the VideoDownloader children.
child_settings = dict()
if channel_id:
child_settings.update(dict(collection_id=collection_id, channel_url=prepared.url))
if prepared.destination:
destination_abs = get_absolute_media_path(prepared.destination)
child_settings['destination'] = str(destination_abs) # Need str for JSON conversion
if prepared.tag_names:
child_settings['tag_names'] = prepared.tag_names
# Pass all channel-level inheritable settings to child video downloads.
for key in CHANNEL_INHERITABLE_SETTINGS:
if key in download_settings:
settings[key] = download_settings[key]
if key in prepared.settings:
child_settings[key] = prepared.settings[key]
is_a_playlist = self.is_a_playlist(info)
try:
if not is_a_playlist:
await self.prepare_channel_for_downloads(download, channel)
else:
if not is_a_playlist and channel_id is not None:
await self.prepare_channel_for_downloads(info, channel_id)
elif is_a_playlist:
logger.debug('Not updating channel because this is a playlist')
downloads = self.get_missing_videos(session, download)
return DownloadResult(
success=True,
location=location,
downloads=downloads,
settings=settings,
)
except Exception as e:
if PYTEST:
raise
kind = 'playlist' if is_a_playlist else 'channel'
logger.error(f'Failed to update catalog of {kind} {download.url}', exc_info=e)
return DownloadResult(
success=False,
location=location,
logger.error(f'Failed to update catalog of {kind} {prepared.url}', exc_info=e)
return ExecutedChannel(
info_json=info, channel_id=channel_id, collection_id=collection_id,
location=location, child_settings=child_settings,
error=str(traceback.format_exc()),
settings=settings,
)
return ExecutedChannel(
info_json=info, channel_id=channel_id, collection_id=collection_id,
location=location, child_settings=child_settings,
)
def finalize_download(self, session: Session, download: Download,
executed: ExecutedChannel) -> DownloadResult:
"""Mutate the parent Download (sub_downloader / info_json / collection_id) and
return the list of missing video URLs as child downloads."""
download.sub_downloader = video_downloader.name
download.info_json = executed.info_json
if executed.collection_id and not download.collection_id:
download.collection_id = executed.collection_id
if executed.error:
return DownloadResult(
success=False,
location=executed.location,
error=executed.error,
settings=executed.child_settings,
)
downloads = self.get_missing_videos(session, download)
return DownloadResult(
success=True,
location=executed.location,
downloads=downloads,
settings=executed.child_settings,
)
@classmethod
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
"""Update the Channel's video catalog. Refresh the Channel's files if necessary."""
logger.debug(f'Preparing {channel} for downloads')
channel_id = channel.id
update_channel_catalog(channel, download.info_json)
async def prepare_channel_for_downloads(cls, info_json: dict, channel_id: int):
"""Update the Channel's video catalog. Refresh the Channel's files if necessary.
Takes channel_id (not Channel) and info_json (not Download) so the caller doesn't
have to hold the manager's session open while we await the file_worker job.
"""
with get_db_session() as session:
channel = get_channel(session, channel_id=channel_id, return_dict=False)
logger.debug(f'Preparing {channel} for downloads')
update_channel_catalog(channel, info_json)
if not channel.videos or not channel.refreshed:
logger.warning(f'Refreshing videos in {channel.directory} for channel {channel}')
job_id = channel.refresh_files(channel.id, send_events=False)
@ -589,6 +655,42 @@ class ChannelDownloader(Downloader, ABC):
return downloads
@dataclass
class PreparedVideo:
"""Plan produced by VideoDownloader.prepare_download.
`info_json` is None unless the upstream download row already has it (e.g. set by
ChannelDownloader before queuing this child). `location` is the existing
Video.location if any, used for error attribution if the new download fails.
"""
url: str # normalized
settings: dict
tag_names: List[str] = field(default_factory=list)
destination: Optional[pathlib.Path] = None
info_json: Optional[dict] = None
location: Optional[str] = None # existing video, for error reports
@dataclass
class ExecutedVideo:
"""Output of VideoDownloader.execute_download once the file is on disk and validated.
Carries everything finalize_download needs to persist the Video record without
re-running any I/O. When the metadata-only flow runs, execute returns a
DownloadResult directly and the dispatch skips finalize_download entirely.
"""
url: str # normalized
video_path: pathlib.Path
video_paths: List[pathlib.Path] # all files in the FileGroup, post-glob+normalize
info_json: dict
entry: dict
channel_id: Optional[int]
audio_only: bool
tag_names: List[str]
settings: dict
location: Optional[str] # for error reporting
class VideoDownloader(Downloader, ABC):
"""Downloads a single video.
@ -608,84 +710,126 @@ class VideoDownloader(Downloader, ABC):
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
return file_groups
async def do_download(self, download: Download) -> DownloadResult:
def prepare_download(self, session: Session, download: Download) -> PreparedVideo:
"""Sync prep: validate attempts cap, normalize URL, copy upstream-provided settings
onto the Download row, and look up any pre-existing Video for error attribution.
Holding the session here is brief: no network or subprocess work runs in this
phase. extract_info and channel resolution move to execute_download.
"""
if (download.attempts or 0) >= 10:
raise UnrecoverableDownloadError('Max download attempts reached')
url = normalize_video_url(download.url)
# ChannelDownloader passes destination/tag_names via settings; copy onto the row
# so the rest of the flow can read them off the Download object as before.
settings = download.settings or dict()
download.destination = download.destination or settings.get('destination')
tag_names = download.tag_names = download.tag_names or settings.get('tag_names') or []
# If a Video already exists at this URL, capture its location so a failure later
# can still tell the user where the previous attempt landed.
existing = Video.get_by_url(session, url)
location = existing.location if existing else None
return PreparedVideo(
url=url,
settings=settings,
tag_names=list(tag_names),
destination=download.destination,
info_json=download.info_json,
location=location,
)
async def execute_download(self, prepared: PreparedVideo, ctx: DownloadContext,
download: Download = None):
"""Async I/O phase: extract_info, channel resolution (brief session),
yt-dlp subprocess, file resolution, ffmpeg validation, sibling file globbing.
Returns either an ExecutedVideo (file is on disk; finalize persists) or a
DownloadResult (an early-exit case the dispatch will surface verbatim — used
for the metadata-only refresh flow and recoverable failures like a missing
file or incomplete stream).
"""
_log_ytdlp_versions()
with get_db_session() as session:
if download.attempts >= 10:
raise UnrecoverableDownloadError('Max download attempts reached')
url = normalize_video_url(download.url)
# Copy settings into Download (they may be from the ChannelDownloader)
settings = download.settings or dict()
download.destination = download.destination or settings.get('destination')
tag_names = download.tag_names = download.tag_names or settings.get('tag_names')
# Video may have been downloaded previously, get its location for error reporting.
video_ = Video.get_by_url(session, url)
location = video_.location if video_ else None
url = prepared.url
settings = prepared.settings
location = prepared.location
# extract_info is sync but expensive; legacy held a session across it. We don't.
info_json = prepared.info_json
if not info_json:
try:
download.info_json = download.info_json or extract_info(url)
info_json = extract_info(url)
except yt_dlp.utils.DownloadError as e:
if _bot_blocked(e):
raise BotBlockedDownloadError(str(e)) from e
# Video may be private.
# Video may be private — surface as a recoverable failure.
try:
raise RuntimeError('Failed to extract_info') from e
except Exception as e:
except Exception as e2:
return DownloadResult(
success=False,
location=location,
error='\n'.join(traceback.format_exception(e)),
error='\n'.join(traceback.format_exception(e2)),
)
if not download.info_json:
raise ValueError(f'Cannot download video with no info_json.')
if not info_json:
raise ValueError(f'Cannot download video with no info_json.')
channel = await self._get_channel(session, download)
# Brief session for channel resolution. _get_channel reads info_json (already
# populated above) and may create a Channel record.
download_for_channel = download if download is not None else Download(
url=url, settings=settings, destination=prepared.destination,
)
download_for_channel.info_json = info_json
with get_db_session(commit=True) as session:
channel = await self._get_channel(session, download_for_channel)
channel_id = channel.id if channel else None
destination = download.destination
channel_directory = channel.directory if channel else None
if destination:
# Download to the directory specified in the settings.
out_dir = destination
logger.debug(f'Downloading {url} to destination {destination} from settings')
elif channel:
out_dir = channel.directory
logger.debug(f'Downloading {url} to channel directory: {channel.directory}')
else:
# Download to the default directory if this video has no channel.
out_dir = get_no_channel_directory()
logger.debug(f'Downloading {url} to default directory')
if prepared.destination:
out_dir = prepared.destination
logger.debug(f'Downloading {url} to destination {prepared.destination} from settings')
elif channel_directory:
out_dir = channel_directory
logger.debug(f'Downloading {url} to channel directory: {channel_directory}')
else:
out_dir = get_no_channel_directory()
logger.debug(f'Downloading {url} to default directory')
out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir)
out_dir = get_absolute_media_path(out_dir)
out_dir.mkdir(exist_ok=True, parents=True)
# Make output directory. (Maybe string from settings)
out_dir = out_dir if isinstance(out_dir, pathlib.Path) else pathlib.Path(out_dir)
out_dir = get_absolute_media_path(out_dir)
out_dir.mkdir(exist_ok=True, parents=True)
if tag_names and logger.isEnabledFor(logging.DEBUG):
logger.debug(f'Downloading {url} with {tag_names=}')
if prepared.tag_names and logger.isEnabledFor(logging.DEBUG):
logger.debug(f'Downloading {url} with tag_names={prepared.tag_names}')
effective = get_effective_video_settings(settings)
audio_only = effective.get('audio_only', False)
audio_format = effective.get('audio_format', 'mp3')
logs = None # noqa
# download_for_log carries .id (for progress) and .url (for log messages).
# Falling back to a stub keeps unit tests viable.
download_for_log = download if download is not None else Download(url=url)
try:
video_resolutions = effective['video_resolutions']
# yt-dlp expects a string like so: 299+140,298+140,bestvideo*+bestaudio/best
video_resolutions_str = ','.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i])
video_format = effective['video_format']
video_path, entry = self.prepare_filename(url, out_dir, video_resolutions_str, video_format,
audio_only=audio_only, audio_format=audio_format)
if settings.get('download_metadata_only'):
# User has requested refresh of video metadata, skip the rest of the video downloader.
return await self.download_info_json(download, video_path)
# download_info_json manages its own DB sessions; we surface its DownloadResult
# directly and the dispatch skips finalize_download.
download_for_metadata = download if download is not None else Download(
url=url, settings=settings,
)
download_for_metadata.info_json = info_json
return await self.download_info_json(download_for_metadata, video_path)
# Common yt-dlp flags shared by both video and audio-only downloads.
cmd = (
str(YT_DLP_BIN),
'--match-filter', '!is_live',
@ -719,27 +863,26 @@ class VideoDownloader(Downloader, ABC):
if effective['sleep_requests'] > 0:
cmd = (*cmd, '--sleep-requests', str(effective['sleep_requests']))
on_stdout = make_progress_callback(download.id, parse_ytdlp_progress)
on_stdout = make_progress_callback(download_for_log.id, parse_ytdlp_progress)
try:
if cookies_unlocked():
# Use encrypted cookies file
logger.info(f'Using encrypted cookies for video download: {url}')
with cookies_for_download() as cookies_path:
cmd = (*cmd, '--cookies', str(cookies_path))
if effective['user_agent']:
cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
cmd = (*cmd, '-o', video_path, url)
result = await self.process_runner(download, cmd, out_dir, debug=True,
stdout_callback=on_stdout)
result = await self.process_runner(download_for_log, cmd, out_dir, debug=True,
stdout_callback=on_stdout, ctx=ctx)
else:
# No cookies available
logger.info(f'No cookies available for video download: {url}')
cmd = (*cmd, '-o', video_path, url)
result = await self.process_runner(download, cmd, out_dir, debug=True,
stdout_callback=on_stdout)
result = await self.process_runner(download_for_log, cmd, out_dir, debug=True,
stdout_callback=on_stdout, ctx=ctx)
finally:
clear_download_progress(download.id)
clear_download_progress(download_for_log.id)
stdout = result.stdout.decode()
stderr = result.stderr.decode()
@ -747,36 +890,27 @@ class VideoDownloader(Downloader, ABC):
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}'
if _bot_blocked(error):
raise BotBlockedDownloadError(f'Bot detection or invalid cookies: {error}')
return DownloadResult(
success=False,
error=error,
location=location,
)
return DownloadResult(success=False, error=error, location=location)
output_format = audio_output_extension(audio_format) if audio_only else video_format
preferred_path = video_path.with_suffix(f'.{output_format}')
if not video_path.is_file() and preferred_path.is_file():
# Prepared filename does not exist, but file with preferred extension does, it was probably
# remuxed/converted by yt-dlp.
video_path = preferred_path
logger.info(f'Using preferred file which exists: {preferred_path}')
if not video_path.is_file():
# yt-dlp may have appended the real extension to a bogus one (e.g. .NA.mp4 from a playlist entry).
# yt-dlp may have appended the real extension to a bogus one.
appended_path = video_path.parent / f'{video_path.name}.{output_format}'
if appended_path.is_file():
video_path = appended_path
logger.info(f'Using file with appended extension: {appended_path}')
if video_path.suffix == '.part':
return DownloadResult(
success=False,
error=f'Video file that completed was a .part file: {video_path}',
)
return DownloadResult(success=False,
error=f'Video file that completed was a .part file: {video_path}')
if not video_path.is_file():
error = f'{stdout}\n\n\n{stderr}\n\n' \
f'Video file could not be found! {video_path}'
error = f'{stdout}\n\n\n{stderr}\n\nVideo file could not be found! {video_path}'
retry_seconds = None
if '!is_live' in stdout:
error = f'{stdout}\n\nVideo was live and did not finish downloading.'
@ -784,82 +918,29 @@ class VideoDownloader(Downloader, ABC):
if ' live event ' in stdout:
error = f'{stdout}\n\nVideo will be live and did not finish downloading.'
retry_seconds = 3600
return DownloadResult(
success=False,
error=error,
location=location,
retry_seconds=retry_seconds,
)
return DownloadResult(success=False, error=error, location=location,
retry_seconds=retry_seconds)
if not audio_only and not ffmpeg_video_complete(video_path):
return DownloadResult(
success=False,
error='Video was incomplete',
location=location,
)
return DownloadResult(success=False, error='Video was incomplete', location=location)
with get_db_session(commit=True) as session:
# Find any files downloaded with the video (poster, caption, etc.).
video_paths = glob_shared_stem(video_path)
# Delete any leftover part files now that we have verified the video is complete.
video_paths = self._delete_part_files(video_path, video_paths)
# Rename any special files that do not match the stem.
video_paths = self.normalize_video_file_names(video_path, video_paths)
# Create the Video record.
video = Video.from_paths(session, *video_paths)
video.source_id = entry['id']
video.channel_id = channel_id
video_id, video_info_json_path = video.id, video.info_json_path
if video_info_json_path and (new_info_json := video.clean_info_json()):
video.replace_info_json(new_info_json, clean=False)
# File is on disk and complete. Glob siblings, drop part files, normalize names.
video_paths = glob_shared_stem(video_path)
video_paths = self._delete_part_files(video_path, video_paths)
video_paths = self.normalize_video_file_names(video_path, video_paths)
if parent_download_url := settings.get('parent_download_url'):
try:
video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
except ValueError as e:
logger.warning(f'Failed to update wrolpi json for {video}: {e}')
# Update channel's collection's file_format to track the format used
if channel_id:
from modules.videos.models import Channel
from modules.videos.lib import save_channels_config
channel_obj = session.query(Channel).get(channel_id)
if channel_obj and channel_obj.collection and not channel_obj.collection.file_format:
video_config = get_videos_downloader_config()
channel_obj.collection.file_format = video_config.file_name_format
save_channels_config.activate_switch()
session.commit()
with get_db_session(commit=True) as session:
# Second session is started because SQLAlchemy will forget what we have done.
video = Video.get_by_id(session, video_id)
if tag_names:
existing_names = video.file_group.tag_names
for name in tag_names:
if name not in existing_names:
video.add_tag(session, name)
location = video.location
await video.get_ffprobe_json()
# Check that the download has the expected streams.
if not audio_only and not video.get_streams_by_codec_type('video'):
return DownloadResult(
success=False,
error='Video was downloaded but did not contain video stream',
location=location,
)
if not video.get_streams_by_codec_type('audio'):
return DownloadResult(
success=False,
error='Download did not contain audio stream',
location=location,
)
logger.info(f'Successfully downloaded video {url} {video}')
with get_db_session(commit=True) as session:
await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path)
return ExecutedVideo(
url=url,
video_path=video_path,
video_paths=video_paths,
info_json=info_json,
entry=entry,
channel_id=channel_id,
audio_only=audio_only,
tag_names=prepared.tag_names,
settings=settings,
location=location,
)
except BotBlockedDownloadError:
raise
@ -874,30 +955,73 @@ class VideoDownloader(Downloader, ABC):
if _skip_download(e):
# The video failed to download, and the error will never be fixed. Skip it forever.
try:
source_id = download.info_json.get('id')
source_id = (info_json or {}).get('id')
logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
f'attempt to download it again.')
download.add_to_skip_list()
if download is not None:
download.add_to_skip_list()
except Exception:
# Could not skip this video, it may not have a channel.
logger.warning(f'Could not skip video {url}')
# Skipped downloads should not be tried again.
raise UnrecoverableDownloadError() from e
# Download did not succeed, try again later.
if logs and (stderr := logs.get('stderr')):
error = f'{stderr}\n\n{traceback.format_exc()}'
else:
error = str(traceback.format_exc())
return DownloadResult(success=False, error=error, location=location)
return DownloadResult(success=False, error=str(traceback.format_exc()), location=location)
logger.debug(f'Downloaded video {location=}')
async def finalize_download(self, session: Session, download: Download,
executed: ExecutedVideo) -> DownloadResult:
"""Async DB-bound persistence: create the Video record, normalize info_json,
update the channel's collection format, apply tags, run ffprobe, validate
streams, and dedupe. Caller commits.
"""
# Create the Video record.
video = Video.from_paths(session, *executed.video_paths)
video.source_id = executed.entry['id']
video.channel_id = executed.channel_id
if video.info_json_path and (new_info_json := video.clean_info_json()):
video.replace_info_json(new_info_json, clean=False)
result = DownloadResult(
success=True,
location=location,
)
return result
if parent_download_url := executed.settings.get('parent_download_url'):
try:
video.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
except ValueError as e:
logger.warning(f'Failed to update wrolpi json for {video}: {e}')
# Update channel's collection's file_format to track the format used.
if executed.channel_id:
from modules.videos.lib import save_channels_config
channel_obj = session.query(Channel).get(executed.channel_id)
if channel_obj and channel_obj.collection and not channel_obj.collection.file_format:
video_config = get_videos_downloader_config()
channel_obj.collection.file_format = video_config.file_name_format
save_channels_config.activate_switch()
# Apply tags now that the file_group is bound to the session.
if executed.tag_names:
existing_names = video.file_group.tag_names
for name in executed.tag_names:
if name not in existing_names:
video.add_tag(session, name)
location = video.location
# ffprobe runs the ffprobe subprocess (may persist .ffprobe.json on disk).
await video.get_ffprobe_json()
if not executed.audio_only and not video.get_streams_by_codec_type('video'):
return DownloadResult(success=False,
error='Video was downloaded but did not contain video stream',
location=location)
if not video.get_streams_by_codec_type('audio'):
return DownloadResult(success=False,
error='Download did not contain audio stream',
location=location)
logger.info(f'Successfully downloaded video {executed.url} {video}')
# Delete duplicate Videos that share the URL or source_id. Runs against the same
# session as the new Video record so deletion is atomic with creation.
await Video.delete_duplicate_videos(session, download.url, executed.entry['id'],
executed.video_path)
return DownloadResult(success=True, location=location)
@staticmethod
async def _get_channel(session: Session, download: Download) -> Optional[Channel]:

View file

@ -482,7 +482,9 @@ async def test_channel_downloader_rejects_rss_feed_url(test_session, mock_video_
}
with pytest.raises(UnrecoverableDownloadError):
await channel_downloader.do_download(download)
# RSS-feed rejection happens in the prepare phase (sync), so we don't even
# need to drive execute_download here.
channel_downloader.prepare_download(test_session, download)
# No Channel should have been created from the RSS URL.
assert test_session.query(Channel).count() == 0
@ -668,9 +670,13 @@ async def test_download_playlist(test_session, test_directory, mock_video_extrac
mock_video_extract_info.return_value = example_playlist_json # Playlist info is fetched first.
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download:
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download:
mock_video_do_download.return_value = DownloadResult(success=True) # Don't download the videos.
result = await channel_downloader.do_download(download)
# Drive the phase-split API directly: prepare → execute → finalize.
from modules.archive.conftest import make_test_ctx
prepared = channel_downloader.prepare_download(test_session, download)
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
result = channel_downloader.finalize_download(test_session, download, executed)
assert result.success is True, 'Download was not successful'
assert set(result.downloads) == {
'https://www.youtube.com/watch?v=video_2_url', # Shorts is converted to regular URL.
@ -899,11 +905,14 @@ async def test_channel_downloader_sets_collection_id(test_session, channel_facto
mock_video_extract_info.return_value = mock_info
# Mock Channel.refresh_files and file_worker.wait_for_job to avoid background tasks
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \
mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
mock_video_do_download.return_value = DownloadResult(success=True)
await channel_downloader.do_download(download)
from modules.archive.conftest import make_test_ctx
prepared = channel_downloader.prepare_download(test_session, download)
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
channel_downloader.finalize_download(test_session, download, executed)
await await_switches()
@ -947,11 +956,14 @@ async def test_channel_passes_all_inheritable_settings_to_video(test_session, ch
}
mock_video_extract_info.return_value = mock_info
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \
mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
mock_video_do_download.return_value = DownloadResult(success=True)
result = await channel_downloader.do_download(download)
from modules.archive.conftest import make_test_ctx
prepared = channel_downloader.prepare_download(test_session, download)
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
result = channel_downloader.finalize_download(test_session, download, executed)
# Verify all channel-level inheritable settings are passed through
for key in ('video_resolutions', 'video_format', 'writesubtitles', 'sleep_requests', 'user_agent'):
@ -991,11 +1003,14 @@ async def test_channel_does_not_pass_continue_dl_nooverwrites(test_session, chan
}
mock_video_extract_info.return_value = mock_info
with mock.patch('modules.videos.downloader.VideoDownloader.do_download') as mock_video_do_download, \
with mock.patch('modules.videos.downloader.VideoDownloader.execute_download') as mock_video_do_download, \
mock.patch.object(Channel, 'refresh_files', return_value='test-job-id'), \
mock.patch('modules.videos.downloader.file_worker.wait_for_job', new_callable=mock.AsyncMock):
mock_video_do_download.return_value = DownloadResult(success=True)
result = await channel_downloader.do_download(download)
from modules.archive.conftest import make_test_ctx
prepared = channel_downloader.prepare_download(test_session, download)
executed = await channel_downloader.execute_download(prepared, make_test_ctx(), download=download)
result = channel_downloader.finalize_download(test_session, download, executed)
assert 'continue_dl' not in result.settings, 'continue_dl should not be passed to child video downloads'
assert 'nooverwrites' not in result.settings, 'nooverwrites should not be passed to child video downloads'
@ -1063,7 +1078,12 @@ async def test_live_video_retry_seconds(test_session, test_directory, mock_video
mock_prepare_filename.return_value = (simple_channel.directory / 'live video.mp4', {'id': 'foo'})
download = Download(url=url, downloader='video', attempts=0)
result = await video_downloader.do_download(download)
# Drive the phase-split API directly: when execute_download detects a live video
# it short-circuits with a DownloadResult (the dispatch surfaces it verbatim and
# skips finalize_download).
from modules.archive.conftest import make_test_ctx
prepared = video_downloader.prepare_download(test_session, download)
result = await video_downloader.execute_download(prepared, make_test_ctx(), download=download)
assert result.success is False
assert result.retry_seconds == 3600
@ -1252,3 +1272,70 @@ async def test_channel_download_preserves_skip_list(test_session, test_directory
once_downloads = video_download_manager.get_once_downloads(test_session)
video_downloads = [i for i in once_downloads if 'watch' in i.url]
assert {i.url for i in video_downloads} == {'https://youtube.com/watch?v=video_2_url'}
# ---------------------------------------------------------------------------
# Phase-split unit tests for VideoDownloader.prepare_download.
#
# These exercise the sync prep invariants in isolation — no async_client, no
# extract_info, no subprocess. Heavier execute/finalize behavior is covered by
# the integration tests above (which drive through video_download_manager).
# ---------------------------------------------------------------------------
from modules.videos.downloader import PreparedVideo # noqa: E402
def test_prepare_max_attempts_raises(test_session):
"""prepare_download enforces the 10-attempt cap (videos retry more than other
downloaders because YouTube transients are common)."""
download = Download(url='https://youtube.com/watch?v=foo', downloader='video', attempts=10)
with pytest.raises(UnrecoverableDownloadError, match='Max download attempts'):
video_downloader.prepare_download(test_session, download)
def test_prepare_normalizes_url(test_session):
"""Shorts URLs become watch URLs in the PreparedVideo (matches DB-stored format)."""
download = Download(
url='https://www.youtube.com/shorts/ABC123',
downloader='video',
settings={},
)
prepared = video_downloader.prepare_download(test_session, download)
assert isinstance(prepared, PreparedVideo)
assert prepared.url == 'https://www.youtube.com/watch?v=ABC123'
def test_prepare_copies_upstream_settings_onto_download(test_session):
"""ChannelDownloader passes destination + tag_names via settings; prep copies them
onto the Download row so downstream code sees them as columns."""
download = Download(
url='https://youtube.com/watch?v=foo',
downloader='video',
destination=None,
tag_names=None,
settings={'destination': '/media/wrolpi/videos/foo', 'tag_names': ['news', 'tech']},
)
prepared = video_downloader.prepare_download(test_session, download)
assert download.destination == '/media/wrolpi/videos/foo'
assert download.tag_names == ['news', 'tech']
assert prepared.tag_names == ['news', 'tech']
def test_prepare_uses_existing_video_location(test_session, video_factory):
"""If a Video already exists at the URL, prep captures its location for error
attribution (so a failed re-download still tells the user where the previous file is)."""
url = 'https://youtube.com/watch?v=existing'
existing = video_factory(with_video_file=True)
existing.file_group.url = url
test_session.commit()
download = Download(url=url, downloader='video', settings={})
prepared = video_downloader.prepare_download(test_session, download)
assert prepared.location == existing.location
assert prepared.location is not None

View file

@ -35,7 +35,7 @@ from sqlalchemy.orm import Session, sessionmaker
import wrolpi.root_api # noqa
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import CommandResult
from wrolpi.cmd import CommandResult, run_command
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL, \
get_wrolpi_config
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
@ -43,7 +43,7 @@ from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloader, \
from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \
downloads_manager_config_context, download_cache_config_context
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.models import Directory, FileGroup
@ -378,16 +378,52 @@ def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
return asserter
def make_test_ctx(*, is_cancelled=None, can_download=None,
outside_download_window=None, run_command_=None,
download_timeout=None) -> DownloadContext:
"""Build a DownloadContext suitable for downloader unit tests.
Defaults: never-cancelled, can_download True, no window restriction, real run_command,
no global timeout. Override any callable to simulate cancellation, throttling, or
fake subprocess output.
Lives in wrolpi/conftest.py so any test (including primitive tests of process_runner /
cancel_wrapper) can construct a ctx without crossing module boundaries.
"""
return DownloadContext(
is_cancelled=is_cancelled or (lambda: False),
can_download=can_download or (lambda: True),
outside_download_window=outside_download_window or (lambda: False),
run_command=run_command_ or run_command,
download_timeout=download_timeout,
)
@pytest.fixture
def test_downloader(test_download_manager) -> Downloader:
class TestDownloader(Downloader, ABC):
"""A testing Downloader"""
"""A testing Downloader.
prepare_download returns a non-None sentinel so the manager's phase-split dispatch
routes through this subclass. execute_download is an AsyncMock — tests wire its
side_effect via the set_test_* methods. Returning a DownloadResult from
execute_download tells the dispatch to skip finalize (see signal_download_download).
"""
name = 'test_downloader'
def __repr__(self):
return '<TESTING Downloader>'
do_download = AsyncMock()
execute_download = AsyncMock()
def prepare_download(self, session, download):
return object() # sentinel; activates the phase-split path
def finalize_download(self, session, download, executed):
# execute_download returns a DownloadResult directly, so the dispatch
# short-circuits before reaching finalize. Provided here only to satisfy
# the abstract contract.
return DownloadResult(success=True)
def set_test_success(self):
async def _(*a, **kwargs):
@ -395,7 +431,7 @@ def test_downloader(test_download_manager) -> Downloader:
await asyncio.sleep(1)
return DownloadResult(success=True)
self.do_download.side_effect = _
self.execute_download.side_effect = _
def set_test_failure(self):
async def _(*a, **kwargs):
@ -403,7 +439,7 @@ def test_downloader(test_download_manager) -> Downloader:
await asyncio.sleep(1)
return DownloadResult(success=False)
self.do_download.side_effect = _
self.execute_download.side_effect = _
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
async def _(*a, **kwargs):
@ -411,7 +447,7 @@ def test_downloader(test_download_manager) -> Downloader:
await asyncio.sleep(1)
raise exception
self.do_download.side_effect = _
self.execute_download.side_effect = _
def set_test_unrecoverable_exception(self):
async def _(*a, **kwargs):
@ -419,7 +455,7 @@ def test_downloader(test_download_manager) -> Downloader:
await asyncio.sleep(1)
raise UnrecoverableDownloadError()
self.do_download.side_effect = _
self.execute_download.side_effect = _
test_downloader = TestDownloader()
# Default to successful download.

View file

@ -1,6 +1,5 @@
import asyncio
import contextlib
import inspect
import logging
import multiprocessing
import os
@ -419,23 +418,19 @@ class Downloader:
def __repr__(self):
return f'<Downloader name={self.name}>'
async def do_download(self, download: Download, ctx: 'DownloadContext' = None) -> DownloadResult:
"""Legacy single-phase entry point.
def prepare_download(self, session: Session, download: Download) -> Any:
"""DB-bound prep. Caller (DownloadManager) opens and commits the session.
Subclasses opting in to the phase-split API override prepare_download /
execute_download / finalize_download instead and leave this unimplemented.
Sync by default; subclasses may override with `async def prepare_download` when
they need network or subprocess work during prep (e.g. yt-dlp extract_info).
The dispatch awaits the coroutine. Holding the session across an await means
the connection is in use — keep async prep work brief.
Subclasses must override and return a value (typically a dataclass) that is
passed unchanged to execute_download.
"""
raise NotImplementedError()
def prepare_download(self, session: Session, download: Download) -> Optional[Any]:
"""Synchronous DB-bound prep. Caller (DownloadManager) opens and commits the session.
Returning None signals the subclass has not opted in to the phase-split API and
DownloadManager should fall back to do_download. Returning any other value
(typically a dataclass) is passed unchanged to execute_download.
"""
return None
async def execute_download(self, prepared: Any, ctx: 'DownloadContext',
download: Download = None) -> Any:
"""Async I/O-bound work. No DB session — does network and subprocess work,
@ -448,7 +443,12 @@ class Downloader:
raise NotImplementedError()
def finalize_download(self, session: Session, download: Download, executed: Any) -> DownloadResult:
"""Synchronous DB-bound persistence. Caller opens and commits the session."""
"""DB-bound persistence. Caller opens and commits the session.
Sync by default; subclasses may override with `async def finalize_download` when
post-download persistence requires async work (e.g. ffprobe metadata, dedupe
queries). The dispatch awaits the coroutine.
"""
raise NotImplementedError()
def already_downloaded(self, session: Session, *urls: List[str]):
@ -465,25 +465,22 @@ class Downloader:
Global timeout takes precedence over the timeout argument, unless it is 0. (Smaller global timeout wins)
When ctx is provided, ctx.run_command and ctx.download_timeout are used and cancellation
is polled through ctx. When ctx is None (legacy callers), behavior is unchanged.
ctx is required from the dispatch path; the kwarg default of None is a convenience
for unit tests that drive process_runner directly with a stub ctx.
"""
if cwd and not cwd.is_dir():
raise RuntimeError('cwd directory does not exist')
if ctx is None:
raise RuntimeError('process_runner requires a DownloadContext')
if ctx is not None:
config_timeout = ctx.download_timeout
runner = ctx.run_command
else:
config_timeout = get_wrolpi_config().download_timeout
runner = run_command
config_timeout = ctx.download_timeout
# 0 means no timeout. Prefer config timeout, then argument, then class default.
if config_timeout:
timeout = config_timeout
elif timeout is None:
timeout = self.timeout
coro = runner(cmd, cwd=cwd, timeout=timeout, log_command=debug,
stdout_callback=stdout_callback, env=env)
coro = ctx.run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
stdout_callback=stdout_callback, env=env)
result = await self.cancel_wrapper(coro, download, ctx=ctx)
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
@ -497,23 +494,17 @@ class Downloader:
@staticmethod
async def cancel_wrapper(coro: Coroutine, download: Download, ctx: 'DownloadContext' = None):
"""
Converts an async coroutine to a task. If DownloadManager receives a kill request, this method will cancel
the task.
When ctx is provided, cancellation is polled through ctx.is_cancelled / ctx.can_download /
ctx.outside_download_window. When ctx is None, the global download_manager singleton is
used (legacy path for unmigrated subclasses).
Converts an async coroutine to a task. If DownloadManager receives a kill request,
this method will cancel the task. Cancellation state is read through ctx.
"""
if ctx is None:
raise RuntimeError('cancel_wrapper requires a DownloadContext')
download_id = download.id
task = asyncio.create_task(coro)
try:
while not task.done():
if ctx is not None:
cancelled = ctx.is_cancelled() or not ctx.can_download()
outside_window = ctx.outside_download_window()
else:
cancelled = download_manager.download_is_killed(download_id) or not download_manager.can_download
outside_window = download_manager.outside_download_window
cancelled = ctx.is_cancelled() or not ctx.can_download()
outside_window = ctx.outside_download_window()
if cancelled:
logger.warning(f'Cancel download of {download.url}')
task.cancel()
@ -528,16 +519,13 @@ class Downloader:
error=error,
)
finally:
if ctx is None:
# The killed-downloads list lives on the manager; clear via the
# singleton. Tests that bypass the dispatch may not have a manager
# singleton wired up — silently ignore in that case.
try:
download_manager.unkill_download(download_id)
else:
# Manager still owns the killed-downloads list; clear via singleton.
# The ctx is read-only with respect to cancellation state.
try:
download_manager.unkill_download(download_id)
except Exception:
# In tests without a manager, silently ignore.
pass
except Exception:
pass
else:
# Wait for the download to complete. Cancel if requested.
await asyncio.sleep(0.1)
@ -1539,30 +1527,33 @@ async def signal_download_download(download_id: int, download_url: str):
try_again = True
try:
# Phase-split path: subclass opts in by overriding prepare_download to return non-None.
# Legacy path: do_download remains the single entry point.
# Phase-split dispatch. Caller (us) owns transaction boundaries: a session
# is opened for prepare_download (committing any pre-flight DB work like
# destination resolution) and a fresh session for finalize_download (so we
# don't hold a connection across the long-running execute_download).
# cancel_wrapper may short-circuit with a DownloadResult instead of an
# ExecutedX dataclass; subclasses also use that protocol to skip finalize
# for early-exit cases (e.g. metadata-only video downloads).
ctx = DownloadContext.production(download_manager, download_id)
with get_db_session(commit=True) as s:
prepared = downloader.prepare_download(s, download)
# Subclasses with unavoidable async prep work (e.g. VideoDownloader's
# extract_info / channel resolution) may declare prepare_download as
# `async def`; await the coroutine here.
if asyncio.iscoroutine(prepared):
prepared = await prepared
if prepared is not None:
# New phase-split path: caller (us) owns transaction boundaries.
coro = downloader.execute_download(prepared, ctx, download=download)
executed = await downloader.cancel_wrapper(coro, download, ctx=ctx)
if isinstance(executed, DownloadResult):
# cancel_wrapper short-circuited with a cancel result.
result = executed
else:
with get_db_session(commit=True) as s:
result = downloader.finalize_download(s, download, executed)
coro = downloader.execute_download(prepared, ctx, download=download)
executed = await downloader.cancel_wrapper(coro, download, ctx=ctx)
if isinstance(executed, DownloadResult):
# cancel_wrapper or execute_download short-circuited with a result.
result = executed
else:
# Legacy path. Subclasses on this path have signature do_download(self, download)
# without ctx, so we must NOT pass ctx as a positional arg. cancel_wrapper continues
# to use the global download_manager (ctx=None).
if not inspect.iscoroutinefunction(downloader.do_download):
raise RuntimeError(f'Coroutine expected from {downloader} do_download method.')
coro = downloader.do_download(download)
result = await downloader.cancel_wrapper(coro, download)
with get_db_session(commit=True) as s:
result = downloader.finalize_download(s, download, executed)
# finalize_download may also be async (same rationale as prepare).
if asyncio.iscoroutine(result):
result = await result
except BotBlockedDownloadError as e:
# Bot detection or invalid cookies - stop all pending downloads from this domain.
worker_logger.warning(f'BotBlockedDownloadError for {url}', exc_info=e)
@ -2048,6 +2039,34 @@ def parse_feed(url: str) -> FeedParserDict:
return feedparser.parse(url)
@dataclass
class PreparedRSS:
"""Plan produced by RSSDownloader.prepare_download.
sub_downloader is the resolved Downloader instance used both for already_downloaded
DB queries and to detect the 'video' branch (where URL normalization + duration
filtering applies). Carries the original Download stand-in fields execute and
finalize need so neither phase has to reach back to the DB row.
"""
url: str
sub_downloader_name: str
sub_downloader: 'Downloader'
settings: dict
@dataclass
class ExecutedRSS:
"""Output of RSSDownloader.execute_download.
candidate_urls is the post-filter, post-duration-check URL list ready for DB dedupe
in finalize. yt_channel_id (when present) lets finalize apply Channel info to the
Download row. error covers feed-parse failures.
"""
yt_channel_id: Optional[str]
candidate_urls: List[str]
error: Optional[str] = None
class RSSDownloader(Downloader):
"""Downloads an RSS feed and creates new downloads for every unique link in the feed."""
name = 'rss'
@ -2057,27 +2076,34 @@ class RSSDownloader(Downloader):
def __repr__(self):
return '<RSSDownloader>'
async def do_download(self, download: Download) -> DownloadResult:
def prepare_download(self, session: Session, download: Download) -> PreparedRSS:
sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader)
if not sub_downloader:
raise ValueError(f'Unable to find sub_downloader for {download.url}')
return PreparedRSS(
url=download.url,
sub_downloader_name=download.sub_downloader,
sub_downloader=sub_downloader,
settings=download.settings or dict(),
)
feed: FeedParserDict = parse_feed(download.url)
async def execute_download(self, prepared: PreparedRSS, ctx: 'DownloadContext',
download: Download = None) -> ExecutedRSS:
feed: FeedParserDict = parse_feed(prepared.url)
if feed['bozo'] and not self.acceptable_bozo_errors(feed):
# Feed did not parse
return DownloadResult(success=False, error='Failed to parse RSS feed')
return ExecutedRSS(yt_channel_id=None, candidate_urls=[],
error='Failed to parse RSS feed')
if not isinstance(feed, dict) or not feed.get('entries'):
# RSS parsed but does not have entries.
return DownloadResult(success=False, error='RSS feed did not have any entries')
return ExecutedRSS(yt_channel_id=None, candidate_urls=[],
error='RSS feed did not have any entries')
# Apply YT channel to the Download, if not already applied.
if yt_channel_id := feed.get('feed', dict()).get('yt_channelid'):
if not (download.location or download.collection_id):
self.apply_yt_channel(download.id, yt_channel_id)
yt_channel_id = feed.get('feed', dict()).get('yt_channelid')
# Filter entries using Download.settings.
entries = self.filter_entries(download, feed['entries'])
# Filter entries using settings; preserve filter_entries' Download-keyed signature
# by passing a stand-in (test_rss.py calls filter_entries directly with a Download).
log_stub = Download(url=prepared.url, settings=prepared.settings)
entries = self.filter_entries(log_stub, feed['entries'])
# Filter URL links.
urls = []
@ -2087,24 +2113,43 @@ class RSSDownloader(Downloader):
else:
logger.warning(f'RSS entry {idx} did not have a link!')
if download.sub_downloader == 'video':
if prepared.sub_downloader_name == 'video':
# Normalize video URLs before checking if already downloaded.
# YouTube Shorts URLs (e.g., youtube.com/shorts/ABC123) must be converted to watch URLs
# (e.g., youtube.com/watch?v=ABC123) to match what is stored in the database.
from modules.videos.normalize_video_url import normalize_video_url
urls = [normalize_video_url(i) for i in urls]
# Duration filtering performs network I/O (yt-dlp duration probe) — must run
# in execute, not finalize. This may probe URLs that DB-dedupe would have
# eliminated; fetch_video_duration is cached, so duplicate probes are cheap.
urls = await self.filter_videos(log_stub, urls)
# Only download new URLs.
with get_db_session() as session:
urls = [i for i in urls if i not in [i.url for i in sub_downloader.already_downloaded(session, *urls)]]
# Filter out URLs already in the Download table (pending downloads).
return ExecutedRSS(yt_channel_id=yt_channel_id, candidate_urls=urls)
def finalize_download(self, session: Session, download: Download,
executed: ExecutedRSS) -> DownloadResult:
if executed.error:
return DownloadResult(success=False, error=executed.error)
# Apply YT channel info to the Download, if not already applied.
if executed.yt_channel_id and not (download.location or download.collection_id):
self.apply_yt_channel(session, download, executed.yt_channel_id)
# Resolve sub_downloader via the manager (matches prepare; cheap dict lookup).
sub_downloader = download_manager.find_downloader_by_name(download.sub_downloader)
urls = executed.candidate_urls
# DB dedupe: drop URLs already downloaded under this sub_downloader, plus URLs
# already queued in the Download table.
if sub_downloader and urls:
already = sub_downloader.already_downloaded(session, *urls)
already_urls = {a.url for a in already}
urls = [u for u in urls if u not in already_urls]
existing_download_urls = {d[0] for d in session.query(Download.url).filter(Download.url.in_(urls))}
urls = [url for url in urls if url not in existing_download_urls]
# Remove skipped URLs before duration checks. (Typically done after this by the download worker).
urls = [i for i in urls if not download_manager.is_skipped(i)]
urls = [u for u in urls if u not in existing_download_urls]
if download.sub_downloader == 'video': # VideoDownloader
urls = await self.filter_videos(download, urls)
# Skip-list filter (manager-owned shared_ctx state, not DB).
urls = [u for u in urls if not download_manager.is_skipped(u)]
logger.info(f'Successfully got {len(urls)} new URLs from RSS {download.url}')
@ -2123,12 +2168,7 @@ class RSSDownloader(Downloader):
# Fallback to legacy settings for backwards compatibility
next_download_settings['destination'] = i
result = DownloadResult(
success=True,
downloads=urls,
settings=next_download_settings,
)
return result
return DownloadResult(success=True, downloads=urls, settings=next_download_settings)
@staticmethod
async def filter_videos(download: Download, urls: list[str]) -> list[str]:
@ -2190,17 +2230,17 @@ class RSSDownloader(Downloader):
return entries
@staticmethod
def apply_yt_channel(download_id: int, yt_channel_id: str):
"""Get Channel that matches this Download, apply Channel/Collection information to the Download."""
with get_db_session() as session:
from modules.videos.models import Channel
channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}')
if channel:
download_ = Download.get_by_id(session, download_id)
download_.collection = channel.collection
download_.collection_id = channel.collection_id
download_.location = download_.location or channel.location
session.commit()
def apply_yt_channel(session: Session, download: Download, yt_channel_id: str):
"""Apply Channel/Collection information to the given Download row.
Caller (finalize_download) owns the session and commits.
"""
from modules.videos.models import Channel
channel = Channel.get_by_source_id(session, f'UC{yt_channel_id}')
if channel:
download.collection = channel.collection
download.collection_id = channel.collection_id
download.location = download.location or channel.location
@staticmethod
def acceptable_bozo_errors(feed):

View file

@ -53,7 +53,7 @@ async def test_scrape_html_downloader(test_directory, test_session, test_downloa
assert_download_urls(['https://example.com/dir', ])
with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \
mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download):
mock.patch('wrolpi.files.downloader.FileDownloader.execute_download', fake_file_do_download):
await test_download_manager.wait_for_all_downloads()
await await_switches()
@ -77,7 +77,7 @@ async def test_scrape_html_downloader_html(test_directory, test_session, test_do
assert_download_urls(['https://example.com/dir', ])
with mock.patch('wrolpi.scrape_downloader.ScrapeHTMLDownloader.fetch_html', fake_fetch_html), \
mock.patch('wrolpi.files.downloader.FileDownloader.do_download', fake_file_do_download):
mock.patch('wrolpi.files.downloader.FileDownloader.execute_download', fake_file_do_download):
await test_download_manager.wait_for_all_downloads()
await await_switches()

View file

@ -115,6 +115,7 @@ async def test_make_progress_callback(async_client):
@pytest.mark.asyncio
async def test_cancel_wrapper_outer_cancellation(test_session, test_download_manager, test_downloader):
"""cancel_wrapper propagates outer CancelledError and cancels the inner task."""
from wrolpi.conftest import make_test_ctx
download = test_download_manager.create_download(test_session, 'https://example.com/cancel', test_downloader.name)
test_session.commit()
@ -127,7 +128,7 @@ async def test_cancel_wrapper_outer_cancellation(test_session, test_download_man
inner_cancelled.set()
raise
wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download))
wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download, ctx=make_test_ctx()))
# Give the wrapper time to start polling.
await asyncio.sleep(0.3)
@ -461,7 +462,7 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
# Download is processed and successful, no longer "new".
await test_download_manager.wait_for_all_downloads()
test_downloader.do_download.assert_called_once()
test_downloader.execute_download.assert_called_once()
assert list(test_download_manager.get_new_downloads(test_session)) == []
downloads = list(test_download_manager.get_recurring_downloads(test_session))
assert len(downloads) == 1
@ -487,10 +488,10 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
assert download.last_successful_download == now_
# Try the download, but it fails.
test_downloader.do_download.reset_mock()
test_downloader.execute_download.reset_mock()
test_downloader.set_test_failure()
await test_download_manager.wait_for_all_downloads()
test_downloader.do_download.assert_called_once()
test_downloader.execute_download.assert_called_once()
download = test_session.query(Download).one()
# Download is deferred, last successful download remains the same.
assert download.is_deferred, download.status_code
@ -500,12 +501,12 @@ async def test_recurring_downloads(test_session, test_download_manager, fake_now
assert download.next_download == expected
# Try the download again, it finally succeeds.
test_downloader.do_download.reset_mock()
test_downloader.execute_download.reset_mock()
now_ = fake_now(datetime(2020, 1, 1, 4, 0, 1, tzinfo=pytz.UTC))
test_downloader.set_test_success()
test_download_manager.renew_recurring_downloads()
await test_download_manager.wait_for_all_downloads()
test_downloader.do_download.assert_called_once()
test_downloader.execute_download.assert_called_once()
download = test_session.query(Download).one()
assert download.is_complete, download.status_code
assert download.last_successful_download == now_
@ -664,31 +665,36 @@ async def test_skip_already_downloaded_disabled(test_session, test_download_mana
@pytest.mark.asyncio
async def test_process_runner_timeout(async_client, test_session, test_directory):
"""A Downloader can cancel its download using a timeout."""
from wrolpi.conftest import make_test_ctx
# Default timeout of 3 seconds.
downloader = Downloader('downloader', timeout=3)
# Sleep for 8 seconds really takes 8 seconds.
start = datetime.now()
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=10)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory,
timeout=10, ctx=make_test_ctx())
elapsed = datetime.now() - start
assert elapsed.total_seconds() >= 8
# Download.timeout is obeyed.
# Downloader.timeout (class default) is obeyed.
start = datetime.now()
await downloader.process_runner(Download(), ('sleep', '8'), test_directory)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, ctx=make_test_ctx())
elapsed = datetime.now() - start
assert 3 < elapsed.total_seconds() < 5
# One-off timeout is obeyed.
start = datetime.now()
await downloader.process_runner(Download(), ('sleep', '8'), test_directory, timeout=1)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory,
timeout=1, ctx=make_test_ctx())
elapsed = datetime.now() - start
assert 1 < elapsed.total_seconds() < 3
# Global timeout is obeyed.
get_wrolpi_config().download_timeout = 3
# ctx.download_timeout takes precedence over class/per-call timeouts.
# Production builds ctx.download_timeout from get_wrolpi_config().download_timeout
# (see DownloadContext.production); we bake the value directly here.
start = datetime.now()
await downloader.process_runner(Download(), ('sleep', '8'), test_directory)
await downloader.process_runner(Download(), ('sleep', '8'), test_directory,
ctx=make_test_ctx(download_timeout=3))
elapsed = datetime.now() - start
assert 2 < elapsed.total_seconds() < 5
@ -707,6 +713,7 @@ exit 123
@pytest.mark.asyncio
async def test_process_runner_output(async_client, test_directory, test_downloader):
from wrolpi.conftest import make_test_ctx
script = test_directory / 'echo_script.sh'
script.write_text(GOOD_SCRIPT)
cmd = ('/bin/bash', script)
@ -714,7 +721,8 @@ async def test_process_runner_output(async_client, test_directory, test_download
Download(),
cmd,
test_directory,
timeout=1
timeout=1,
ctx=make_test_ctx(),
)
assert result.return_code == 0
assert result.stdout == b'standard output\n'
@ -726,7 +734,8 @@ async def test_process_runner_output(async_client, test_directory, test_download
Download(),
cmd,
test_directory,
timeout=1
timeout=1,
ctx=make_test_ctx(),
)
assert result.return_code == 123
assert result.stdout == b'bad standard output\n'
@ -1123,16 +1132,17 @@ async def test_child_downloads_have_parent_download_url(test_session, test_downl
# Track which URLs have been processed to avoid returning children for child downloads
processed_urls = set()
# Configure the test_downloader to return child downloads only for the parent URL
async def download_with_children(download, *a, **kwargs):
# Configure the test_downloader to return child downloads only for the parent URL.
# New signature matches Downloader.execute_download(prepared, ctx, download=...).
async def download_with_children(prepared, ctx, *, download=None, **kwargs):
await asyncio.sleep(0.1)
if download.url == parent_url and parent_url not in processed_urls:
if download is not None and download.url == parent_url and parent_url not in processed_urls:
processed_urls.add(parent_url)
return DownloadResult(success=True, downloads=child_urls)
# Return success without children for child downloads
return DownloadResult(success=True)
test_downloader.do_download.side_effect = download_with_children
test_downloader.execute_download.side_effect = download_with_children
# Create the parent download with a sub_downloader specified
test_download_manager.create_download(test_session, parent_url, test_downloader.name,

View file

@ -9,9 +9,17 @@ from wrolpi.downloader import Download, DownloadResult, Downloader, DownloadMana
class RSSHTTPDownloader(Downloader):
"""Test fixture: a no-op sub-downloader RSS hands child URLs to."""
name = 'rss_http'
async def do_download(self, download: Download) -> DownloadResult:
def prepare_download(self, session: Session, download: Download):
# Return a non-None sentinel so the dispatch routes to the phase-split path.
return object()
async def execute_download(self, prepared, ctx, download: Download = None):
return None
def finalize_download(self, session: Session, download: Download, executed) -> DownloadResult:
return DownloadResult(success=True)
def already_downloaded(self, session: Session, *urls: List[str]):
@ -295,3 +303,180 @@ async def test_rss_downloader_normalizes_skipped_youtube_shorts_urls(test_sessio
# The new video should be added for download with the normalized URL.
assert 'https://www.youtube.com/watch?v=NEW456' in download_urls, \
'New video should be added to downloads with normalized URL'
# ---------------------------------------------------------------------------
# Phase-split unit tests.
#
# These exercise prepare_download / execute_download / finalize_download in
# isolation. They drive the new code paths directly rather than through the
# DownloadManager + wait_for_all_downloads polling loop the integration tests
# above use.
# ---------------------------------------------------------------------------
from wrolpi.downloader import ExecutedRSS, PreparedRSS # noqa: E402
from modules.archive.conftest import make_test_ctx # noqa: E402
def test_prepare_resolves_sub_downloader(test_session, test_download_manager):
"""prepare_download finds the sub_downloader instance and surfaces settings."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
download = Download(
url='https://example.com/feed',
downloader='rss',
sub_downloader='rss_http',
settings={'title_include': 'foo'},
)
prepared = rss_downloader.prepare_download(test_session, download)
assert isinstance(prepared, PreparedRSS)
assert prepared.url == 'https://example.com/feed'
assert prepared.sub_downloader_name == 'rss_http'
assert prepared.sub_downloader is http_downloader
assert prepared.settings == {'title_include': 'foo'}
def test_prepare_unknown_sub_downloader_raises(test_session, test_download_manager):
"""No matching sub_downloader → manager raises InvalidDownload (manager will record
the failure as a deferred download). The unreachable ValueError branch in the legacy
code was dead because find_downloader_by_name raises rather than returning None."""
from wrolpi.errors import InvalidDownload
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
download = Download(
url='https://example.com/feed',
downloader='rss',
sub_downloader='no_such_downloader',
)
with pytest.raises(InvalidDownload, match='Cannot find downloader'):
rss_downloader.prepare_download(test_session, download)
@pytest.mark.asyncio
async def test_execute_failed_parse_returns_error(test_session, test_download_manager):
"""A bozo feed without acceptable errors short-circuits with executed.error."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
prepared = PreparedRSS(
url='https://example.com/feed',
sub_downloader_name='rss_http',
sub_downloader=http_downloader,
settings={},
)
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(bozo=1, bozo_exception=Exception('nope'), entries=[])
executed = await rss_downloader.execute_download(prepared, make_test_ctx())
assert executed.candidate_urls == []
assert executed.error == 'Failed to parse RSS feed'
@pytest.mark.asyncio
async def test_execute_collects_links_and_yt_channel(test_session, test_download_manager):
"""A well-formed feed yields candidate_urls plus the yt_channelid for finalize."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
prepared = PreparedRSS(
url='https://example.com/feed',
sub_downloader_name='rss_http',
sub_downloader=http_downloader,
settings={},
)
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(
bozo=0,
feed=dict(yt_channelid='abc123'),
entries=[
dict(link='https://example.com/a'),
dict(link='https://example.com/b'),
],
)
executed = await rss_downloader.execute_download(prepared, make_test_ctx())
assert executed.error is None
assert executed.yt_channel_id == 'abc123'
assert executed.candidate_urls == ['https://example.com/a', 'https://example.com/b']
def test_finalize_propagates_executed_error(test_session, test_download_manager):
"""An executed-phase error becomes a failure DownloadResult."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
download = Download(url='https://example.com/feed', downloader='rss', sub_downloader='rss_http')
executed = ExecutedRSS(yt_channel_id=None, candidate_urls=[], error='Failed to parse RSS feed')
result = rss_downloader.finalize_download(test_session, download, executed)
assert result.success is False
assert result.error == 'Failed to parse RSS feed'
def test_finalize_dedupes_existing_downloads(test_session, test_download_manager):
"""URLs already present in the Download table are filtered out before being returned
as child downloads."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
# Pre-existing pending download for /a.
test_session.add(Download(url='https://example.com/a', downloader='rss_http'))
test_session.commit()
download = Download(
url='https://example.com/feed',
downloader='rss',
sub_downloader='rss_http',
)
executed = ExecutedRSS(
yt_channel_id=None,
candidate_urls=['https://example.com/a', 'https://example.com/b', 'https://example.com/c'],
)
result = rss_downloader.finalize_download(test_session, download, executed)
assert result.success is True
assert sorted(result.downloads) == ['https://example.com/b', 'https://example.com/c']
def test_finalize_passes_destination_from_column(test_session, test_download_manager, test_directory):
"""Child downloads inherit destination from the RSS row's destination column."""
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
http_downloader = RSSHTTPDownloader()
test_download_manager.register_downloader(http_downloader)
dest_path = test_directory / 'archive/custom/path'
download = Download(
url='https://example.com/feed',
downloader='rss',
sub_downloader='rss_http',
destination=str(dest_path),
settings={},
)
executed = ExecutedRSS(
yt_channel_id=None,
candidate_urls=['https://example.com/x'],
)
result = rss_downloader.finalize_download(test_session, download, executed)
assert result.success is True
assert result.settings['destination'].endswith('archive/custom/path')