From 4579562be5d7869f952ff75ba979e298695893fb Mon Sep 17 00:00:00 2001
From: Roland
Date: Thu, 12 Mar 2026 17:48:35 -0600
Subject: [PATCH] Displaying download progress (speed, eta) for file and video
downloads.
---
app/src/App.css | 11 +
app/src/components/Theme.test.js | 81 +++++++
app/src/components/Theme.tsx | 15 +-
app/src/components/admin/Downloads.js | 75 ++++++-
modules/videos/downloader.py | 68 ++++--
modules/videos/models.py | 18 +-
.../test_channel_refresh_external_files.py | 198 ++++++++++++++++++
modules/videos/test/test_downloader.py | 50 ++++-
wrolpi/cmd.py | 142 ++++++++-----
wrolpi/collections/lib.py | 41 +++-
wrolpi/collections/test/test_reorganize.py | 2 +-
wrolpi/contexts.py | 1 +
wrolpi/downloader.py | 136 ++++++++++--
wrolpi/files/test/test_downloader.py | 12 +-
wrolpi/test/test_cmd.py | 41 ++++
wrolpi/test/test_downloader.py | 122 ++++++++++-
16 files changed, 912 insertions(+), 101 deletions(-)
create mode 100644 app/src/components/Theme.test.js
create mode 100644 modules/videos/test/test_channel_refresh_external_files.py
diff --git a/app/src/App.css b/app/src/App.css
index 69928256..2529efbf 100644
--- a/app/src/App.css
+++ b/app/src/App.css
@@ -111,6 +111,17 @@ input:checked + .slider:before {
transform: translateX(26px);
}
+.inverted-progress-text.ui.indicating.progress .bar > .progress,
+.inverted-progress-text.ui.progress > .label {
+ color: white !important;
+}
+
+/* Light bar colors (yellow, olive) need dark text on the bar — must override inverted-progress-text. */
+.light-bar-progress-text.ui.indicating.progress .bar > .progress,
+.light-bar-progress-text.ui.progress .bar > .progress {
+ color: rgba(0, 0, 0, 0.85) !important;
+}
+
div.full-height {
overflow: hidden;
padding-top: 100%;
diff --git a/app/src/components/Theme.test.js b/app/src/components/Theme.test.js
new file mode 100644
index 00000000..1c76ea88
--- /dev/null
+++ b/app/src/components/Theme.test.js
@@ -0,0 +1,81 @@
+import React from 'react';
+import {Progress} from './Theme';
+import {renderInDarkMode, renderInLightMode} from '../test-utils';
+
+// All semantic-ui colors.
+const allColors = ['red', 'orange', 'yellow', 'olive', 'green', 'teal', 'blue', 'violet', 'purple', 'pink', 'brown', 'grey'];
+
+// Light bar colors that always need dark percent text (light bars in any theme).
+const lightBarColors = ['yellow', 'olive', 'teal'];
+// Bar colors that need dark percent text only in dark mode (inverted makes bar lighter).
+const darkModeLightBarColors = ['grey', 'blue'];
+
+describe('Progress component theme support', () => {
+ describe('dark mode', () => {
+ it('adds inverted-progress-text class for label readability', () => {
+ const {container} = renderInDarkMode(
+
+ );
+
+ const progressEl = container.querySelector('.ui.progress');
+ expect(progressEl).toHaveClass('inverted-progress-text');
+
+ const progressLabel = container.querySelector('.bar > .progress');
+ expect(progressLabel).toHaveTextContent('39%');
+
+ const label = container.querySelector('.ui.progress > .label');
+ expect(label).toHaveTextContent('2.1 GB / 5.3 GB');
+ });
+
+ it('does NOT add inverted-progress-text in light mode', () => {
+ const {container} = renderInLightMode(
+
+ );
+
+ const progressEl = container.querySelector('.ui.progress');
+ expect(progressEl).not.toHaveClass('inverted-progress-text');
+ });
+ });
+
+ describe('percent text contrast per color', () => {
+ allColors.forEach(color => {
+ it(`${color} bar has readable percent text in light mode`, () => {
+ const {container} = renderInLightMode(
+
)}
@@ -456,8 +510,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
{parentDownloadUrl && (
From:
- {parentDownloadUrl}
-
+ {parentDownloadUrl}
+
)}
@@ -477,7 +531,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
{completedAtCell}
- {status === 'pending' ? : null}
+ {status === 'pending' && !download.progress ? : null}
+ {download.progress && }
{buttonCell}
{editModal}
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 48e91aa2..a69a4c32 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -5,6 +5,7 @@ import logging
import os.path
import pathlib
import re
+import time
import traceback
from abc import ABC
from datetime import timedelta
@@ -23,7 +24,8 @@ from wrolpi.common import logger, get_media_directory, escape_file_name, resolve
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
from wrolpi.dates import now
from wrolpi.db import get_db_session
-from wrolpi.downloader import Downloader, Download, DownloadResult
+from wrolpi.downloader import Downloader, Download, DownloadResult, \
+ clear_download_progress, _parse_size, make_progress_callback
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
@@ -43,6 +45,36 @@ from .video.lib import download_video_info_json
logger = logger.getChild(__name__)
+YTDLP_PROGRESS_RE = re.compile(
+ r'\[download\]\s+'
+ r'([\d.]+)%\s+' # percent
+ r'of\s+~?\s*([\d.]+\w+)\s+' # total size (with optional ~)
+ r'at\s+([\d.]+\w+/s)\s+' # speed
+ r'ETA\s+(\S+)' # ETA
+)
+
+
+def parse_ytdlp_progress(line: str) -> dict | None:
+ """Parse a yt-dlp progress line (with --newline) and return a dict, or None if not a progress line."""
+ m = YTDLP_PROGRESS_RE.search(line)
+ if not m:
+ return None
+ percent = float(m.group(1))
+ total_bytes = _parse_size(m.group(2))
+ speed_str = m.group(3).rstrip('/s')
+ speed = _parse_size(speed_str)
+ eta = m.group(4)
+ if eta == 'Unknown':
+ eta = None
+ return dict(
+ bytes_downloaded=int(percent / 100.0 * total_bytes),
+ total_bytes=total_bytes,
+ percent=int(percent),
+ speed=speed,
+ eta=eta,
+ )
+
+
VIDEO_RESOLUTION_MAP = {
'360p': [
'134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
@@ -585,6 +617,7 @@ class VideoDownloader(Downloader, ABC):
'--convert-thumbnails', DEFAULT_POSTER_FORMAT,
'--merge-output-format', video_format,
'--no-cache-dir',
+ '--newline',
'--compat-options', 'no-live-chat',
# Get top 20 comments, 10 replies per parent.
'--write-comments',
@@ -605,20 +638,27 @@ class VideoDownloader(Downloader, ABC):
if effective['sleep_requests'] > 0:
cmd = (*cmd, '--sleep-requests', str(effective['sleep_requests']))
- if cookies_unlocked():
- # Use encrypted cookies file
- logger.info(f'Using encrypted cookies for video download: {url}')
- with cookies_for_download() as cookies_path:
- cmd = (*cmd, '--cookies', str(cookies_path))
- if effective['user_agent']:
- cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
+ on_stdout = make_progress_callback(download.id, parse_ytdlp_progress)
+
+ try:
+ if cookies_unlocked():
+ # Use encrypted cookies file
+ logger.info(f'Using encrypted cookies for video download: {url}')
+ with cookies_for_download() as cookies_path:
+ cmd = (*cmd, '--cookies', str(cookies_path))
+ if effective['user_agent']:
+ cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
+ cmd = (*cmd, '-o', video_path, url)
+ result = await self.process_runner(download, cmd, out_dir, debug=True,
+ stdout_callback=on_stdout)
+ else:
+ # No cookies available
+ logger.info(f'No cookies available for video download: {url}')
cmd = (*cmd, '-o', video_path, url)
- result = await self.process_runner(download, cmd, out_dir, debug=True)
- else:
- # No cookies available
- logger.info(f'No cookies available for video download: {url}')
- cmd = (*cmd, '-o', video_path, url)
- result = await self.process_runner(download, cmd, out_dir, debug=True)
+ result = await self.process_runner(download, cmd, out_dir, debug=True,
+ stdout_callback=on_stdout)
+ finally:
+ clear_download_progress(download.id)
stdout = result.stdout.decode()
stderr = result.stderr.decode()
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 87633d05..e25d437e 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -955,6 +955,17 @@ class Channel(ModelHelper, Base):
self.tag_id = self.tag.id if self.tag else None
return self.tag
+ @staticmethod
+ def _get_external_file_paths(session, channel_id: int, directory: pathlib.Path) -> list[pathlib.Path]:
+ """Return primary_path for all Videos in this channel whose FileGroup is outside the channel directory."""
+ dir_str = f'{directory}/'
+ file_groups = session.query(FileGroup).join(Video, Video.file_group_id == FileGroup.id).filter(
+ Video.channel_id == channel_id,
+ FileGroup.directory != str(directory),
+ ~FileGroup.directory.startswith(dir_str),
+ ).all()
+ return [fg.primary_path for fg in file_groups]
+
@classmethod
def refresh_files(cls, id_: int, send_events: bool = True) -> str:
"""Refresh all files within this Channel's directory. Returns job_id for awaiting.
@@ -962,10 +973,13 @@ class Channel(ModelHelper, Base):
The channel is marked as refreshed after the background task completes post-processing."""
# Get this Channel's info for later. Refresh may take a long time.
with get_db_session() as session:
- directory = cls.find_by_id(session, id_).directory
+ channel = cls.find_by_id(session, id_)
+ directory = channel.directory
+ # Find files associated with this channel but outside its directory.
+ external_paths = cls._get_external_file_paths(session, id_, directory)
# Queue refresh immediately so caller can await the job.
- job_id = file_worker.queue_refresh([directory])
+ job_id = file_worker.queue_refresh([directory] + external_paths)
# Background task handles post-processing after job completes.
async def _():
diff --git a/modules/videos/test/test_channel_refresh_external_files.py b/modules/videos/test/test_channel_refresh_external_files.py
new file mode 100644
index 00000000..0eca3c16
--- /dev/null
+++ b/modules/videos/test/test_channel_refresh_external_files.py
@@ -0,0 +1,198 @@
+"""Tests that Channel.refresh_files() and refresh_collection() detect changes to files
+that are associated with the channel but located outside the channel's directory."""
+import shutil
+
+import pytest
+
+from modules.videos.models import Channel, Video
+from wrolpi.collections import Collection
+from wrolpi.collections.lib import refresh_collection
+from wrolpi.collections.models import CollectionItem
+from wrolpi.common import await_background_tasks
+from wrolpi.conftest import await_file_worker
+from wrolpi.files.models import FileGroup
+
+
+@pytest.mark.asyncio
+class TestChannelRefreshExternalFiles:
+
+ async def _setup_channel_with_external_video(
+ self, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
+ """Create a channel with an internal and external video, then delete the external video from disk.
+
+ Returns (channel, internal_video_id, external_video_id, external_path)."""
+ channel = channel_factory(name='MyChannel')
+ assert channel.directory.is_dir()
+
+ # Create a video inside the channel directory.
+ internal_path = channel.directory / 'internal_video.mp4'
+ video_file_factory(internal_path)
+
+ # Create a video outside the channel directory.
+ external_dir = test_directory / 'videos' / 'other'
+ external_dir.mkdir(parents=True, exist_ok=True)
+ external_path = external_dir / 'external_video.mp4'
+ video_file_factory(external_path)
+
+ # Refresh to index all files and model videos.
+ await refresh_files()
+
+ # Verify both videos exist.
+ internal_video = test_session.query(Video).join(Video.file_group).filter(
+ FileGroup.primary_path == internal_path,
+ ).one()
+ external_video = test_session.query(Video).join(Video.file_group).filter(
+ FileGroup.primary_path == external_path,
+ ).one()
+
+ # Assign the external video to the channel.
+ external_video.channel_id = channel.id
+ test_session.commit()
+
+ internal_video_id = internal_video.id
+ external_video_id = external_video.id
+
+ # Verify channel has 2 videos.
+ channel_videos = test_session.query(Video).filter_by(channel_id=channel.id).all()
+ assert len(channel_videos) == 2, f'Expected 2 channel videos, got {len(channel_videos)}'
+
+ # Delete ALL files for the external video by removing the directory.
+ shutil.rmtree(external_dir)
+ assert not external_path.exists()
+
+ return channel, internal_video_id, external_video_id, external_path
+
+ def _assert_external_deleted_internal_kept(self, test_session, internal_video_id, external_video_id, external_path):
+ """Assert the external video was cleaned up and the internal video remains."""
+ test_session.expire_all()
+
+ external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one_or_none()
+ assert external_fg is None, \
+ 'External video FileGroup should have been removed after files were deleted from disk'
+
+ external_vid = test_session.query(Video).filter_by(id=external_video_id).one_or_none()
+ assert external_vid is None, \
+ 'External Video record should have been removed after its files were deleted'
+
+ internal_vid = test_session.query(Video).filter_by(id=internal_video_id).one_or_none()
+ assert internal_vid is not None, 'Internal video should still exist'
+
+ async def test_channel_refresh_files_detects_deleted_external_video(
+ self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
+ """Channel.refresh_files() should detect and remove an external video whose files were deleted."""
+ channel, internal_id, external_id, external_path = await self._setup_channel_with_external_video(
+ test_session, test_directory, video_file_factory, channel_factory, refresh_files)
+
+ Channel.refresh_files(channel.id)
+ await await_file_worker()
+ await await_background_tasks()
+
+ self._assert_external_deleted_internal_kept(test_session, internal_id, external_id, external_path)
+
+ async def test_refresh_collection_detects_deleted_external_video(
+ self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
+ """refresh_collection() should detect and remove an external video whose files were deleted."""
+ channel, internal_id, external_id, external_path = await self._setup_channel_with_external_video(
+ test_session, test_directory, video_file_factory, channel_factory, refresh_files)
+
+ collection = test_session.query(Collection).filter_by(id=channel.collection_id).one()
+ refresh_collection(collection.id)
+ await await_file_worker()
+
+ self._assert_external_deleted_internal_kept(test_session, internal_id, external_id, external_path)
+
+ async def test_refresh_collection_detects_deleted_external_collection_item(
+ self, async_client, test_session, test_directory, video_file_factory, refresh_files):
+ """refresh_collection() should detect and remove an external CollectionItem-linked FileGroup
+ whose files were deleted from disk."""
+ # Create a generic collection with a directory.
+ collection_dir = test_directory / 'my_collection'
+ collection_dir.mkdir(parents=True, exist_ok=True)
+ collection = Collection(name='MyCollection', directory=collection_dir, kind='simple')
+ test_session.add(collection)
+ test_session.flush()
+
+ # Create a video inside the collection directory.
+ internal_path = collection_dir / 'internal_video.mp4'
+ video_file_factory(internal_path)
+
+ # Create a video outside the collection directory.
+ external_dir = test_directory / 'elsewhere'
+ external_dir.mkdir(parents=True, exist_ok=True)
+ external_path = external_dir / 'external_video.mp4'
+ video_file_factory(external_path)
+
+ # Refresh to index all files.
+ await refresh_files()
+
+ internal_fg = test_session.query(FileGroup).filter_by(primary_path=internal_path).one()
+ external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one()
+ external_fg_id = external_fg.id
+
+ # Link the external FileGroup to the collection via CollectionItem.
+ item = CollectionItem(collection_id=collection.id, file_group_id=external_fg.id)
+ test_session.add(item)
+ test_session.commit()
+
+ # Delete the external files.
+ shutil.rmtree(external_dir)
+ assert not external_path.exists()
+
+ # Refresh via collection.
+ refresh_collection(collection.id)
+ await await_file_worker()
+
+ test_session.expire_all()
+
+ external_fg = test_session.query(FileGroup).filter_by(id=external_fg_id).one_or_none()
+ assert external_fg is None, \
+ 'External CollectionItem FileGroup should have been removed after files were deleted from disk'
+
+ internal_fg = test_session.query(FileGroup).filter_by(primary_path=internal_path).one_or_none()
+ assert internal_fg is not None, 'Internal FileGroup should still exist'
+
+ async def test_channel_refresh_detects_new_file_in_external_file_group(
+ self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
+ """Channel.refresh_files() should detect a new sibling file added to an external video's FileGroup."""
+ channel = channel_factory(name='MyChannel')
+
+ # Create a video outside the channel directory.
+ external_dir = test_directory / 'videos' / 'other'
+ external_dir.mkdir(parents=True, exist_ok=True)
+ external_path = external_dir / 'external_video.mp4'
+ video_file_factory(external_path)
+
+ # Refresh to index the video.
+ await refresh_files()
+
+ external_video = test_session.query(Video).join(Video.file_group).filter(
+ FileGroup.primary_path == external_path,
+ ).one()
+
+ # Assign the external video to the channel.
+ external_video.channel_id = channel.id
+ test_session.commit()
+
+ # Record the initial file count.
+ external_fg = external_video.file_group
+ initial_file_names = {f['path'] for f in external_fg.files}
+ assert 'external_video.mp4' in initial_file_names
+
+ # Add a subtitle file next to the video with the same stem.
+ subtitle_path = external_dir / 'external_video.en.vtt'
+ subtitle_path.write_text('WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nHello\n')
+
+ # Refresh the channel's files.
+ Channel.refresh_files(channel.id)
+ await await_file_worker()
+ await await_background_tasks()
+
+ test_session.expire_all()
+
+ # The FileGroup should now include the subtitle file.
+ external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one()
+ file_names = {f['path'] for f in external_fg.files}
+ assert 'external_video.en.vtt' in file_names, \
+ f'Subtitle file should have been detected in external FileGroup, got: {file_names}'
+ assert 'external_video.mp4' in file_names, 'Original video file should still be present'
+ assert external_fg.primary_path == external_path, 'Primary path should still be the .mp4'
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index e1c6c989..62cee2de 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -9,7 +9,8 @@ import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
- prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download
+ prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
+ parse_ytdlp_progress
from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
@@ -915,3 +916,50 @@ async def test_video_download_playlist_appended_extension(test_session, test_dir
video: Video = test_session.query(Video).one()
assert video.video_path == video_path_na
+
+
+def test_parse_ytdlp_progress():
+ """yt-dlp progress lines are parsed correctly."""
+ line = '[download] 50.0% of ~ 45.67MiB at 2.34MiB/s ETA 00:19'
+ result = parse_ytdlp_progress(line)
+ assert result is not None
+ total = int(45.67 * 1024 ** 2)
+ assert result['total_bytes'] == total
+ assert result['percent'] == 50
+ assert result['bytes_downloaded'] == int(0.5 * total)
+ assert result['speed'] == int(2.34 * 1024 ** 2)
+ assert result['eta'] == '00:19'
+
+
+def test_parse_ytdlp_progress_no_estimated():
+ """yt-dlp progress lines without ~ estimated marker are parsed."""
+ line = '[download] 100% of 45.67MiB at 3.45MiB/s ETA 00:00'
+ result = parse_ytdlp_progress(line)
+ assert result is not None
+ assert result['percent'] == 100
+ assert result['total_bytes'] == int(45.67 * 1024 ** 2)
+ assert result['speed'] == int(3.45 * 1024 ** 2)
+ assert result['eta'] == '00:00'
+
+
+def test_parse_ytdlp_progress_small_percent():
+ """yt-dlp progress lines with small percentages are parsed."""
+ line = '[download] 0.1% of ~ 45.67MiB at 1.23MiB/s ETA 00:37'
+ result = parse_ytdlp_progress(line)
+ assert result is not None
+ assert result['percent'] == 0
+ assert result['eta'] == '00:37'
+
+
+def test_parse_ytdlp_progress_completion_line():
+ """The final yt-dlp line with 'in' instead of 'at/ETA' returns None (no speed/ETA)."""
+ line = '[download] 100% of 45.67MiB in 00:13'
+ result = parse_ytdlp_progress(line)
+ assert result is None
+
+
+def test_parse_ytdlp_progress_not_progress_line():
+ """Non-progress lines return None."""
+ assert parse_ytdlp_progress('[info] Downloading video 1 of 1') is None
+ assert parse_ytdlp_progress('') is None
+ assert parse_ytdlp_progress('[download] Destination: video.mp4') is None
diff --git a/wrolpi/cmd.py b/wrolpi/cmd.py
index 5f156161..f7641942 100644
--- a/wrolpi/cmd.py
+++ b/wrolpi/cmd.py
@@ -155,10 +155,15 @@ TESTING_RUN_COMMAND_RESULT = None
async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
- timeout: int = 600, log_command: bool = True) -> CommandResult:
+ timeout: int = 600, log_command: bool = True,
+ stdout_callback: callable = None) -> CommandResult:
"""Run a shell command, return the results (stdout/stderr/return code).
+ When stdout_callback is provided, stdout is piped and each line is passed to the callback
+ in real-time. Otherwise, stdout is written to a temporary file.
+
:param log_command: Enable debug logging.
+ :param stdout_callback: Called with each decoded stdout line (str). May be None.
"""
if not isinstance(cmd, (list, tuple)):
raise RuntimeError('Command must be a list or tuple')
@@ -167,61 +172,102 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
logger.debug('run_command: returning mock result')
return TESTING_RUN_COMMAND_RESULT() # call the mock
- with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh:
- stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name)
+ streaming = stdout_callback is not None
+
+ with tempfile.NamedTemporaryFile() as stderr_fh:
+ stderr_file = pathlib.Path(stderr_fh.name)
cmd = tuple(str(i) for i in cmd)
cmd_str = ' '.join(shlex.quote(i) for i in cmd)
if log_command:
logger.info(f'run_command: running ({timeout=}): {cmd_str}')
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'run_command: running ({timeout=}): {cmd_str}')
- proc = await asyncio.create_subprocess_exec(
- *cmd,
- stdout=stdout_fh, # Use stdout/stderr files to avoid buffer filling up.
- stderr=stderr_fh,
- cwd=cwd,
- )
- pid = proc.pid
- cancelled = False
- start = time()
+ stdout_fh = None
+ stdout_file = None
+ reader_task = None
+ stdout_buf = None
+
+ if streaming:
+ stdout_arg = asyncio.subprocess.PIPE
+ stdout_buf = bytearray()
+ else:
+ stdout_fh = tempfile.NamedTemporaryFile()
+ stdout_file = pathlib.Path(stdout_fh.name)
+ stdout_arg = stdout_fh
+
try:
- while True:
- elapsed = int(time() - start)
- if timeout and elapsed >= timeout:
- logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
- proc.kill()
- await proc.wait()
- break
+ proc = await asyncio.create_subprocess_exec(
+ *cmd,
+ stdout=stdout_arg,
+ stderr=stderr_fh,
+ cwd=cwd,
+ )
+ pid = proc.pid
+ if streaming:
+ async def _read_stdout():
+ while True:
+ line = await proc.stdout.readline()
+ if not line:
+ break
+ stdout_buf.extend(line)
+ try:
+ stdout_callback(line.decode(errors='replace').rstrip('\n'))
+ except Exception as e:
+ logger.warning(f'run_command: stdout_callback error: {e}')
+
+ reader_task = asyncio.create_task(_read_stdout())
+
+ cancelled = False
+ start = time()
+ try:
+ while True:
+ elapsed = int(time() - start)
+ if timeout and elapsed >= timeout:
+ logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
+ proc.kill()
+ await proc.wait()
+ break
+
+ try:
+ await asyncio.wait_for(proc.wait(), timeout=1)
+ break
+ except asyncio.TimeoutError:
+ continue
+ except CancelledError as e:
+ logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
+ cancelled = True
+ proc.kill()
+ await proc.wait()
+
+ if reader_task:
+ reader_task.cancel()
try:
- # Wait for the process to finish.
- await asyncio.wait_for(proc.wait(), timeout=1)
- break
- except asyncio.TimeoutError:
- # Task is not done, keep waiting...
- continue
- except CancelledError as e:
- logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
- cancelled = True
- proc.kill()
- await proc.wait()
+ await reader_task
+ except (CancelledError, asyncio.CancelledError):
+ pass
- elapsed = int(time() - start)
- stdout = stdout_file.read_bytes() or b''
- stderr = stderr_file.read_bytes() or b''
- # Logs details of the call, but only if it took a long time or TRACE is enabled.
- if log_command:
- logger.debug(
- f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
- elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
- logger.trace(
- f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
- return CommandResult(
- return_code=proc.returncode,
- cancelled=cancelled,
- stdout=stdout,
- stderr=stderr,
- elapsed=elapsed,
- pid=pid,
- )
+ elapsed = int(time() - start)
+ if streaming:
+ stdout = bytes(stdout_buf)
+ else:
+ stdout = stdout_file.read_bytes() or b''
+ stderr = stderr_file.read_bytes() or b''
+ if log_command:
+ logger.debug(
+ f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
+ elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
+ logger.trace(
+ f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
+ return CommandResult(
+ return_code=proc.returncode,
+ cancelled=cancelled,
+ stdout=stdout,
+ stderr=stderr,
+ elapsed=elapsed,
+ pid=pid,
+ )
+ finally:
+ if stdout_fh:
+ stdout_fh.close()
diff --git a/wrolpi/collections/lib.py b/wrolpi/collections/lib.py
index fcb7aa13..ff4d126f 100644
--- a/wrolpi/collections/lib.py
+++ b/wrolpi/collections/lib.py
@@ -273,6 +273,43 @@ def update_collection(
return collection
+def _get_external_collection_file_paths(session, collection_id: int, directory: pathlib.Path) -> list[pathlib.Path]:
+ """Return primary_path for FileGroups associated with a collection but outside its directory.
+
+ Checks both CollectionItem relationships and kind-specific relationships (e.g. Channel → Video)."""
+ from wrolpi.files.models import FileGroup
+ from .models import CollectionItem
+ dir_str = f'{directory}/'
+
+ external_paths = []
+
+ # Check CollectionItems.
+ item_fgs = session.query(FileGroup).join(
+ CollectionItem, CollectionItem.file_group_id == FileGroup.id
+ ).filter(
+ CollectionItem.collection_id == collection_id,
+ FileGroup.directory != str(directory),
+ ~FileGroup.directory.startswith(dir_str),
+ ).all()
+ for fg in item_fgs:
+ external_paths.append(fg.primary_path)
+
+ # For channel-kind collections, also check Videos linked via Channel.
+ collection = session.query(Collection).filter_by(id=collection_id).one()
+ if collection.kind == 'channel':
+ from modules.videos.models import Channel
+ channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
+ if channel:
+ channel_paths = Channel._get_external_file_paths(session, channel.id, directory)
+ # Deduplicate against paths already found via CollectionItems.
+ seen = set(external_paths)
+ for p in channel_paths:
+ if p not in seen:
+ external_paths.append(p)
+
+ return external_paths
+
+
def refresh_collection(collection_id: int, send_events: bool = True) -> None:
"""
Refresh all files in a collection's directory.
@@ -301,9 +338,11 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
)
directory = collection.directory
+ # Find files associated with this collection but outside its directory.
+ external_paths = _get_external_collection_file_paths(session, collection_id, directory)
# Refresh files asynchronously
- file_worker.queue_refresh([directory])
+ file_worker.queue_refresh([directory] + external_paths)
if send_events:
relative_dir = get_relative_to_media_directory(directory)
diff --git a/wrolpi/collections/test/test_reorganize.py b/wrolpi/collections/test/test_reorganize.py
index a1ba9639..f13fb704 100644
--- a/wrolpi/collections/test/test_reorganize.py
+++ b/wrolpi/collections/test/test_reorganize.py
@@ -603,7 +603,7 @@ async def test_execute_batch_reorganization(async_client, test_directory, kind,
assert result['collection_count'] >= 0 # May be 0 if no files to move
-def test_batch_reorganization_status_tracking(test_session):
+def test_batch_reorganization_status_tracking(async_client, test_session):
"""Status should show overall + per-collection progress."""
# Test with a non-existent batch job
status = get_batch_reorganization_status('nonexistent-batch-job')
diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py
index 4901a2d8..83f86046 100644
--- a/wrolpi/contexts.py
+++ b/wrolpi/contexts.py
@@ -134,6 +134,7 @@ def reset_shared_contexts(app: Sanic):
app.shared_ctx.download_manager_data.update(dict(
processing_domains=[],
killed_downloads=[],
+ download_progress={},
))
# Configs
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index efdba3e7..5b8eb467 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -6,6 +6,7 @@ import multiprocessing
import os
import pathlib
import random
+import re
import tempfile
import time
import traceback
@@ -29,6 +30,7 @@ from sqlalchemy.sql import Delete
from wrolpi import flags
from wrolpi.api_utils import api_app, perpetual_signal
+
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
@@ -46,6 +48,79 @@ logger = logger.getChild(__name__)
ARIA2C_PATH = which('aria2c', '/usr/bin/aria2c')
+ARIA2C_PROGRESS_RE = re.compile(
+ r'\[#\w+\s+'
+ r'([\d.]+\w+)/([\d.]+\w+)' # downloaded/total
+ r'\((\d+)%\)' # percent
+ r'.*?DL:([\d.]+\w+)' # speed
+ r'(?:.*?ETA:([^\s\]]+))?' # optional ETA
+)
+
+_SIZE_UNITS = [('TiB', 1024 ** 4), ('GiB', 1024 ** 3), ('MiB', 1024 ** 2), ('KiB', 1024), ('B', 1)]
+
+
+def _parse_size(s: str) -> int:
+ """Convert a size string like '4.5MiB' to bytes (int)."""
+ for suffix, multiplier in _SIZE_UNITS:
+ if s.endswith(suffix):
+ return int(float(s[:-len(suffix)]) * multiplier)
+ # Fallback: try parsing as pure number.
+ return int(float(s))
+
+
+def parse_aria2c_progress(line: str) -> dict | None:
+ """Parse an aria2c progress line and return a dict, or None if not a progress line."""
+ m = ARIA2C_PROGRESS_RE.search(line)
+ if not m:
+ return None
+ return dict(
+ bytes_downloaded=_parse_size(m.group(1)),
+ total_bytes=_parse_size(m.group(2)),
+ percent=int(m.group(3)),
+ speed=_parse_size(m.group(4)),
+ eta=m.group(5),
+ )
+
+
+def set_download_progress(download_id: int, progress: dict):
+ """Store download progress in shared_ctx."""
+ data = dict(api_app.shared_ctx.download_manager_data)
+ dp = dict(data.get('download_progress', {}))
+ dp[download_id] = progress
+ data['download_progress'] = dp
+ api_app.shared_ctx.download_manager_data.update(data)
+
+
+def clear_download_progress(download_id: int):
+ """Remove download progress from shared_ctx."""
+ data = dict(api_app.shared_ctx.download_manager_data)
+ dp = dict(data.get('download_progress', {}))
+ dp.pop(download_id, None)
+ data['download_progress'] = dp
+ api_app.shared_ctx.download_manager_data.update(data)
+
+
+def make_progress_callback(download_id: int, parse_fn: callable) -> callable:
+ """Create a stdout callback that parses progress lines and updates shared_ctx.
+
+ The callback throttles updates to at most once per second.
+
+ :param download_id: The download ID to update progress for.
+ :param parse_fn: A function that takes a line string and returns a progress dict or None.
+ """
+ last_update = [0.0]
+
+ def on_stdout(line: str):
+ now_ = time.monotonic()
+ if now_ - last_update[0] < 1.0:
+ return
+ progress = parse_fn(line)
+ if progress:
+ last_update[0] = now_
+ set_download_progress(download_id, progress)
+
+ return on_stdout
+
@perpetual_signal(sleep=1 if PYTEST else 5)
async def perpetual_download_worker():
@@ -314,7 +389,8 @@ class Downloader:
raise NotImplementedError()
async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
- timeout: int = None, debug: bool = True) -> CommandResult:
+ timeout: int = None, debug: bool = True,
+ stdout_callback: callable = None) -> CommandResult:
"""
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
@@ -324,7 +400,8 @@ class Downloader:
raise RuntimeError('cwd directory does not exist')
timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
- coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug)
+ coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
+ stdout_callback=stdout_callback)
result = await self.cancel_wrapper(coro, download)
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
@@ -343,23 +420,33 @@ class Downloader:
"""
download_id = download.id
task = asyncio.create_task(coro)
- while not task.done():
- if download_manager.download_is_killed(download_id) or not download_manager.can_download:
- logger.warning(f'Cancel download of {download.url}')
+ try:
+ while not task.done():
+ if download_manager.download_is_killed(download_id) or not download_manager.can_download:
+ logger.warning(f'Cancel download of {download.url}')
+ task.cancel()
+ try:
+ await task
+ except asyncio.CancelledError as e:
+ logger.info(f'Successful cancel of {download.url}', exc_info=e)
+ return DownloadResult(
+ success=False,
+ error='Download was canceled',
+ )
+ finally:
+ download_manager.unkill_download(download_id)
+ else:
+ # Wait for the download to complete. Cancel if requested.
+ await asyncio.sleep(0.1)
+ except asyncio.CancelledError:
+ # Outer cancellation (e.g. from a parent cancel_wrapper). Ensure the inner task is also cancelled.
+ if not task.done():
task.cancel()
try:
await task
- except asyncio.CancelledError as e:
- logger.info(f'Successful cancel of {download.url}', exc_info=e)
- return DownloadResult(
- success=False,
- error='Download was canceled',
- )
- finally:
- download_manager.unkill_download(download_id)
- else:
- # Wait for the download to complete. Cancel if requested.
- await asyncio.sleep(0.1)
+ except asyncio.CancelledError:
+ pass
+ raise
# Return the result of the download attempt.
return task.result()
@@ -424,7 +511,12 @@ class Downloader:
# the URL redirects to a CDN path that doesn't match the original filename.
cmd = (*cmd, '-o', output_path.name)
- result = await self.process_runner(download, cmd, destination)
+ on_stdout = make_progress_callback(download.id, parse_aria2c_progress)
+
+ try:
+ result = await self.process_runner(download, cmd, destination, stdout_callback=on_stdout)
+ finally:
+ clear_download_progress(download.id)
error = result.stderr.decode()
if result.return_code != 0:
@@ -1034,6 +1126,14 @@ class DownloadManager:
curs.execute(stmt)
pending_once_downloads = curs.fetchone()[0]
+ try:
+ progress_data = dict(api_app.shared_ctx.download_manager_data.get('download_progress', {}))
+ except Exception:
+ progress_data = {}
+ for download in once_downloads + recurring_downloads:
+ if download['id'] in progress_data:
+ download['progress'] = progress_data[download['id']]
+
data = dict(
recurring_downloads=recurring_downloads,
once_downloads=once_downloads,
@@ -1354,6 +1454,8 @@ async def signal_download_download(download_id: int, download_url: str):
# Remove this domain from the running list.
if download_domain:
download_manager._delete_processing_domain(download_domain)
+ # Ensure download progress is always cleaned up.
+ clear_download_progress(download_id)
@dataclass
diff --git a/wrolpi/files/test/test_downloader.py b/wrolpi/files/test/test_downloader.py
index 645ef149..69ae1d22 100644
--- a/wrolpi/files/test/test_downloader.py
+++ b/wrolpi/files/test/test_downloader.py
@@ -117,6 +117,7 @@ async def test_get_download_info_uuid_fallback():
assert info.name == 'somefile.tar.gz'
+@skip_circleci
async def test_file_downloader_aria2c_uses_output_flag(test_directory):
"""The aria2c command includes -o with the predicted filename when not using metalink."""
file_downloader = FileDownloader()
@@ -126,7 +127,7 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
captured_cmd = None
- async def fake_process_runner(download, cmd, dest):
+ async def fake_process_runner(download, cmd, dest, **kwargs):
nonlocal captured_cmd
captured_cmd = cmd
# Create the expected output file so the downloader doesn't raise.
@@ -141,7 +142,8 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
fake_info = DownloadFileInfo(name='somefile.tar.gz', size=1024, status=200)
with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \
- mock.patch.object(file_downloader, 'process_runner', fake_process_runner):
+ mock.patch.object(file_downloader, 'process_runner', fake_process_runner), \
+ mock.patch('wrolpi.downloader.clear_download_progress'):
download = mock.MagicMock()
url = 'https://github.com/user/repo/releases/download/v1.0/somefile.tar.gz'
await file_downloader.download_file(download, url, destination)
@@ -152,6 +154,7 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
assert captured_cmd[o_index + 1] == 'somefile.tar.gz'
+@skip_circleci
async def test_file_downloader_meta4_no_output_flag(test_directory):
"""The aria2c command should NOT include -o when using a metalink file."""
file_downloader = FileDownloader()
@@ -161,7 +164,7 @@ async def test_file_downloader_meta4_no_output_flag(test_directory):
captured_cmd = None
- async def fake_process_runner(download, cmd, dest):
+ async def fake_process_runner(download, cmd, dest, **kwargs):
nonlocal captured_cmd
captured_cmd = cmd
# Create the expected output file.
@@ -177,7 +180,8 @@ async def test_file_downloader_meta4_no_output_flag(test_directory):
with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \
mock.patch.object(file_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=b'')), \
- mock.patch.object(file_downloader, 'process_runner', fake_process_runner):
+ mock.patch.object(file_downloader, 'process_runner', fake_process_runner), \
+ mock.patch('wrolpi.downloader.clear_download_progress'):
download = mock.MagicMock()
url = 'https://example.com/somefile.tar.gz'
await file_downloader.download_file(download, url, destination)
diff --git a/wrolpi/test/test_cmd.py b/wrolpi/test/test_cmd.py
index c9b0c568..1e2681c5 100644
--- a/wrolpi/test/test_cmd.py
+++ b/wrolpi/test/test_cmd.py
@@ -67,3 +67,44 @@ async def test_run_command_cancel(test_directory):
assert result.stdout == b''
assert result.stderr == b''
assert result.elapsed == 1
+
+
+STREAMING_SCRIPT = '''#!/bin/sh
+echo "line one"
+echo "line two"
+echo "line three"
+echo stderr >&2
+'''
+
+
+async def test_run_command_streaming(test_directory):
+ """run_command_streaming captures stdout line-by-line and calls callback."""
+ (test_directory / 'stream.sh').write_text(STREAMING_SCRIPT)
+ lines = []
+ result = await cmd.run_command(
+ ('sh', 'stream.sh'),
+ cwd=test_directory,
+ stdout_callback=lambda line: lines.append(line),
+ )
+ assert result.return_code == 0
+ assert result.cancelled is False
+ assert result.stdout == b'line one\nline two\nline three\n'
+ assert result.stderr == b'stderr\n'
+ assert lines == ['line one', 'line two', 'line three']
+
+
+async def test_run_command_streaming_no_callback(test_directory):
+ """run_command_streaming works without a callback."""
+ (test_directory / 'stream.sh').write_text(STREAMING_SCRIPT)
+ result = await cmd.run_command(('sh', 'stream.sh'), cwd=test_directory)
+ assert result.return_code == 0
+ assert result.stdout == b'line one\nline two\nline three\n'
+ assert result.stderr == b'stderr\n'
+
+
+async def test_run_command_streaming_timeout(test_directory):
+ """Timeout is obeyed for streaming."""
+ result = await cmd.run_command(('sleep', '10'), timeout=2)
+ assert result.return_code == -9
+ assert result.cancelled is False
+ assert result.elapsed >= 2
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index 85c9c1b0..11ab9677 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -15,11 +15,131 @@ from wrolpi.common import get_wrolpi_config
from wrolpi.dates import Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
- get_download_manager_config, RSSDownloader
+ get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
+ set_download_progress, clear_download_progress, make_progress_callback, download_manager
from wrolpi.errors import InvalidDownload, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
+def test_parse_size():
+ """Size strings are parsed to bytes."""
+ assert _parse_size('0B') == 0
+ assert _parse_size('1024B') == 1024
+ assert _parse_size('1.0KiB') == 1024
+ assert _parse_size('4.5MiB') == int(4.5 * 1024 ** 2)
+ assert _parse_size('1.0GiB') == 1024 ** 3
+ assert _parse_size('2.5TiB') == int(2.5 * 1024 ** 4)
+
+
+def test_parse_aria2c_progress():
+ """aria2c progress lines are parsed correctly."""
+ line = '[#c52705 1.0MiB/1.0GiB(0%) CN:1 DL:4.5MiB ETA:3m44s]'
+ result = parse_aria2c_progress(line)
+ assert result is not None
+ assert result['bytes_downloaded'] == int(1.0 * 1024 ** 2)
+ assert result['total_bytes'] == 1024 ** 3
+ assert result['percent'] == 0
+ assert result['speed'] == int(4.5 * 1024 ** 2)
+ assert result['eta'] == '3m44s'
+
+
+def test_parse_aria2c_progress_no_eta():
+ """aria2c progress lines without ETA are parsed."""
+ line = '[#abc123 500.0MiB/1.0GiB(48%) CN:3 DL:10.0MiB]'
+ result = parse_aria2c_progress(line)
+ assert result is not None
+ assert result['percent'] == 48
+ assert result['speed'] == int(10.0 * 1024 ** 2)
+ assert result['eta'] is None
+
+
+def test_parse_aria2c_progress_not_progress_line():
+ """Non-progress lines return None."""
+ assert parse_aria2c_progress('03/12 10:00:00 [NOTICE] Download complete') is None
+ assert parse_aria2c_progress('') is None
+ assert parse_aria2c_progress('some random text') is None
+
+
+@pytest.mark.asyncio
+async def test_set_and_clear_download_progress(async_client):
+ """set_download_progress stores progress in shared_ctx and clear_download_progress removes it."""
+ from wrolpi.api_utils import api_app
+ # No progress initially.
+ data = dict(api_app.shared_ctx.download_manager_data)
+ assert data.get('download_progress', {}) == {}
+
+ # Set progress for a download.
+ progress = dict(bytes_downloaded=1024, total_bytes=2048, percent=50, speed=512, eta='2s')
+ set_download_progress(42, progress)
+ data = dict(api_app.shared_ctx.download_manager_data)
+ assert data['download_progress'][42] == progress
+
+ # Clear it.
+ clear_download_progress(42)
+ data = dict(api_app.shared_ctx.download_manager_data)
+ assert 42 not in data.get('download_progress', {})
+
+
+@pytest.mark.asyncio
+async def test_make_progress_callback(async_client):
+ """make_progress_callback creates a callback that throttles and updates shared_ctx."""
+ from wrolpi.api_utils import api_app
+
+ def fake_parse(line):
+ if 'progress' in line:
+ return dict(percent=50)
+ return None
+
+ callback = make_progress_callback(99, fake_parse)
+
+ # Non-matching line does nothing.
+ callback('some other line')
+ data = dict(api_app.shared_ctx.download_manager_data)
+ assert 99 not in data.get('download_progress', {})
+
+ # Matching line updates shared_ctx.
+ with mock.patch('wrolpi.downloader.time') as mock_time:
+ mock_time.monotonic.return_value = 100.0
+ callback('progress line')
+ data = dict(api_app.shared_ctx.download_manager_data)
+ assert data['download_progress'][99] == dict(percent=50)
+
+ # A call within 1 second is throttled (uses a different progress value to verify).
+ mock_time.monotonic.return_value = 100.5
+ callback('progress line') # Would update, but throttled.
+
+ # Clean up.
+ clear_download_progress(99)
+
+
+@pytest.mark.asyncio
+async def test_cancel_wrapper_outer_cancellation(test_session, test_download_manager, test_downloader):
+ """cancel_wrapper propagates outer CancelledError and cancels the inner task."""
+ download = test_download_manager.create_download(test_session, 'https://example.com/cancel', test_downloader.name)
+ test_session.commit()
+
+ inner_cancelled = asyncio.Event()
+
+ async def slow_coro():
+ try:
+ await asyncio.sleep(60)
+ except asyncio.CancelledError:
+ inner_cancelled.set()
+ raise
+
+ wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download))
+ # Give the wrapper time to start polling.
+ await asyncio.sleep(0.3)
+
+ # Cancel the wrapper from outside (simulating parent cancellation).
+ wrapper_task.cancel()
+ with pytest.raises(asyncio.CancelledError):
+ await wrapper_task
+
+ # The inner coroutine should have been cancelled too.
+ assert inner_cancelled.is_set()
+
+
@pytest.mark.asyncio
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
"""Once-downloads over a month old should be deleted."""