Displaying download progress (speed, eta) for file and video downloads.

This commit is contained in:
Roland 2026-03-12 17:48:35 -06:00
parent fa97b25831
commit 4579562be5
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
16 changed files with 912 additions and 101 deletions

View file

@ -111,6 +111,17 @@ input:checked + .slider:before {
transform: translateX(26px);
}
.inverted-progress-text.ui.indicating.progress .bar > .progress,
.inverted-progress-text.ui.progress > .label {
color: white !important;
}
/* Light bar colors (yellow, olive) need dark text on the bar — must override inverted-progress-text. */
.light-bar-progress-text.ui.indicating.progress .bar > .progress,
.light-bar-progress-text.ui.progress .bar > .progress {
color: rgba(0, 0, 0, 0.85) !important;
}
div.full-height {
overflow: hidden;
padding-top: 100%;

View file

@ -0,0 +1,81 @@
import React from 'react';
import {Progress} from './Theme';
import {renderInDarkMode, renderInLightMode} from '../test-utils';
// All semantic-ui colors.
const allColors = ['red', 'orange', 'yellow', 'olive', 'green', 'teal', 'blue', 'violet', 'purple', 'pink', 'brown', 'grey'];
// Light bar colors that always need dark percent text (light bars in any theme).
const lightBarColors = ['yellow', 'olive', 'teal'];
// Bar colors that need dark percent text only in dark mode (inverted makes bar lighter).
const darkModeLightBarColors = ['grey', 'blue'];
describe('Progress component theme support', () => {
describe('dark mode', () => {
it('adds inverted-progress-text class for label readability', () => {
const {container} = renderInDarkMode(
<Progress percent={39} progress indicating color='violet'>
2.1 GB / 5.3 GB
</Progress>
);
const progressEl = container.querySelector('.ui.progress');
expect(progressEl).toHaveClass('inverted-progress-text');
const progressLabel = container.querySelector('.bar > .progress');
expect(progressLabel).toHaveTextContent('39%');
const label = container.querySelector('.ui.progress > .label');
expect(label).toHaveTextContent('2.1 GB / 5.3 GB');
});
it('does NOT add inverted-progress-text in light mode', () => {
const {container} = renderInLightMode(
<Progress percent={39} progress indicating color='violet'>
2.1 GB / 5.3 GB
</Progress>
);
const progressEl = container.querySelector('.ui.progress');
expect(progressEl).not.toHaveClass('inverted-progress-text');
});
});
describe('percent text contrast per color', () => {
allColors.forEach(color => {
it(`${color} bar has readable percent text in light mode`, () => {
const {container} = renderInLightMode(
<Progress percent={50} progress color={color}/>
);
const progressLabel = container.querySelector('.bar > .progress');
expect(progressLabel).toBeInTheDocument();
expect(progressLabel).toHaveTextContent('50%');
const progressEl = container.querySelector('.ui.progress');
if (lightBarColors.includes(color)) {
expect(progressEl).toHaveClass('light-bar-progress-text');
} else {
expect(progressEl).not.toHaveClass('light-bar-progress-text');
}
});
it(`${color} bar has readable percent text in dark mode`, () => {
const {container} = renderInDarkMode(
<Progress percent={50} progress color={color}/>
);
const progressLabel = container.querySelector('.bar > .progress');
expect(progressLabel).toBeInTheDocument();
expect(progressLabel).toHaveTextContent('50%');
const progressEl = container.querySelector('.ui.progress');
if (lightBarColors.includes(color) || darkModeLightBarColors.includes(color)) {
expect(progressEl).toHaveClass('light-bar-progress-text');
} else {
expect(progressEl).not.toHaveClass('light-bar-progress-text');
}
});
});
});
});

View file

@ -313,9 +313,20 @@ export function Popup(props: PopupProps) {
return <SPopup {...i} style={{border: '1px solid grey'}} {...props}/>
}
// Bar colors that always need dark percent text (light bars in any theme).
const PROGRESS_BAR_COLORS = ['yellow', 'olive', 'teal'];
// Bar colors that need dark percent text only in dark mode (inverted makes bar lighter).
const DARK_MODE_PROGRESS_BAR_COLORS = ['grey', 'blue'];
export function Progress(props: ProgressProps) {
const {i} = useContext(ThemeContext);
return <SProgress {...i} {...props}/>
const {i, inverted} = useContext(ThemeContext);
const classes = [props.className || ''];
if (inverted) classes.push('inverted-progress-text');
const color = props.color as string;
if (PROGRESS_BAR_COLORS.includes(color) || (inverted && DARK_MODE_PROGRESS_BAR_COLORS.includes(color))) {
classes.push('light-bar-progress-text');
}
return <SProgress {...i} {...props} className={classes.join(' ').trim()}/>
}
export function Segment(props: SegmentProps) {

View file

@ -17,7 +17,10 @@ import {
DisableDownloadsToggle,
ErrorMessage,
formatFrequency,
humanBandwidth,
humanFileSize,
isoDatetimeToElapsedPopup,
useLocalStorage,
useTitle,
WROLModeMessage
} from "../Common";
@ -25,7 +28,6 @@ import {
Button as SButton,
ButtonGroup,
Checkbox,
Icon,
Label,
Loader,
PlaceholderLine,
@ -36,7 +38,7 @@ import {
TableHeaderCell,
TableRow
} from "semantic-ui-react";
import {Button, Header, Modal, Placeholder, Segment, Table} from "../Theme";
import {Button, Header, Modal, Placeholder, Progress, Segment, Table} from "../Theme";
import {useDownloads} from "../../hooks/customHooks";
import {
EditArchiveDownloadForm,
@ -49,6 +51,57 @@ import {
import {Downloaders} from "../Vars";
import {SortableTable} from "../SortableTable";
function DownloadProgressModal({progress, url}) {
const [open, setOpen] = React.useState(false);
const [navColor] = useLocalStorage('nav_color', 'violet');
const speedText = humanBandwidth(progress.speed);
const etaText = progress.eta || '...';
const buttonLabel = `${speedText} / ${etaText}`;
return <>
<Button size='small' onClick={() => setOpen(true)} style={{marginLeft: '0.5em'}}>
{buttonLabel}
</Button>
<Modal closeIcon open={open} onClose={() => setOpen(false)} size='small'>
<Modal.Header>Download Progress</Modal.Header>
<Modal.Content>
<Progress
percent={progress.percent}
progress
indicating
color={navColor}
>
{humanFileSize(progress.bytes_downloaded)} / {humanFileSize(progress.total_bytes)}
</Progress>
<Table definition>
<TableBody>
<TableRow>
<TableCell width={4}>Speed</TableCell>
<TableCell>{speedText}</TableCell>
</TableRow>
<TableRow>
<TableCell>ETA</TableCell>
<TableCell>{progress.eta || 'Calculating...'}</TableCell>
</TableRow>
<TableRow>
<TableCell>Downloaded</TableCell>
<TableCell>{humanFileSize(progress.bytes_downloaded)} / {humanFileSize(progress.total_bytes)}</TableCell>
</TableRow>
{url && <TableRow>
<TableCell>URL</TableCell>
<TableCell style={{wordBreak: 'break-all'}}>{url}</TableCell>
</TableRow>}
</TableBody>
</Table>
</Modal.Content>
<Modal.Actions>
<Button onClick={() => setOpen(false)}>Close</Button>
</Modal.Actions>
</Modal>
</>;
}
function ClearDownloadsButton({callback, selectedIds, clearSelection}) {
const hasSelection = selectedIds && selectedIds.length > 0;
@ -271,11 +324,12 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
<TableCell>{formatFrequency(frequency)}</TableCell>
<TableCell>
{last_successful_download && isoDatetimeToElapsedPopup(last_successful_download)}
{status === 'pending' && <Loader active inline size='tiny'/>}
{status === 'pending' && !download.progress && <Loader active inline size='tiny'/>}
{download.progress && <DownloadProgressModal progress={download.progress} url={download.url}/>}
</TableCell>
<TableCell>{next}</TableCell>
<TableCell textAlign='right'>
{error && errorModal}
{error && !download.progress && errorModal}
{editButton}
{restartButton}
</TableCell>
@ -387,7 +441,7 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
</TableCell>
);
}
if (error) {
if (error && !download.progress) {
completedAtCell = (
<Modal
closeIcon
@ -427,8 +481,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
{parentDownloadUrl && (
<p style={{marginTop: '1em', color: '#666'}}>
From: <a href={parentDownloadUrl} target='_blank' rel='noopener noreferrer'>
{parentDownloadUrl}
</a>
{parentDownloadUrl}
</a>
</p>
)}
</Modal.Content>
@ -456,8 +510,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
{parentDownloadUrl && (
<p style={{marginTop: '1em', color: '#666'}}>
From: <a href={parentDownloadUrl} target='_blank' rel='noopener noreferrer'>
{parentDownloadUrl}
</a>
{parentDownloadUrl}
</a>
</p>
)}
</Modal.Content>
@ -477,7 +531,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
</TableCell>
<TableCell>
{completedAtCell}
{status === 'pending' ? <Loader active inline size='tiny'/> : null}
{status === 'pending' && !download.progress ? <Loader active inline size='tiny'/> : null}
{download.progress && <DownloadProgressModal progress={download.progress} url={download.url}/>}
</TableCell>
{buttonCell}
{editModal}

View file

@ -5,6 +5,7 @@ import logging
import os.path
import pathlib
import re
import time
import traceback
from abc import ABC
from datetime import timedelta
@ -23,7 +24,8 @@ from wrolpi.common import logger, get_media_directory, escape_file_name, resolve
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
from wrolpi.dates import now
from wrolpi.db import get_db_session
from wrolpi.downloader import Downloader, Download, DownloadResult
from wrolpi.downloader import Downloader, Download, DownloadResult, \
clear_download_progress, _parse_size, make_progress_callback
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
from wrolpi.files.models import FileGroup
@ -43,6 +45,36 @@ from .video.lib import download_video_info_json
logger = logger.getChild(__name__)
YTDLP_PROGRESS_RE = re.compile(
r'\[download\]\s+'
r'([\d.]+)%\s+' # percent
r'of\s+~?\s*([\d.]+\w+)\s+' # total size (with optional ~)
r'at\s+([\d.]+\w+/s)\s+' # speed
r'ETA\s+(\S+)' # ETA
)
def parse_ytdlp_progress(line: str) -> dict | None:
"""Parse a yt-dlp progress line (with --newline) and return a dict, or None if not a progress line."""
m = YTDLP_PROGRESS_RE.search(line)
if not m:
return None
percent = float(m.group(1))
total_bytes = _parse_size(m.group(2))
speed_str = m.group(3).rstrip('/s')
speed = _parse_size(speed_str)
eta = m.group(4)
if eta == 'Unknown':
eta = None
return dict(
bytes_downloaded=int(percent / 100.0 * total_bytes),
total_bytes=total_bytes,
percent=int(percent),
speed=speed,
eta=eta,
)
VIDEO_RESOLUTION_MAP = {
'360p': [
'134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
@ -585,6 +617,7 @@ class VideoDownloader(Downloader, ABC):
'--convert-thumbnails', DEFAULT_POSTER_FORMAT,
'--merge-output-format', video_format,
'--no-cache-dir',
'--newline',
'--compat-options', 'no-live-chat',
# Get top 20 comments, 10 replies per parent.
'--write-comments',
@ -605,20 +638,27 @@ class VideoDownloader(Downloader, ABC):
if effective['sleep_requests'] > 0:
cmd = (*cmd, '--sleep-requests', str(effective['sleep_requests']))
if cookies_unlocked():
# Use encrypted cookies file
logger.info(f'Using encrypted cookies for video download: {url}')
with cookies_for_download() as cookies_path:
cmd = (*cmd, '--cookies', str(cookies_path))
if effective['user_agent']:
cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
on_stdout = make_progress_callback(download.id, parse_ytdlp_progress)
try:
if cookies_unlocked():
# Use encrypted cookies file
logger.info(f'Using encrypted cookies for video download: {url}')
with cookies_for_download() as cookies_path:
cmd = (*cmd, '--cookies', str(cookies_path))
if effective['user_agent']:
cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
cmd = (*cmd, '-o', video_path, url)
result = await self.process_runner(download, cmd, out_dir, debug=True,
stdout_callback=on_stdout)
else:
# No cookies available
logger.info(f'No cookies available for video download: {url}')
cmd = (*cmd, '-o', video_path, url)
result = await self.process_runner(download, cmd, out_dir, debug=True)
else:
# No cookies available
logger.info(f'No cookies available for video download: {url}')
cmd = (*cmd, '-o', video_path, url)
result = await self.process_runner(download, cmd, out_dir, debug=True)
result = await self.process_runner(download, cmd, out_dir, debug=True,
stdout_callback=on_stdout)
finally:
clear_download_progress(download.id)
stdout = result.stdout.decode()
stderr = result.stderr.decode()

View file

@ -955,6 +955,17 @@ class Channel(ModelHelper, Base):
self.tag_id = self.tag.id if self.tag else None
return self.tag
@staticmethod
def _get_external_file_paths(session, channel_id: int, directory: pathlib.Path) -> list[pathlib.Path]:
"""Return primary_path for all Videos in this channel whose FileGroup is outside the channel directory."""
dir_str = f'{directory}/'
file_groups = session.query(FileGroup).join(Video, Video.file_group_id == FileGroup.id).filter(
Video.channel_id == channel_id,
FileGroup.directory != str(directory),
~FileGroup.directory.startswith(dir_str),
).all()
return [fg.primary_path for fg in file_groups]
@classmethod
def refresh_files(cls, id_: int, send_events: bool = True) -> str:
"""Refresh all files within this Channel's directory. Returns job_id for awaiting.
@ -962,10 +973,13 @@ class Channel(ModelHelper, Base):
The channel is marked as refreshed after the background task completes post-processing."""
# Get this Channel's info for later. Refresh may take a long time.
with get_db_session() as session:
directory = cls.find_by_id(session, id_).directory
channel = cls.find_by_id(session, id_)
directory = channel.directory
# Find files associated with this channel but outside its directory.
external_paths = cls._get_external_file_paths(session, id_, directory)
# Queue refresh immediately so caller can await the job.
job_id = file_worker.queue_refresh([directory])
job_id = file_worker.queue_refresh([directory] + external_paths)
# Background task handles post-processing after job completes.
async def _():

View file

@ -0,0 +1,198 @@
"""Tests that Channel.refresh_files() and refresh_collection() detect changes to files
that are associated with the channel but located outside the channel's directory."""
import shutil
import pytest
from modules.videos.models import Channel, Video
from wrolpi.collections import Collection
from wrolpi.collections.lib import refresh_collection
from wrolpi.collections.models import CollectionItem
from wrolpi.common import await_background_tasks
from wrolpi.conftest import await_file_worker
from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
class TestChannelRefreshExternalFiles:
async def _setup_channel_with_external_video(
self, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
"""Create a channel with an internal and external video, then delete the external video from disk.
Returns (channel, internal_video_id, external_video_id, external_path)."""
channel = channel_factory(name='MyChannel')
assert channel.directory.is_dir()
# Create a video inside the channel directory.
internal_path = channel.directory / 'internal_video.mp4'
video_file_factory(internal_path)
# Create a video outside the channel directory.
external_dir = test_directory / 'videos' / 'other'
external_dir.mkdir(parents=True, exist_ok=True)
external_path = external_dir / 'external_video.mp4'
video_file_factory(external_path)
# Refresh to index all files and model videos.
await refresh_files()
# Verify both videos exist.
internal_video = test_session.query(Video).join(Video.file_group).filter(
FileGroup.primary_path == internal_path,
).one()
external_video = test_session.query(Video).join(Video.file_group).filter(
FileGroup.primary_path == external_path,
).one()
# Assign the external video to the channel.
external_video.channel_id = channel.id
test_session.commit()
internal_video_id = internal_video.id
external_video_id = external_video.id
# Verify channel has 2 videos.
channel_videos = test_session.query(Video).filter_by(channel_id=channel.id).all()
assert len(channel_videos) == 2, f'Expected 2 channel videos, got {len(channel_videos)}'
# Delete ALL files for the external video by removing the directory.
shutil.rmtree(external_dir)
assert not external_path.exists()
return channel, internal_video_id, external_video_id, external_path
def _assert_external_deleted_internal_kept(self, test_session, internal_video_id, external_video_id, external_path):
"""Assert the external video was cleaned up and the internal video remains."""
test_session.expire_all()
external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one_or_none()
assert external_fg is None, \
'External video FileGroup should have been removed after files were deleted from disk'
external_vid = test_session.query(Video).filter_by(id=external_video_id).one_or_none()
assert external_vid is None, \
'External Video record should have been removed after its files were deleted'
internal_vid = test_session.query(Video).filter_by(id=internal_video_id).one_or_none()
assert internal_vid is not None, 'Internal video should still exist'
async def test_channel_refresh_files_detects_deleted_external_video(
self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
"""Channel.refresh_files() should detect and remove an external video whose files were deleted."""
channel, internal_id, external_id, external_path = await self._setup_channel_with_external_video(
test_session, test_directory, video_file_factory, channel_factory, refresh_files)
Channel.refresh_files(channel.id)
await await_file_worker()
await await_background_tasks()
self._assert_external_deleted_internal_kept(test_session, internal_id, external_id, external_path)
async def test_refresh_collection_detects_deleted_external_video(
self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
"""refresh_collection() should detect and remove an external video whose files were deleted."""
channel, internal_id, external_id, external_path = await self._setup_channel_with_external_video(
test_session, test_directory, video_file_factory, channel_factory, refresh_files)
collection = test_session.query(Collection).filter_by(id=channel.collection_id).one()
refresh_collection(collection.id)
await await_file_worker()
self._assert_external_deleted_internal_kept(test_session, internal_id, external_id, external_path)
async def test_refresh_collection_detects_deleted_external_collection_item(
self, async_client, test_session, test_directory, video_file_factory, refresh_files):
"""refresh_collection() should detect and remove an external CollectionItem-linked FileGroup
whose files were deleted from disk."""
# Create a generic collection with a directory.
collection_dir = test_directory / 'my_collection'
collection_dir.mkdir(parents=True, exist_ok=True)
collection = Collection(name='MyCollection', directory=collection_dir, kind='simple')
test_session.add(collection)
test_session.flush()
# Create a video inside the collection directory.
internal_path = collection_dir / 'internal_video.mp4'
video_file_factory(internal_path)
# Create a video outside the collection directory.
external_dir = test_directory / 'elsewhere'
external_dir.mkdir(parents=True, exist_ok=True)
external_path = external_dir / 'external_video.mp4'
video_file_factory(external_path)
# Refresh to index all files.
await refresh_files()
internal_fg = test_session.query(FileGroup).filter_by(primary_path=internal_path).one()
external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one()
external_fg_id = external_fg.id
# Link the external FileGroup to the collection via CollectionItem.
item = CollectionItem(collection_id=collection.id, file_group_id=external_fg.id)
test_session.add(item)
test_session.commit()
# Delete the external files.
shutil.rmtree(external_dir)
assert not external_path.exists()
# Refresh via collection.
refresh_collection(collection.id)
await await_file_worker()
test_session.expire_all()
external_fg = test_session.query(FileGroup).filter_by(id=external_fg_id).one_or_none()
assert external_fg is None, \
'External CollectionItem FileGroup should have been removed after files were deleted from disk'
internal_fg = test_session.query(FileGroup).filter_by(primary_path=internal_path).one_or_none()
assert internal_fg is not None, 'Internal FileGroup should still exist'
async def test_channel_refresh_detects_new_file_in_external_file_group(
self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
"""Channel.refresh_files() should detect a new sibling file added to an external video's FileGroup."""
channel = channel_factory(name='MyChannel')
# Create a video outside the channel directory.
external_dir = test_directory / 'videos' / 'other'
external_dir.mkdir(parents=True, exist_ok=True)
external_path = external_dir / 'external_video.mp4'
video_file_factory(external_path)
# Refresh to index the video.
await refresh_files()
external_video = test_session.query(Video).join(Video.file_group).filter(
FileGroup.primary_path == external_path,
).one()
# Assign the external video to the channel.
external_video.channel_id = channel.id
test_session.commit()
# Record the initial file count.
external_fg = external_video.file_group
initial_file_names = {f['path'] for f in external_fg.files}
assert 'external_video.mp4' in initial_file_names
# Add a subtitle file next to the video with the same stem.
subtitle_path = external_dir / 'external_video.en.vtt'
subtitle_path.write_text('WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nHello\n')
# Refresh the channel's files.
Channel.refresh_files(channel.id)
await await_file_worker()
await await_background_tasks()
test_session.expire_all()
# The FileGroup should now include the subtitle file.
external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one()
file_names = {f['path'] for f in external_fg.files}
assert 'external_video.en.vtt' in file_names, \
f'Subtitle file should have been detected in external FileGroup, got: {file_names}'
assert 'external_video.mp4' in file_names, 'Original video file should still be present'
assert external_fg.primary_path == external_path, 'Primary path should still be the .mp4'

View file

@ -9,7 +9,8 @@ import pytest
from modules.videos.downloader import VideoDownloader, \
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
parse_ytdlp_progress
from modules.videos.lib import get_videos_downloader_config
from modules.videos.models import Channel, Video
from wrolpi.conftest import test_directory, await_switches
@ -915,3 +916,50 @@ async def test_video_download_playlist_appended_extension(test_session, test_dir
video: Video = test_session.query(Video).one()
assert video.video_path == video_path_na
def test_parse_ytdlp_progress():
"""yt-dlp progress lines are parsed correctly."""
line = '[download] 50.0% of ~ 45.67MiB at 2.34MiB/s ETA 00:19'
result = parse_ytdlp_progress(line)
assert result is not None
total = int(45.67 * 1024 ** 2)
assert result['total_bytes'] == total
assert result['percent'] == 50
assert result['bytes_downloaded'] == int(0.5 * total)
assert result['speed'] == int(2.34 * 1024 ** 2)
assert result['eta'] == '00:19'
def test_parse_ytdlp_progress_no_estimated():
"""yt-dlp progress lines without ~ estimated marker are parsed."""
line = '[download] 100% of 45.67MiB at 3.45MiB/s ETA 00:00'
result = parse_ytdlp_progress(line)
assert result is not None
assert result['percent'] == 100
assert result['total_bytes'] == int(45.67 * 1024 ** 2)
assert result['speed'] == int(3.45 * 1024 ** 2)
assert result['eta'] == '00:00'
def test_parse_ytdlp_progress_small_percent():
"""yt-dlp progress lines with small percentages are parsed."""
line = '[download] 0.1% of ~ 45.67MiB at 1.23MiB/s ETA 00:37'
result = parse_ytdlp_progress(line)
assert result is not None
assert result['percent'] == 0
assert result['eta'] == '00:37'
def test_parse_ytdlp_progress_completion_line():
"""The final yt-dlp line with 'in' instead of 'at/ETA' returns None (no speed/ETA)."""
line = '[download] 100% of 45.67MiB in 00:13'
result = parse_ytdlp_progress(line)
assert result is None
def test_parse_ytdlp_progress_not_progress_line():
"""Non-progress lines return None."""
assert parse_ytdlp_progress('[info] Downloading video 1 of 1') is None
assert parse_ytdlp_progress('') is None
assert parse_ytdlp_progress('[download] Destination: video.mp4') is None

View file

@ -155,10 +155,15 @@ TESTING_RUN_COMMAND_RESULT = None
async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
timeout: int = 600, log_command: bool = True) -> CommandResult:
timeout: int = 600, log_command: bool = True,
stdout_callback: callable = None) -> CommandResult:
"""Run a shell command, return the results (stdout/stderr/return code).
When stdout_callback is provided, stdout is piped and each line is passed to the callback
in real-time. Otherwise, stdout is written to a temporary file.
:param log_command: Enable debug logging.
:param stdout_callback: Called with each decoded stdout line (str). May be None.
"""
if not isinstance(cmd, (list, tuple)):
raise RuntimeError('Command must be a list or tuple')
@ -167,61 +172,102 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
logger.debug('run_command: returning mock result')
return TESTING_RUN_COMMAND_RESULT() # call the mock
with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh:
stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name)
streaming = stdout_callback is not None
with tempfile.NamedTemporaryFile() as stderr_fh:
stderr_file = pathlib.Path(stderr_fh.name)
cmd = tuple(str(i) for i in cmd)
cmd_str = ' '.join(shlex.quote(i) for i in cmd)
if log_command:
logger.info(f'run_command: running ({timeout=}): {cmd_str}')
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(f'run_command: running ({timeout=}): {cmd_str}')
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=stdout_fh, # Use stdout/stderr files to avoid buffer filling up.
stderr=stderr_fh,
cwd=cwd,
)
pid = proc.pid
cancelled = False
start = time()
stdout_fh = None
stdout_file = None
reader_task = None
stdout_buf = None
if streaming:
stdout_arg = asyncio.subprocess.PIPE
stdout_buf = bytearray()
else:
stdout_fh = tempfile.NamedTemporaryFile()
stdout_file = pathlib.Path(stdout_fh.name)
stdout_arg = stdout_fh
try:
while True:
elapsed = int(time() - start)
if timeout and elapsed >= timeout:
logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
proc.kill()
await proc.wait()
break
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=stdout_arg,
stderr=stderr_fh,
cwd=cwd,
)
pid = proc.pid
if streaming:
async def _read_stdout():
while True:
line = await proc.stdout.readline()
if not line:
break
stdout_buf.extend(line)
try:
stdout_callback(line.decode(errors='replace').rstrip('\n'))
except Exception as e:
logger.warning(f'run_command: stdout_callback error: {e}')
reader_task = asyncio.create_task(_read_stdout())
cancelled = False
start = time()
try:
while True:
elapsed = int(time() - start)
if timeout and elapsed >= timeout:
logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
proc.kill()
await proc.wait()
break
try:
await asyncio.wait_for(proc.wait(), timeout=1)
break
except asyncio.TimeoutError:
continue
except CancelledError as e:
logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
cancelled = True
proc.kill()
await proc.wait()
if reader_task:
reader_task.cancel()
try:
# Wait for the process to finish.
await asyncio.wait_for(proc.wait(), timeout=1)
break
except asyncio.TimeoutError:
# Task is not done, keep waiting...
continue
except CancelledError as e:
logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
cancelled = True
proc.kill()
await proc.wait()
await reader_task
except (CancelledError, asyncio.CancelledError):
pass
elapsed = int(time() - start)
stdout = stdout_file.read_bytes() or b''
stderr = stderr_file.read_bytes() or b''
# Logs details of the call, but only if it took a long time or TRACE is enabled.
if log_command:
logger.debug(
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
return CommandResult(
return_code=proc.returncode,
cancelled=cancelled,
stdout=stdout,
stderr=stderr,
elapsed=elapsed,
pid=pid,
)
elapsed = int(time() - start)
if streaming:
stdout = bytes(stdout_buf)
else:
stdout = stdout_file.read_bytes() or b''
stderr = stderr_file.read_bytes() or b''
if log_command:
logger.debug(
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
logger.trace(
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
return CommandResult(
return_code=proc.returncode,
cancelled=cancelled,
stdout=stdout,
stderr=stderr,
elapsed=elapsed,
pid=pid,
)
finally:
if stdout_fh:
stdout_fh.close()

View file

@ -273,6 +273,43 @@ def update_collection(
return collection
def _get_external_collection_file_paths(session, collection_id: int, directory: pathlib.Path) -> list[pathlib.Path]:
"""Return primary_path for FileGroups associated with a collection but outside its directory.
Checks both CollectionItem relationships and kind-specific relationships (e.g. Channel → Video)."""
from wrolpi.files.models import FileGroup
from .models import CollectionItem
dir_str = f'{directory}/'
external_paths = []
# Check CollectionItems.
item_fgs = session.query(FileGroup).join(
CollectionItem, CollectionItem.file_group_id == FileGroup.id
).filter(
CollectionItem.collection_id == collection_id,
FileGroup.directory != str(directory),
~FileGroup.directory.startswith(dir_str),
).all()
for fg in item_fgs:
external_paths.append(fg.primary_path)
# For channel-kind collections, also check Videos linked via Channel.
collection = session.query(Collection).filter_by(id=collection_id).one()
if collection.kind == 'channel':
from modules.videos.models import Channel
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
if channel:
channel_paths = Channel._get_external_file_paths(session, channel.id, directory)
# Deduplicate against paths already found via CollectionItems.
seen = set(external_paths)
for p in channel_paths:
if p not in seen:
external_paths.append(p)
return external_paths
def refresh_collection(collection_id: int, send_events: bool = True) -> None:
"""
Refresh all files in a collection's directory.
@ -301,9 +338,11 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
)
directory = collection.directory
# Find files associated with this collection but outside its directory.
external_paths = _get_external_collection_file_paths(session, collection_id, directory)
# Refresh files asynchronously
file_worker.queue_refresh([directory])
file_worker.queue_refresh([directory] + external_paths)
if send_events:
relative_dir = get_relative_to_media_directory(directory)

View file

@ -603,7 +603,7 @@ async def test_execute_batch_reorganization(async_client, test_directory, kind,
assert result['collection_count'] >= 0 # May be 0 if no files to move
def test_batch_reorganization_status_tracking(test_session):
def test_batch_reorganization_status_tracking(async_client, test_session):
"""Status should show overall + per-collection progress."""
# Test with a non-existent batch job
status = get_batch_reorganization_status('nonexistent-batch-job')

View file

@ -134,6 +134,7 @@ def reset_shared_contexts(app: Sanic):
app.shared_ctx.download_manager_data.update(dict(
processing_domains=[],
killed_downloads=[],
download_progress={},
))
# Configs

View file

@ -6,6 +6,7 @@ import multiprocessing
import os
import pathlib
import random
import re
import tempfile
import time
import traceback
@ -29,6 +30,7 @@ from sqlalchemy.sql import Delete
from wrolpi import flags
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
@ -46,6 +48,79 @@ logger = logger.getChild(__name__)
ARIA2C_PATH = which('aria2c', '/usr/bin/aria2c')
ARIA2C_PROGRESS_RE = re.compile(
r'\[#\w+\s+'
r'([\d.]+\w+)/([\d.]+\w+)' # downloaded/total
r'\((\d+)%\)' # percent
r'.*?DL:([\d.]+\w+)' # speed
r'(?:.*?ETA:([^\s\]]+))?' # optional ETA
)
_SIZE_UNITS = [('TiB', 1024 ** 4), ('GiB', 1024 ** 3), ('MiB', 1024 ** 2), ('KiB', 1024), ('B', 1)]
def _parse_size(s: str) -> int:
"""Convert a size string like '4.5MiB' to bytes (int)."""
for suffix, multiplier in _SIZE_UNITS:
if s.endswith(suffix):
return int(float(s[:-len(suffix)]) * multiplier)
# Fallback: try parsing as pure number.
return int(float(s))
def parse_aria2c_progress(line: str) -> dict | None:
"""Parse an aria2c progress line and return a dict, or None if not a progress line."""
m = ARIA2C_PROGRESS_RE.search(line)
if not m:
return None
return dict(
bytes_downloaded=_parse_size(m.group(1)),
total_bytes=_parse_size(m.group(2)),
percent=int(m.group(3)),
speed=_parse_size(m.group(4)),
eta=m.group(5),
)
def set_download_progress(download_id: int, progress: dict):
"""Store download progress in shared_ctx."""
data = dict(api_app.shared_ctx.download_manager_data)
dp = dict(data.get('download_progress', {}))
dp[download_id] = progress
data['download_progress'] = dp
api_app.shared_ctx.download_manager_data.update(data)
def clear_download_progress(download_id: int):
"""Remove download progress from shared_ctx."""
data = dict(api_app.shared_ctx.download_manager_data)
dp = dict(data.get('download_progress', {}))
dp.pop(download_id, None)
data['download_progress'] = dp
api_app.shared_ctx.download_manager_data.update(data)
def make_progress_callback(download_id: int, parse_fn: callable) -> callable:
"""Create a stdout callback that parses progress lines and updates shared_ctx.
The callback throttles updates to at most once per second.
:param download_id: The download ID to update progress for.
:param parse_fn: A function that takes a line string and returns a progress dict or None.
"""
last_update = [0.0]
def on_stdout(line: str):
now_ = time.monotonic()
if now_ - last_update[0] < 1.0:
return
progress = parse_fn(line)
if progress:
last_update[0] = now_
set_download_progress(download_id, progress)
return on_stdout
@perpetual_signal(sleep=1 if PYTEST else 5)
async def perpetual_download_worker():
@ -314,7 +389,8 @@ class Downloader:
raise NotImplementedError()
async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
timeout: int = None, debug: bool = True) -> CommandResult:
timeout: int = None, debug: bool = True,
stdout_callback: callable = None) -> CommandResult:
"""
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
@ -324,7 +400,8 @@ class Downloader:
raise RuntimeError('cwd directory does not exist')
timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug)
coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
stdout_callback=stdout_callback)
result = await self.cancel_wrapper(coro, download)
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
@ -343,23 +420,33 @@ class Downloader:
"""
download_id = download.id
task = asyncio.create_task(coro)
while not task.done():
if download_manager.download_is_killed(download_id) or not download_manager.can_download:
logger.warning(f'Cancel download of {download.url}')
try:
while not task.done():
if download_manager.download_is_killed(download_id) or not download_manager.can_download:
logger.warning(f'Cancel download of {download.url}')
task.cancel()
try:
await task
except asyncio.CancelledError as e:
logger.info(f'Successful cancel of {download.url}', exc_info=e)
return DownloadResult(
success=False,
error='Download was canceled',
)
finally:
download_manager.unkill_download(download_id)
else:
# Wait for the download to complete. Cancel if requested.
await asyncio.sleep(0.1)
except asyncio.CancelledError:
# Outer cancellation (e.g. from a parent cancel_wrapper). Ensure the inner task is also cancelled.
if not task.done():
task.cancel()
try:
await task
except asyncio.CancelledError as e:
logger.info(f'Successful cancel of {download.url}', exc_info=e)
return DownloadResult(
success=False,
error='Download was canceled',
)
finally:
download_manager.unkill_download(download_id)
else:
# Wait for the download to complete. Cancel if requested.
await asyncio.sleep(0.1)
except asyncio.CancelledError:
pass
raise
# Return the result of the download attempt.
return task.result()
@ -424,7 +511,12 @@ class Downloader:
# the URL redirects to a CDN path that doesn't match the original filename.
cmd = (*cmd, '-o', output_path.name)
result = await self.process_runner(download, cmd, destination)
on_stdout = make_progress_callback(download.id, parse_aria2c_progress)
try:
result = await self.process_runner(download, cmd, destination, stdout_callback=on_stdout)
finally:
clear_download_progress(download.id)
error = result.stderr.decode()
if result.return_code != 0:
@ -1034,6 +1126,14 @@ class DownloadManager:
curs.execute(stmt)
pending_once_downloads = curs.fetchone()[0]
try:
progress_data = dict(api_app.shared_ctx.download_manager_data.get('download_progress', {}))
except Exception:
progress_data = {}
for download in once_downloads + recurring_downloads:
if download['id'] in progress_data:
download['progress'] = progress_data[download['id']]
data = dict(
recurring_downloads=recurring_downloads,
once_downloads=once_downloads,
@ -1354,6 +1454,8 @@ async def signal_download_download(download_id: int, download_url: str):
# Remove this domain from the running list.
if download_domain:
download_manager._delete_processing_domain(download_domain)
# Ensure download progress is always cleaned up.
clear_download_progress(download_id)
@dataclass

View file

@ -117,6 +117,7 @@ async def test_get_download_info_uuid_fallback():
assert info.name == 'somefile.tar.gz'
@skip_circleci
async def test_file_downloader_aria2c_uses_output_flag(test_directory):
"""The aria2c command includes -o with the predicted filename when not using metalink."""
file_downloader = FileDownloader()
@ -126,7 +127,7 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
captured_cmd = None
async def fake_process_runner(download, cmd, dest):
async def fake_process_runner(download, cmd, dest, **kwargs):
nonlocal captured_cmd
captured_cmd = cmd
# Create the expected output file so the downloader doesn't raise.
@ -141,7 +142,8 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
fake_info = DownloadFileInfo(name='somefile.tar.gz', size=1024, status=200)
with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \
mock.patch.object(file_downloader, 'process_runner', fake_process_runner):
mock.patch.object(file_downloader, 'process_runner', fake_process_runner), \
mock.patch('wrolpi.downloader.clear_download_progress'):
download = mock.MagicMock()
url = 'https://github.com/user/repo/releases/download/v1.0/somefile.tar.gz'
await file_downloader.download_file(download, url, destination)
@ -152,6 +154,7 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
assert captured_cmd[o_index + 1] == 'somefile.tar.gz'
@skip_circleci
async def test_file_downloader_meta4_no_output_flag(test_directory):
"""The aria2c command should NOT include -o when using a metalink file."""
file_downloader = FileDownloader()
@ -161,7 +164,7 @@ async def test_file_downloader_meta4_no_output_flag(test_directory):
captured_cmd = None
async def fake_process_runner(download, cmd, dest):
async def fake_process_runner(download, cmd, dest, **kwargs):
nonlocal captured_cmd
captured_cmd = cmd
# Create the expected output file.
@ -177,7 +180,8 @@ async def test_file_downloader_meta4_no_output_flag(test_directory):
with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \
mock.patch.object(file_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=b'<metalink/>')), \
mock.patch.object(file_downloader, 'process_runner', fake_process_runner):
mock.patch.object(file_downloader, 'process_runner', fake_process_runner), \
mock.patch('wrolpi.downloader.clear_download_progress'):
download = mock.MagicMock()
url = 'https://example.com/somefile.tar.gz'
await file_downloader.download_file(download, url, destination)

View file

@ -67,3 +67,44 @@ async def test_run_command_cancel(test_directory):
assert result.stdout == b''
assert result.stderr == b''
assert result.elapsed == 1
STREAMING_SCRIPT = '''#!/bin/sh
echo "line one"
echo "line two"
echo "line three"
echo stderr >&2
'''
async def test_run_command_streaming(test_directory):
"""run_command_streaming captures stdout line-by-line and calls callback."""
(test_directory / 'stream.sh').write_text(STREAMING_SCRIPT)
lines = []
result = await cmd.run_command(
('sh', 'stream.sh'),
cwd=test_directory,
stdout_callback=lambda line: lines.append(line),
)
assert result.return_code == 0
assert result.cancelled is False
assert result.stdout == b'line one\nline two\nline three\n'
assert result.stderr == b'stderr\n'
assert lines == ['line one', 'line two', 'line three']
async def test_run_command_streaming_no_callback(test_directory):
"""run_command_streaming works without a callback."""
(test_directory / 'stream.sh').write_text(STREAMING_SCRIPT)
result = await cmd.run_command(('sh', 'stream.sh'), cwd=test_directory)
assert result.return_code == 0
assert result.stdout == b'line one\nline two\nline three\n'
assert result.stderr == b'stderr\n'
async def test_run_command_streaming_timeout(test_directory):
"""Timeout is obeyed for streaming."""
result = await cmd.run_command(('sleep', '10'), timeout=2)
assert result.return_code == -9
assert result.cancelled is False
assert result.elapsed >= 2

View file

@ -15,11 +15,131 @@ from wrolpi.common import get_wrolpi_config
from wrolpi.dates import Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
get_download_manager_config, RSSDownloader
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
set_download_progress, clear_download_progress, make_progress_callback, download_manager
from wrolpi.errors import InvalidDownload, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
def test_parse_size():
"""Size strings are parsed to bytes."""
assert _parse_size('0B') == 0
assert _parse_size('1024B') == 1024
assert _parse_size('1.0KiB') == 1024
assert _parse_size('4.5MiB') == int(4.5 * 1024 ** 2)
assert _parse_size('1.0GiB') == 1024 ** 3
assert _parse_size('2.5TiB') == int(2.5 * 1024 ** 4)
def test_parse_aria2c_progress():
"""aria2c progress lines are parsed correctly."""
line = '[#c52705 1.0MiB/1.0GiB(0%) CN:1 DL:4.5MiB ETA:3m44s]'
result = parse_aria2c_progress(line)
assert result is not None
assert result['bytes_downloaded'] == int(1.0 * 1024 ** 2)
assert result['total_bytes'] == 1024 ** 3
assert result['percent'] == 0
assert result['speed'] == int(4.5 * 1024 ** 2)
assert result['eta'] == '3m44s'
def test_parse_aria2c_progress_no_eta():
"""aria2c progress lines without ETA are parsed."""
line = '[#abc123 500.0MiB/1.0GiB(48%) CN:3 DL:10.0MiB]'
result = parse_aria2c_progress(line)
assert result is not None
assert result['percent'] == 48
assert result['speed'] == int(10.0 * 1024 ** 2)
assert result['eta'] is None
def test_parse_aria2c_progress_not_progress_line():
"""Non-progress lines return None."""
assert parse_aria2c_progress('03/12 10:00:00 [NOTICE] Download complete') is None
assert parse_aria2c_progress('') is None
assert parse_aria2c_progress('some random text') is None
@pytest.mark.asyncio
async def test_set_and_clear_download_progress(async_client):
"""set_download_progress stores progress in shared_ctx and clear_download_progress removes it."""
from wrolpi.api_utils import api_app
# No progress initially.
data = dict(api_app.shared_ctx.download_manager_data)
assert data.get('download_progress', {}) == {}
# Set progress for a download.
progress = dict(bytes_downloaded=1024, total_bytes=2048, percent=50, speed=512, eta='2s')
set_download_progress(42, progress)
data = dict(api_app.shared_ctx.download_manager_data)
assert data['download_progress'][42] == progress
# Clear it.
clear_download_progress(42)
data = dict(api_app.shared_ctx.download_manager_data)
assert 42 not in data.get('download_progress', {})
@pytest.mark.asyncio
async def test_make_progress_callback(async_client):
"""make_progress_callback creates a callback that throttles and updates shared_ctx."""
from wrolpi.api_utils import api_app
def fake_parse(line):
if 'progress' in line:
return dict(percent=50)
return None
callback = make_progress_callback(99, fake_parse)
# Non-matching line does nothing.
callback('some other line')
data = dict(api_app.shared_ctx.download_manager_data)
assert 99 not in data.get('download_progress', {})
# Matching line updates shared_ctx.
with mock.patch('wrolpi.downloader.time') as mock_time:
mock_time.monotonic.return_value = 100.0
callback('progress line')
data = dict(api_app.shared_ctx.download_manager_data)
assert data['download_progress'][99] == dict(percent=50)
# A call within 1 second is throttled (uses a different progress value to verify).
mock_time.monotonic.return_value = 100.5
callback('progress line') # Would update, but throttled.
# Clean up.
clear_download_progress(99)
@pytest.mark.asyncio
async def test_cancel_wrapper_outer_cancellation(test_session, test_download_manager, test_downloader):
"""cancel_wrapper propagates outer CancelledError and cancels the inner task."""
download = test_download_manager.create_download(test_session, 'https://example.com/cancel', test_downloader.name)
test_session.commit()
inner_cancelled = asyncio.Event()
async def slow_coro():
try:
await asyncio.sleep(60)
except asyncio.CancelledError:
inner_cancelled.set()
raise
wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download))
# Give the wrapper time to start polling.
await asyncio.sleep(0.3)
# Cancel the wrapper from outside (simulating parent cancellation).
wrapper_task.cancel()
with pytest.raises(asyncio.CancelledError):
await wrapper_task
# The inner coroutine should have been cancelled too.
assert inner_cancelled.is_set()
@pytest.mark.asyncio
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
"""Once-downloads over a month old should be deleted."""