mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
Displaying download progress (speed, eta) for file and video downloads.
This commit is contained in:
parent
fa97b25831
commit
4579562be5
16 changed files with 912 additions and 101 deletions
|
|
@ -111,6 +111,17 @@ input:checked + .slider:before {
|
|||
transform: translateX(26px);
|
||||
}
|
||||
|
||||
.inverted-progress-text.ui.indicating.progress .bar > .progress,
|
||||
.inverted-progress-text.ui.progress > .label {
|
||||
color: white !important;
|
||||
}
|
||||
|
||||
/* Light bar colors (yellow, olive) need dark text on the bar — must override inverted-progress-text. */
|
||||
.light-bar-progress-text.ui.indicating.progress .bar > .progress,
|
||||
.light-bar-progress-text.ui.progress .bar > .progress {
|
||||
color: rgba(0, 0, 0, 0.85) !important;
|
||||
}
|
||||
|
||||
div.full-height {
|
||||
overflow: hidden;
|
||||
padding-top: 100%;
|
||||
|
|
|
|||
81
app/src/components/Theme.test.js
Normal file
81
app/src/components/Theme.test.js
Normal file
|
|
@ -0,0 +1,81 @@
|
|||
import React from 'react';
|
||||
import {Progress} from './Theme';
|
||||
import {renderInDarkMode, renderInLightMode} from '../test-utils';
|
||||
|
||||
// All semantic-ui colors.
|
||||
const allColors = ['red', 'orange', 'yellow', 'olive', 'green', 'teal', 'blue', 'violet', 'purple', 'pink', 'brown', 'grey'];
|
||||
|
||||
// Light bar colors that always need dark percent text (light bars in any theme).
|
||||
const lightBarColors = ['yellow', 'olive', 'teal'];
|
||||
// Bar colors that need dark percent text only in dark mode (inverted makes bar lighter).
|
||||
const darkModeLightBarColors = ['grey', 'blue'];
|
||||
|
||||
describe('Progress component theme support', () => {
|
||||
describe('dark mode', () => {
|
||||
it('adds inverted-progress-text class for label readability', () => {
|
||||
const {container} = renderInDarkMode(
|
||||
<Progress percent={39} progress indicating color='violet'>
|
||||
2.1 GB / 5.3 GB
|
||||
</Progress>
|
||||
);
|
||||
|
||||
const progressEl = container.querySelector('.ui.progress');
|
||||
expect(progressEl).toHaveClass('inverted-progress-text');
|
||||
|
||||
const progressLabel = container.querySelector('.bar > .progress');
|
||||
expect(progressLabel).toHaveTextContent('39%');
|
||||
|
||||
const label = container.querySelector('.ui.progress > .label');
|
||||
expect(label).toHaveTextContent('2.1 GB / 5.3 GB');
|
||||
});
|
||||
|
||||
it('does NOT add inverted-progress-text in light mode', () => {
|
||||
const {container} = renderInLightMode(
|
||||
<Progress percent={39} progress indicating color='violet'>
|
||||
2.1 GB / 5.3 GB
|
||||
</Progress>
|
||||
);
|
||||
|
||||
const progressEl = container.querySelector('.ui.progress');
|
||||
expect(progressEl).not.toHaveClass('inverted-progress-text');
|
||||
});
|
||||
});
|
||||
|
||||
describe('percent text contrast per color', () => {
|
||||
allColors.forEach(color => {
|
||||
it(`${color} bar has readable percent text in light mode`, () => {
|
||||
const {container} = renderInLightMode(
|
||||
<Progress percent={50} progress color={color}/>
|
||||
);
|
||||
|
||||
const progressLabel = container.querySelector('.bar > .progress');
|
||||
expect(progressLabel).toBeInTheDocument();
|
||||
expect(progressLabel).toHaveTextContent('50%');
|
||||
|
||||
const progressEl = container.querySelector('.ui.progress');
|
||||
if (lightBarColors.includes(color)) {
|
||||
expect(progressEl).toHaveClass('light-bar-progress-text');
|
||||
} else {
|
||||
expect(progressEl).not.toHaveClass('light-bar-progress-text');
|
||||
}
|
||||
});
|
||||
|
||||
it(`${color} bar has readable percent text in dark mode`, () => {
|
||||
const {container} = renderInDarkMode(
|
||||
<Progress percent={50} progress color={color}/>
|
||||
);
|
||||
|
||||
const progressLabel = container.querySelector('.bar > .progress');
|
||||
expect(progressLabel).toBeInTheDocument();
|
||||
expect(progressLabel).toHaveTextContent('50%');
|
||||
|
||||
const progressEl = container.querySelector('.ui.progress');
|
||||
if (lightBarColors.includes(color) || darkModeLightBarColors.includes(color)) {
|
||||
expect(progressEl).toHaveClass('light-bar-progress-text');
|
||||
} else {
|
||||
expect(progressEl).not.toHaveClass('light-bar-progress-text');
|
||||
}
|
||||
});
|
||||
});
|
||||
});
|
||||
});
|
||||
|
|
@ -313,9 +313,20 @@ export function Popup(props: PopupProps) {
|
|||
return <SPopup {...i} style={{border: '1px solid grey'}} {...props}/>
|
||||
}
|
||||
|
||||
// Bar colors that always need dark percent text (light bars in any theme).
|
||||
const PROGRESS_BAR_COLORS = ['yellow', 'olive', 'teal'];
|
||||
// Bar colors that need dark percent text only in dark mode (inverted makes bar lighter).
|
||||
const DARK_MODE_PROGRESS_BAR_COLORS = ['grey', 'blue'];
|
||||
|
||||
export function Progress(props: ProgressProps) {
|
||||
const {i} = useContext(ThemeContext);
|
||||
return <SProgress {...i} {...props}/>
|
||||
const {i, inverted} = useContext(ThemeContext);
|
||||
const classes = [props.className || ''];
|
||||
if (inverted) classes.push('inverted-progress-text');
|
||||
const color = props.color as string;
|
||||
if (PROGRESS_BAR_COLORS.includes(color) || (inverted && DARK_MODE_PROGRESS_BAR_COLORS.includes(color))) {
|
||||
classes.push('light-bar-progress-text');
|
||||
}
|
||||
return <SProgress {...i} {...props} className={classes.join(' ').trim()}/>
|
||||
}
|
||||
|
||||
export function Segment(props: SegmentProps) {
|
||||
|
|
|
|||
|
|
@ -17,7 +17,10 @@ import {
|
|||
DisableDownloadsToggle,
|
||||
ErrorMessage,
|
||||
formatFrequency,
|
||||
humanBandwidth,
|
||||
humanFileSize,
|
||||
isoDatetimeToElapsedPopup,
|
||||
useLocalStorage,
|
||||
useTitle,
|
||||
WROLModeMessage
|
||||
} from "../Common";
|
||||
|
|
@ -25,7 +28,6 @@ import {
|
|||
Button as SButton,
|
||||
ButtonGroup,
|
||||
Checkbox,
|
||||
Icon,
|
||||
Label,
|
||||
Loader,
|
||||
PlaceholderLine,
|
||||
|
|
@ -36,7 +38,7 @@ import {
|
|||
TableHeaderCell,
|
||||
TableRow
|
||||
} from "semantic-ui-react";
|
||||
import {Button, Header, Modal, Placeholder, Segment, Table} from "../Theme";
|
||||
import {Button, Header, Modal, Placeholder, Progress, Segment, Table} from "../Theme";
|
||||
import {useDownloads} from "../../hooks/customHooks";
|
||||
import {
|
||||
EditArchiveDownloadForm,
|
||||
|
|
@ -49,6 +51,57 @@ import {
|
|||
import {Downloaders} from "../Vars";
|
||||
import {SortableTable} from "../SortableTable";
|
||||
|
||||
function DownloadProgressModal({progress, url}) {
|
||||
const [open, setOpen] = React.useState(false);
|
||||
const [navColor] = useLocalStorage('nav_color', 'violet');
|
||||
|
||||
const speedText = humanBandwidth(progress.speed);
|
||||
const etaText = progress.eta || '...';
|
||||
const buttonLabel = `${speedText} / ${etaText}`;
|
||||
|
||||
return <>
|
||||
<Button size='small' onClick={() => setOpen(true)} style={{marginLeft: '0.5em'}}>
|
||||
{buttonLabel}
|
||||
</Button>
|
||||
<Modal closeIcon open={open} onClose={() => setOpen(false)} size='small'>
|
||||
<Modal.Header>Download Progress</Modal.Header>
|
||||
<Modal.Content>
|
||||
<Progress
|
||||
percent={progress.percent}
|
||||
progress
|
||||
indicating
|
||||
color={navColor}
|
||||
>
|
||||
{humanFileSize(progress.bytes_downloaded)} / {humanFileSize(progress.total_bytes)}
|
||||
</Progress>
|
||||
<Table definition>
|
||||
<TableBody>
|
||||
<TableRow>
|
||||
<TableCell width={4}>Speed</TableCell>
|
||||
<TableCell>{speedText}</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>ETA</TableCell>
|
||||
<TableCell>{progress.eta || 'Calculating...'}</TableCell>
|
||||
</TableRow>
|
||||
<TableRow>
|
||||
<TableCell>Downloaded</TableCell>
|
||||
<TableCell>{humanFileSize(progress.bytes_downloaded)} / {humanFileSize(progress.total_bytes)}</TableCell>
|
||||
</TableRow>
|
||||
{url && <TableRow>
|
||||
<TableCell>URL</TableCell>
|
||||
<TableCell style={{wordBreak: 'break-all'}}>{url}</TableCell>
|
||||
</TableRow>}
|
||||
</TableBody>
|
||||
</Table>
|
||||
</Modal.Content>
|
||||
<Modal.Actions>
|
||||
<Button onClick={() => setOpen(false)}>Close</Button>
|
||||
</Modal.Actions>
|
||||
</Modal>
|
||||
</>;
|
||||
}
|
||||
|
||||
function ClearDownloadsButton({callback, selectedIds, clearSelection}) {
|
||||
const hasSelection = selectedIds && selectedIds.length > 0;
|
||||
|
||||
|
|
@ -271,11 +324,12 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) {
|
|||
<TableCell>{formatFrequency(frequency)}</TableCell>
|
||||
<TableCell>
|
||||
{last_successful_download && isoDatetimeToElapsedPopup(last_successful_download)}
|
||||
{status === 'pending' && <Loader active inline size='tiny'/>}
|
||||
{status === 'pending' && !download.progress && <Loader active inline size='tiny'/>}
|
||||
{download.progress && <DownloadProgressModal progress={download.progress} url={download.url}/>}
|
||||
</TableCell>
|
||||
<TableCell>{next}</TableCell>
|
||||
<TableCell textAlign='right'>
|
||||
{error && errorModal}
|
||||
{error && !download.progress && errorModal}
|
||||
{editButton}
|
||||
{restartButton}
|
||||
</TableCell>
|
||||
|
|
@ -387,7 +441,7 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
|
|||
</TableCell>
|
||||
);
|
||||
}
|
||||
if (error) {
|
||||
if (error && !download.progress) {
|
||||
completedAtCell = (
|
||||
<Modal
|
||||
closeIcon
|
||||
|
|
@ -427,8 +481,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
|
|||
{parentDownloadUrl && (
|
||||
<p style={{marginTop: '1em', color: '#666'}}>
|
||||
From: <a href={parentDownloadUrl} target='_blank' rel='noopener noreferrer'>
|
||||
{parentDownloadUrl}
|
||||
</a>
|
||||
{parentDownloadUrl}
|
||||
</a>
|
||||
</p>
|
||||
)}
|
||||
</Modal.Content>
|
||||
|
|
@ -456,8 +510,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
|
|||
{parentDownloadUrl && (
|
||||
<p style={{marginTop: '1em', color: '#666'}}>
|
||||
From: <a href={parentDownloadUrl} target='_blank' rel='noopener noreferrer'>
|
||||
{parentDownloadUrl}
|
||||
</a>
|
||||
{parentDownloadUrl}
|
||||
</a>
|
||||
</p>
|
||||
)}
|
||||
</Modal.Content>
|
||||
|
|
@ -477,7 +531,8 @@ function OnceDownloadRow({download, fetchDownloads, isSelected, onSelect}) {
|
|||
</TableCell>
|
||||
<TableCell>
|
||||
{completedAtCell}
|
||||
{status === 'pending' ? <Loader active inline size='tiny'/> : null}
|
||||
{status === 'pending' && !download.progress ? <Loader active inline size='tiny'/> : null}
|
||||
{download.progress && <DownloadProgressModal progress={download.progress} url={download.url}/>}
|
||||
</TableCell>
|
||||
{buttonCell}
|
||||
{editModal}
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ import logging
|
|||
import os.path
|
||||
import pathlib
|
||||
import re
|
||||
import time
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from datetime import timedelta
|
||||
|
|
@ -23,7 +24,8 @@ from wrolpi.common import logger, get_media_directory, escape_file_name, resolve
|
|||
trim_file_name, cached_multiprocessing_result, get_absolute_media_path, aiohttp_post
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult, \
|
||||
clear_download_progress, _parse_size, make_progress_callback
|
||||
from wrolpi.errors import UnrecoverableDownloadError, BotBlockedDownloadError
|
||||
from wrolpi.files.lib import glob_shared_stem, split_path_stem_and_suffix
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -43,6 +45,36 @@ from .video.lib import download_video_info_json
|
|||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
YTDLP_PROGRESS_RE = re.compile(
|
||||
r'\[download\]\s+'
|
||||
r'([\d.]+)%\s+' # percent
|
||||
r'of\s+~?\s*([\d.]+\w+)\s+' # total size (with optional ~)
|
||||
r'at\s+([\d.]+\w+/s)\s+' # speed
|
||||
r'ETA\s+(\S+)' # ETA
|
||||
)
|
||||
|
||||
|
||||
def parse_ytdlp_progress(line: str) -> dict | None:
|
||||
"""Parse a yt-dlp progress line (with --newline) and return a dict, or None if not a progress line."""
|
||||
m = YTDLP_PROGRESS_RE.search(line)
|
||||
if not m:
|
||||
return None
|
||||
percent = float(m.group(1))
|
||||
total_bytes = _parse_size(m.group(2))
|
||||
speed_str = m.group(3).rstrip('/s')
|
||||
speed = _parse_size(speed_str)
|
||||
eta = m.group(4)
|
||||
if eta == 'Unknown':
|
||||
eta = None
|
||||
return dict(
|
||||
bytes_downloaded=int(percent / 100.0 * total_bytes),
|
||||
total_bytes=total_bytes,
|
||||
percent=int(percent),
|
||||
speed=speed,
|
||||
eta=eta,
|
||||
)
|
||||
|
||||
|
||||
VIDEO_RESOLUTION_MAP = {
|
||||
'360p': [
|
||||
'134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360',
|
||||
|
|
@ -585,6 +617,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
'--convert-thumbnails', DEFAULT_POSTER_FORMAT,
|
||||
'--merge-output-format', video_format,
|
||||
'--no-cache-dir',
|
||||
'--newline',
|
||||
'--compat-options', 'no-live-chat',
|
||||
# Get top 20 comments, 10 replies per parent.
|
||||
'--write-comments',
|
||||
|
|
@ -605,20 +638,27 @@ class VideoDownloader(Downloader, ABC):
|
|||
if effective['sleep_requests'] > 0:
|
||||
cmd = (*cmd, '--sleep-requests', str(effective['sleep_requests']))
|
||||
|
||||
if cookies_unlocked():
|
||||
# Use encrypted cookies file
|
||||
logger.info(f'Using encrypted cookies for video download: {url}')
|
||||
with cookies_for_download() as cookies_path:
|
||||
cmd = (*cmd, '--cookies', str(cookies_path))
|
||||
if effective['user_agent']:
|
||||
cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
|
||||
on_stdout = make_progress_callback(download.id, parse_ytdlp_progress)
|
||||
|
||||
try:
|
||||
if cookies_unlocked():
|
||||
# Use encrypted cookies file
|
||||
logger.info(f'Using encrypted cookies for video download: {url}')
|
||||
with cookies_for_download() as cookies_path:
|
||||
cmd = (*cmd, '--cookies', str(cookies_path))
|
||||
if effective['user_agent']:
|
||||
cmd = (*cmd, '--add-headers', f'User-Agent:{effective["user_agent"]}')
|
||||
cmd = (*cmd, '-o', video_path, url)
|
||||
result = await self.process_runner(download, cmd, out_dir, debug=True,
|
||||
stdout_callback=on_stdout)
|
||||
else:
|
||||
# No cookies available
|
||||
logger.info(f'No cookies available for video download: {url}')
|
||||
cmd = (*cmd, '-o', video_path, url)
|
||||
result = await self.process_runner(download, cmd, out_dir, debug=True)
|
||||
else:
|
||||
# No cookies available
|
||||
logger.info(f'No cookies available for video download: {url}')
|
||||
cmd = (*cmd, '-o', video_path, url)
|
||||
result = await self.process_runner(download, cmd, out_dir, debug=True)
|
||||
result = await self.process_runner(download, cmd, out_dir, debug=True,
|
||||
stdout_callback=on_stdout)
|
||||
finally:
|
||||
clear_download_progress(download.id)
|
||||
stdout = result.stdout.decode()
|
||||
stderr = result.stderr.decode()
|
||||
|
||||
|
|
|
|||
|
|
@ -955,6 +955,17 @@ class Channel(ModelHelper, Base):
|
|||
self.tag_id = self.tag.id if self.tag else None
|
||||
return self.tag
|
||||
|
||||
@staticmethod
|
||||
def _get_external_file_paths(session, channel_id: int, directory: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Return primary_path for all Videos in this channel whose FileGroup is outside the channel directory."""
|
||||
dir_str = f'{directory}/'
|
||||
file_groups = session.query(FileGroup).join(Video, Video.file_group_id == FileGroup.id).filter(
|
||||
Video.channel_id == channel_id,
|
||||
FileGroup.directory != str(directory),
|
||||
~FileGroup.directory.startswith(dir_str),
|
||||
).all()
|
||||
return [fg.primary_path for fg in file_groups]
|
||||
|
||||
@classmethod
|
||||
def refresh_files(cls, id_: int, send_events: bool = True) -> str:
|
||||
"""Refresh all files within this Channel's directory. Returns job_id for awaiting.
|
||||
|
|
@ -962,10 +973,13 @@ class Channel(ModelHelper, Base):
|
|||
The channel is marked as refreshed after the background task completes post-processing."""
|
||||
# Get this Channel's info for later. Refresh may take a long time.
|
||||
with get_db_session() as session:
|
||||
directory = cls.find_by_id(session, id_).directory
|
||||
channel = cls.find_by_id(session, id_)
|
||||
directory = channel.directory
|
||||
# Find files associated with this channel but outside its directory.
|
||||
external_paths = cls._get_external_file_paths(session, id_, directory)
|
||||
|
||||
# Queue refresh immediately so caller can await the job.
|
||||
job_id = file_worker.queue_refresh([directory])
|
||||
job_id = file_worker.queue_refresh([directory] + external_paths)
|
||||
|
||||
# Background task handles post-processing after job completes.
|
||||
async def _():
|
||||
|
|
|
|||
198
modules/videos/test/test_channel_refresh_external_files.py
Normal file
198
modules/videos/test/test_channel_refresh_external_files.py
Normal file
|
|
@ -0,0 +1,198 @@
|
|||
"""Tests that Channel.refresh_files() and refresh_collection() detect changes to files
|
||||
that are associated with the channel but located outside the channel's directory."""
|
||||
import shutil
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.collections import Collection
|
||||
from wrolpi.collections.lib import refresh_collection
|
||||
from wrolpi.collections.models import CollectionItem
|
||||
from wrolpi.common import await_background_tasks
|
||||
from wrolpi.conftest import await_file_worker
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
class TestChannelRefreshExternalFiles:
|
||||
|
||||
async def _setup_channel_with_external_video(
|
||||
self, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
|
||||
"""Create a channel with an internal and external video, then delete the external video from disk.
|
||||
|
||||
Returns (channel, internal_video_id, external_video_id, external_path)."""
|
||||
channel = channel_factory(name='MyChannel')
|
||||
assert channel.directory.is_dir()
|
||||
|
||||
# Create a video inside the channel directory.
|
||||
internal_path = channel.directory / 'internal_video.mp4'
|
||||
video_file_factory(internal_path)
|
||||
|
||||
# Create a video outside the channel directory.
|
||||
external_dir = test_directory / 'videos' / 'other'
|
||||
external_dir.mkdir(parents=True, exist_ok=True)
|
||||
external_path = external_dir / 'external_video.mp4'
|
||||
video_file_factory(external_path)
|
||||
|
||||
# Refresh to index all files and model videos.
|
||||
await refresh_files()
|
||||
|
||||
# Verify both videos exist.
|
||||
internal_video = test_session.query(Video).join(Video.file_group).filter(
|
||||
FileGroup.primary_path == internal_path,
|
||||
).one()
|
||||
external_video = test_session.query(Video).join(Video.file_group).filter(
|
||||
FileGroup.primary_path == external_path,
|
||||
).one()
|
||||
|
||||
# Assign the external video to the channel.
|
||||
external_video.channel_id = channel.id
|
||||
test_session.commit()
|
||||
|
||||
internal_video_id = internal_video.id
|
||||
external_video_id = external_video.id
|
||||
|
||||
# Verify channel has 2 videos.
|
||||
channel_videos = test_session.query(Video).filter_by(channel_id=channel.id).all()
|
||||
assert len(channel_videos) == 2, f'Expected 2 channel videos, got {len(channel_videos)}'
|
||||
|
||||
# Delete ALL files for the external video by removing the directory.
|
||||
shutil.rmtree(external_dir)
|
||||
assert not external_path.exists()
|
||||
|
||||
return channel, internal_video_id, external_video_id, external_path
|
||||
|
||||
def _assert_external_deleted_internal_kept(self, test_session, internal_video_id, external_video_id, external_path):
|
||||
"""Assert the external video was cleaned up and the internal video remains."""
|
||||
test_session.expire_all()
|
||||
|
||||
external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one_or_none()
|
||||
assert external_fg is None, \
|
||||
'External video FileGroup should have been removed after files were deleted from disk'
|
||||
|
||||
external_vid = test_session.query(Video).filter_by(id=external_video_id).one_or_none()
|
||||
assert external_vid is None, \
|
||||
'External Video record should have been removed after its files were deleted'
|
||||
|
||||
internal_vid = test_session.query(Video).filter_by(id=internal_video_id).one_or_none()
|
||||
assert internal_vid is not None, 'Internal video should still exist'
|
||||
|
||||
async def test_channel_refresh_files_detects_deleted_external_video(
|
||||
self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
|
||||
"""Channel.refresh_files() should detect and remove an external video whose files were deleted."""
|
||||
channel, internal_id, external_id, external_path = await self._setup_channel_with_external_video(
|
||||
test_session, test_directory, video_file_factory, channel_factory, refresh_files)
|
||||
|
||||
Channel.refresh_files(channel.id)
|
||||
await await_file_worker()
|
||||
await await_background_tasks()
|
||||
|
||||
self._assert_external_deleted_internal_kept(test_session, internal_id, external_id, external_path)
|
||||
|
||||
async def test_refresh_collection_detects_deleted_external_video(
|
||||
self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
|
||||
"""refresh_collection() should detect and remove an external video whose files were deleted."""
|
||||
channel, internal_id, external_id, external_path = await self._setup_channel_with_external_video(
|
||||
test_session, test_directory, video_file_factory, channel_factory, refresh_files)
|
||||
|
||||
collection = test_session.query(Collection).filter_by(id=channel.collection_id).one()
|
||||
refresh_collection(collection.id)
|
||||
await await_file_worker()
|
||||
|
||||
self._assert_external_deleted_internal_kept(test_session, internal_id, external_id, external_path)
|
||||
|
||||
async def test_refresh_collection_detects_deleted_external_collection_item(
|
||||
self, async_client, test_session, test_directory, video_file_factory, refresh_files):
|
||||
"""refresh_collection() should detect and remove an external CollectionItem-linked FileGroup
|
||||
whose files were deleted from disk."""
|
||||
# Create a generic collection with a directory.
|
||||
collection_dir = test_directory / 'my_collection'
|
||||
collection_dir.mkdir(parents=True, exist_ok=True)
|
||||
collection = Collection(name='MyCollection', directory=collection_dir, kind='simple')
|
||||
test_session.add(collection)
|
||||
test_session.flush()
|
||||
|
||||
# Create a video inside the collection directory.
|
||||
internal_path = collection_dir / 'internal_video.mp4'
|
||||
video_file_factory(internal_path)
|
||||
|
||||
# Create a video outside the collection directory.
|
||||
external_dir = test_directory / 'elsewhere'
|
||||
external_dir.mkdir(parents=True, exist_ok=True)
|
||||
external_path = external_dir / 'external_video.mp4'
|
||||
video_file_factory(external_path)
|
||||
|
||||
# Refresh to index all files.
|
||||
await refresh_files()
|
||||
|
||||
internal_fg = test_session.query(FileGroup).filter_by(primary_path=internal_path).one()
|
||||
external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one()
|
||||
external_fg_id = external_fg.id
|
||||
|
||||
# Link the external FileGroup to the collection via CollectionItem.
|
||||
item = CollectionItem(collection_id=collection.id, file_group_id=external_fg.id)
|
||||
test_session.add(item)
|
||||
test_session.commit()
|
||||
|
||||
# Delete the external files.
|
||||
shutil.rmtree(external_dir)
|
||||
assert not external_path.exists()
|
||||
|
||||
# Refresh via collection.
|
||||
refresh_collection(collection.id)
|
||||
await await_file_worker()
|
||||
|
||||
test_session.expire_all()
|
||||
|
||||
external_fg = test_session.query(FileGroup).filter_by(id=external_fg_id).one_or_none()
|
||||
assert external_fg is None, \
|
||||
'External CollectionItem FileGroup should have been removed after files were deleted from disk'
|
||||
|
||||
internal_fg = test_session.query(FileGroup).filter_by(primary_path=internal_path).one_or_none()
|
||||
assert internal_fg is not None, 'Internal FileGroup should still exist'
|
||||
|
||||
async def test_channel_refresh_detects_new_file_in_external_file_group(
|
||||
self, async_client, test_session, test_directory, video_file_factory, channel_factory, refresh_files):
|
||||
"""Channel.refresh_files() should detect a new sibling file added to an external video's FileGroup."""
|
||||
channel = channel_factory(name='MyChannel')
|
||||
|
||||
# Create a video outside the channel directory.
|
||||
external_dir = test_directory / 'videos' / 'other'
|
||||
external_dir.mkdir(parents=True, exist_ok=True)
|
||||
external_path = external_dir / 'external_video.mp4'
|
||||
video_file_factory(external_path)
|
||||
|
||||
# Refresh to index the video.
|
||||
await refresh_files()
|
||||
|
||||
external_video = test_session.query(Video).join(Video.file_group).filter(
|
||||
FileGroup.primary_path == external_path,
|
||||
).one()
|
||||
|
||||
# Assign the external video to the channel.
|
||||
external_video.channel_id = channel.id
|
||||
test_session.commit()
|
||||
|
||||
# Record the initial file count.
|
||||
external_fg = external_video.file_group
|
||||
initial_file_names = {f['path'] for f in external_fg.files}
|
||||
assert 'external_video.mp4' in initial_file_names
|
||||
|
||||
# Add a subtitle file next to the video with the same stem.
|
||||
subtitle_path = external_dir / 'external_video.en.vtt'
|
||||
subtitle_path.write_text('WEBVTT\n\n00:00:00.000 --> 00:00:01.000\nHello\n')
|
||||
|
||||
# Refresh the channel's files.
|
||||
Channel.refresh_files(channel.id)
|
||||
await await_file_worker()
|
||||
await await_background_tasks()
|
||||
|
||||
test_session.expire_all()
|
||||
|
||||
# The FileGroup should now include the subtitle file.
|
||||
external_fg = test_session.query(FileGroup).filter_by(primary_path=external_path).one()
|
||||
file_names = {f['path'] for f in external_fg.files}
|
||||
assert 'external_video.en.vtt' in file_names, \
|
||||
f'Subtitle file should have been detected in external FileGroup, got: {file_names}'
|
||||
assert 'external_video.mp4' in file_names, 'Original video file should still be present'
|
||||
assert external_fg.primary_path == external_path, 'Primary path should still be the .mp4'
|
||||
|
|
@ -9,7 +9,8 @@ import pytest
|
|||
|
||||
from modules.videos.downloader import VideoDownloader, \
|
||||
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
|
||||
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download
|
||||
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
|
||||
parse_ytdlp_progress
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.conftest import test_directory, await_switches
|
||||
|
|
@ -915,3 +916,50 @@ async def test_video_download_playlist_appended_extension(test_session, test_dir
|
|||
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_path == video_path_na
|
||||
|
||||
|
||||
def test_parse_ytdlp_progress():
|
||||
"""yt-dlp progress lines are parsed correctly."""
|
||||
line = '[download] 50.0% of ~ 45.67MiB at 2.34MiB/s ETA 00:19'
|
||||
result = parse_ytdlp_progress(line)
|
||||
assert result is not None
|
||||
total = int(45.67 * 1024 ** 2)
|
||||
assert result['total_bytes'] == total
|
||||
assert result['percent'] == 50
|
||||
assert result['bytes_downloaded'] == int(0.5 * total)
|
||||
assert result['speed'] == int(2.34 * 1024 ** 2)
|
||||
assert result['eta'] == '00:19'
|
||||
|
||||
|
||||
def test_parse_ytdlp_progress_no_estimated():
|
||||
"""yt-dlp progress lines without ~ estimated marker are parsed."""
|
||||
line = '[download] 100% of 45.67MiB at 3.45MiB/s ETA 00:00'
|
||||
result = parse_ytdlp_progress(line)
|
||||
assert result is not None
|
||||
assert result['percent'] == 100
|
||||
assert result['total_bytes'] == int(45.67 * 1024 ** 2)
|
||||
assert result['speed'] == int(3.45 * 1024 ** 2)
|
||||
assert result['eta'] == '00:00'
|
||||
|
||||
|
||||
def test_parse_ytdlp_progress_small_percent():
|
||||
"""yt-dlp progress lines with small percentages are parsed."""
|
||||
line = '[download] 0.1% of ~ 45.67MiB at 1.23MiB/s ETA 00:37'
|
||||
result = parse_ytdlp_progress(line)
|
||||
assert result is not None
|
||||
assert result['percent'] == 0
|
||||
assert result['eta'] == '00:37'
|
||||
|
||||
|
||||
def test_parse_ytdlp_progress_completion_line():
|
||||
"""The final yt-dlp line with 'in' instead of 'at/ETA' returns None (no speed/ETA)."""
|
||||
line = '[download] 100% of 45.67MiB in 00:13'
|
||||
result = parse_ytdlp_progress(line)
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_parse_ytdlp_progress_not_progress_line():
|
||||
"""Non-progress lines return None."""
|
||||
assert parse_ytdlp_progress('[info] Downloading video 1 of 1') is None
|
||||
assert parse_ytdlp_progress('') is None
|
||||
assert parse_ytdlp_progress('[download] Destination: video.mp4') is None
|
||||
|
|
|
|||
142
wrolpi/cmd.py
142
wrolpi/cmd.py
|
|
@ -155,10 +155,15 @@ TESTING_RUN_COMMAND_RESULT = None
|
|||
|
||||
|
||||
async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
|
||||
timeout: int = 600, log_command: bool = True) -> CommandResult:
|
||||
timeout: int = 600, log_command: bool = True,
|
||||
stdout_callback: callable = None) -> CommandResult:
|
||||
"""Run a shell command, return the results (stdout/stderr/return code).
|
||||
|
||||
When stdout_callback is provided, stdout is piped and each line is passed to the callback
|
||||
in real-time. Otherwise, stdout is written to a temporary file.
|
||||
|
||||
:param log_command: Enable debug logging.
|
||||
:param stdout_callback: Called with each decoded stdout line (str). May be None.
|
||||
"""
|
||||
if not isinstance(cmd, (list, tuple)):
|
||||
raise RuntimeError('Command must be a list or tuple')
|
||||
|
|
@ -167,61 +172,102 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
|
|||
logger.debug('run_command: returning mock result')
|
||||
return TESTING_RUN_COMMAND_RESULT() # call the mock
|
||||
|
||||
with tempfile.NamedTemporaryFile() as stdout_fh, tempfile.NamedTemporaryFile() as stderr_fh:
|
||||
stdout_file, stderr_file = pathlib.Path(stdout_fh.name), pathlib.Path(stderr_fh.name)
|
||||
streaming = stdout_callback is not None
|
||||
|
||||
with tempfile.NamedTemporaryFile() as stderr_fh:
|
||||
stderr_file = pathlib.Path(stderr_fh.name)
|
||||
cmd = tuple(str(i) for i in cmd)
|
||||
cmd_str = ' '.join(shlex.quote(i) for i in cmd)
|
||||
if log_command:
|
||||
logger.info(f'run_command: running ({timeout=}): {cmd_str}')
|
||||
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(f'run_command: running ({timeout=}): {cmd_str}')
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=stdout_fh, # Use stdout/stderr files to avoid buffer filling up.
|
||||
stderr=stderr_fh,
|
||||
cwd=cwd,
|
||||
)
|
||||
pid = proc.pid
|
||||
|
||||
cancelled = False
|
||||
start = time()
|
||||
stdout_fh = None
|
||||
stdout_file = None
|
||||
reader_task = None
|
||||
stdout_buf = None
|
||||
|
||||
if streaming:
|
||||
stdout_arg = asyncio.subprocess.PIPE
|
||||
stdout_buf = bytearray()
|
||||
else:
|
||||
stdout_fh = tempfile.NamedTemporaryFile()
|
||||
stdout_file = pathlib.Path(stdout_fh.name)
|
||||
stdout_arg = stdout_fh
|
||||
|
||||
try:
|
||||
while True:
|
||||
elapsed = int(time() - start)
|
||||
if timeout and elapsed >= timeout:
|
||||
logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
break
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
*cmd,
|
||||
stdout=stdout_arg,
|
||||
stderr=stderr_fh,
|
||||
cwd=cwd,
|
||||
)
|
||||
pid = proc.pid
|
||||
|
||||
if streaming:
|
||||
async def _read_stdout():
|
||||
while True:
|
||||
line = await proc.stdout.readline()
|
||||
if not line:
|
||||
break
|
||||
stdout_buf.extend(line)
|
||||
try:
|
||||
stdout_callback(line.decode(errors='replace').rstrip('\n'))
|
||||
except Exception as e:
|
||||
logger.warning(f'run_command: stdout_callback error: {e}')
|
||||
|
||||
reader_task = asyncio.create_task(_read_stdout())
|
||||
|
||||
cancelled = False
|
||||
start = time()
|
||||
try:
|
||||
while True:
|
||||
elapsed = int(time() - start)
|
||||
if timeout and elapsed >= timeout:
|
||||
logger.warning(f'run_command: timeout exceeded, killing... {cmd=}')
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
break
|
||||
|
||||
try:
|
||||
await asyncio.wait_for(proc.wait(), timeout=1)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
continue
|
||||
except CancelledError as e:
|
||||
logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
|
||||
cancelled = True
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
|
||||
if reader_task:
|
||||
reader_task.cancel()
|
||||
try:
|
||||
# Wait for the process to finish.
|
||||
await asyncio.wait_for(proc.wait(), timeout=1)
|
||||
break
|
||||
except asyncio.TimeoutError:
|
||||
# Task is not done, keep waiting...
|
||||
continue
|
||||
except CancelledError as e:
|
||||
logger.warning(f'run_command: cancelled, killing... {cmd=}', exc_info=e)
|
||||
cancelled = True
|
||||
proc.kill()
|
||||
await proc.wait()
|
||||
await reader_task
|
||||
except (CancelledError, asyncio.CancelledError):
|
||||
pass
|
||||
|
||||
elapsed = int(time() - start)
|
||||
stdout = stdout_file.read_bytes() or b''
|
||||
stderr = stderr_file.read_bytes() or b''
|
||||
# Logs details of the call, but only if it took a long time or TRACE is enabled.
|
||||
if log_command:
|
||||
logger.debug(
|
||||
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
|
||||
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(
|
||||
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
|
||||
return CommandResult(
|
||||
return_code=proc.returncode,
|
||||
cancelled=cancelled,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
elapsed=elapsed,
|
||||
pid=pid,
|
||||
)
|
||||
elapsed = int(time() - start)
|
||||
if streaming:
|
||||
stdout = bytes(stdout_buf)
|
||||
else:
|
||||
stdout = stdout_file.read_bytes() or b''
|
||||
stderr = stderr_file.read_bytes() or b''
|
||||
if log_command:
|
||||
logger.debug(
|
||||
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
|
||||
elif __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
logger.trace(
|
||||
f'run_command: finished ({elapsed=}s) with stdout={len(stdout)} stderr={len(stderr)}: {cmd_str=}')
|
||||
return CommandResult(
|
||||
return_code=proc.returncode,
|
||||
cancelled=cancelled,
|
||||
stdout=stdout,
|
||||
stderr=stderr,
|
||||
elapsed=elapsed,
|
||||
pid=pid,
|
||||
)
|
||||
finally:
|
||||
if stdout_fh:
|
||||
stdout_fh.close()
|
||||
|
|
|
|||
|
|
@ -273,6 +273,43 @@ def update_collection(
|
|||
return collection
|
||||
|
||||
|
||||
def _get_external_collection_file_paths(session, collection_id: int, directory: pathlib.Path) -> list[pathlib.Path]:
|
||||
"""Return primary_path for FileGroups associated with a collection but outside its directory.
|
||||
|
||||
Checks both CollectionItem relationships and kind-specific relationships (e.g. Channel → Video)."""
|
||||
from wrolpi.files.models import FileGroup
|
||||
from .models import CollectionItem
|
||||
dir_str = f'{directory}/'
|
||||
|
||||
external_paths = []
|
||||
|
||||
# Check CollectionItems.
|
||||
item_fgs = session.query(FileGroup).join(
|
||||
CollectionItem, CollectionItem.file_group_id == FileGroup.id
|
||||
).filter(
|
||||
CollectionItem.collection_id == collection_id,
|
||||
FileGroup.directory != str(directory),
|
||||
~FileGroup.directory.startswith(dir_str),
|
||||
).all()
|
||||
for fg in item_fgs:
|
||||
external_paths.append(fg.primary_path)
|
||||
|
||||
# For channel-kind collections, also check Videos linked via Channel.
|
||||
collection = session.query(Collection).filter_by(id=collection_id).one()
|
||||
if collection.kind == 'channel':
|
||||
from modules.videos.models import Channel
|
||||
channel = session.query(Channel).filter_by(collection_id=collection_id).one_or_none()
|
||||
if channel:
|
||||
channel_paths = Channel._get_external_file_paths(session, channel.id, directory)
|
||||
# Deduplicate against paths already found via CollectionItems.
|
||||
seen = set(external_paths)
|
||||
for p in channel_paths:
|
||||
if p not in seen:
|
||||
external_paths.append(p)
|
||||
|
||||
return external_paths
|
||||
|
||||
|
||||
def refresh_collection(collection_id: int, send_events: bool = True) -> None:
|
||||
"""
|
||||
Refresh all files in a collection's directory.
|
||||
|
|
@ -301,9 +338,11 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
|
|||
)
|
||||
|
||||
directory = collection.directory
|
||||
# Find files associated with this collection but outside its directory.
|
||||
external_paths = _get_external_collection_file_paths(session, collection_id, directory)
|
||||
|
||||
# Refresh files asynchronously
|
||||
file_worker.queue_refresh([directory])
|
||||
file_worker.queue_refresh([directory] + external_paths)
|
||||
|
||||
if send_events:
|
||||
relative_dir = get_relative_to_media_directory(directory)
|
||||
|
|
|
|||
|
|
@ -603,7 +603,7 @@ async def test_execute_batch_reorganization(async_client, test_directory, kind,
|
|||
assert result['collection_count'] >= 0 # May be 0 if no files to move
|
||||
|
||||
|
||||
def test_batch_reorganization_status_tracking(test_session):
|
||||
def test_batch_reorganization_status_tracking(async_client, test_session):
|
||||
"""Status should show overall + per-collection progress."""
|
||||
# Test with a non-existent batch job
|
||||
status = get_batch_reorganization_status('nonexistent-batch-job')
|
||||
|
|
|
|||
|
|
@ -134,6 +134,7 @@ def reset_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.download_manager_data.update(dict(
|
||||
processing_domains=[],
|
||||
killed_downloads=[],
|
||||
download_progress={},
|
||||
))
|
||||
|
||||
# Configs
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ import multiprocessing
|
|||
import os
|
||||
import pathlib
|
||||
import random
|
||||
import re
|
||||
import tempfile
|
||||
import time
|
||||
import traceback
|
||||
|
|
@ -29,6 +30,7 @@ from sqlalchemy.sql import Delete
|
|||
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app, perpetual_signal
|
||||
|
||||
from wrolpi.cmd import which, run_command, CommandResult
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
|
||||
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
|
||||
|
|
@ -46,6 +48,79 @@ logger = logger.getChild(__name__)
|
|||
|
||||
ARIA2C_PATH = which('aria2c', '/usr/bin/aria2c')
|
||||
|
||||
ARIA2C_PROGRESS_RE = re.compile(
|
||||
r'\[#\w+\s+'
|
||||
r'([\d.]+\w+)/([\d.]+\w+)' # downloaded/total
|
||||
r'\((\d+)%\)' # percent
|
||||
r'.*?DL:([\d.]+\w+)' # speed
|
||||
r'(?:.*?ETA:([^\s\]]+))?' # optional ETA
|
||||
)
|
||||
|
||||
_SIZE_UNITS = [('TiB', 1024 ** 4), ('GiB', 1024 ** 3), ('MiB', 1024 ** 2), ('KiB', 1024), ('B', 1)]
|
||||
|
||||
|
||||
def _parse_size(s: str) -> int:
|
||||
"""Convert a size string like '4.5MiB' to bytes (int)."""
|
||||
for suffix, multiplier in _SIZE_UNITS:
|
||||
if s.endswith(suffix):
|
||||
return int(float(s[:-len(suffix)]) * multiplier)
|
||||
# Fallback: try parsing as pure number.
|
||||
return int(float(s))
|
||||
|
||||
|
||||
def parse_aria2c_progress(line: str) -> dict | None:
|
||||
"""Parse an aria2c progress line and return a dict, or None if not a progress line."""
|
||||
m = ARIA2C_PROGRESS_RE.search(line)
|
||||
if not m:
|
||||
return None
|
||||
return dict(
|
||||
bytes_downloaded=_parse_size(m.group(1)),
|
||||
total_bytes=_parse_size(m.group(2)),
|
||||
percent=int(m.group(3)),
|
||||
speed=_parse_size(m.group(4)),
|
||||
eta=m.group(5),
|
||||
)
|
||||
|
||||
|
||||
def set_download_progress(download_id: int, progress: dict):
|
||||
"""Store download progress in shared_ctx."""
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
dp = dict(data.get('download_progress', {}))
|
||||
dp[download_id] = progress
|
||||
data['download_progress'] = dp
|
||||
api_app.shared_ctx.download_manager_data.update(data)
|
||||
|
||||
|
||||
def clear_download_progress(download_id: int):
|
||||
"""Remove download progress from shared_ctx."""
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
dp = dict(data.get('download_progress', {}))
|
||||
dp.pop(download_id, None)
|
||||
data['download_progress'] = dp
|
||||
api_app.shared_ctx.download_manager_data.update(data)
|
||||
|
||||
|
||||
def make_progress_callback(download_id: int, parse_fn: callable) -> callable:
|
||||
"""Create a stdout callback that parses progress lines and updates shared_ctx.
|
||||
|
||||
The callback throttles updates to at most once per second.
|
||||
|
||||
:param download_id: The download ID to update progress for.
|
||||
:param parse_fn: A function that takes a line string and returns a progress dict or None.
|
||||
"""
|
||||
last_update = [0.0]
|
||||
|
||||
def on_stdout(line: str):
|
||||
now_ = time.monotonic()
|
||||
if now_ - last_update[0] < 1.0:
|
||||
return
|
||||
progress = parse_fn(line)
|
||||
if progress:
|
||||
last_update[0] = now_
|
||||
set_download_progress(download_id, progress)
|
||||
|
||||
return on_stdout
|
||||
|
||||
|
||||
@perpetual_signal(sleep=1 if PYTEST else 5)
|
||||
async def perpetual_download_worker():
|
||||
|
|
@ -314,7 +389,8 @@ class Downloader:
|
|||
raise NotImplementedError()
|
||||
|
||||
async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
|
||||
timeout: int = None, debug: bool = True) -> CommandResult:
|
||||
timeout: int = None, debug: bool = True,
|
||||
stdout_callback: callable = None) -> CommandResult:
|
||||
"""
|
||||
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
|
||||
|
||||
|
|
@ -324,7 +400,8 @@ class Downloader:
|
|||
raise RuntimeError('cwd directory does not exist')
|
||||
|
||||
timeout = get_wrolpi_config().download_timeout or timeout or self.timeout
|
||||
coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug)
|
||||
coro = run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
|
||||
stdout_callback=stdout_callback)
|
||||
result = await self.cancel_wrapper(coro, download)
|
||||
|
||||
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
||||
|
|
@ -343,23 +420,33 @@ class Downloader:
|
|||
"""
|
||||
download_id = download.id
|
||||
task = asyncio.create_task(coro)
|
||||
while not task.done():
|
||||
if download_manager.download_is_killed(download_id) or not download_manager.can_download:
|
||||
logger.warning(f'Cancel download of {download.url}')
|
||||
try:
|
||||
while not task.done():
|
||||
if download_manager.download_is_killed(download_id) or not download_manager.can_download:
|
||||
logger.warning(f'Cancel download of {download.url}')
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError as e:
|
||||
logger.info(f'Successful cancel of {download.url}', exc_info=e)
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Download was canceled',
|
||||
)
|
||||
finally:
|
||||
download_manager.unkill_download(download_id)
|
||||
else:
|
||||
# Wait for the download to complete. Cancel if requested.
|
||||
await asyncio.sleep(0.1)
|
||||
except asyncio.CancelledError:
|
||||
# Outer cancellation (e.g. from a parent cancel_wrapper). Ensure the inner task is also cancelled.
|
||||
if not task.done():
|
||||
task.cancel()
|
||||
try:
|
||||
await task
|
||||
except asyncio.CancelledError as e:
|
||||
logger.info(f'Successful cancel of {download.url}', exc_info=e)
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
error='Download was canceled',
|
||||
)
|
||||
finally:
|
||||
download_manager.unkill_download(download_id)
|
||||
else:
|
||||
# Wait for the download to complete. Cancel if requested.
|
||||
await asyncio.sleep(0.1)
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
raise
|
||||
|
||||
# Return the result of the download attempt.
|
||||
return task.result()
|
||||
|
|
@ -424,7 +511,12 @@ class Downloader:
|
|||
# the URL redirects to a CDN path that doesn't match the original filename.
|
||||
cmd = (*cmd, '-o', output_path.name)
|
||||
|
||||
result = await self.process_runner(download, cmd, destination)
|
||||
on_stdout = make_progress_callback(download.id, parse_aria2c_progress)
|
||||
|
||||
try:
|
||||
result = await self.process_runner(download, cmd, destination, stdout_callback=on_stdout)
|
||||
finally:
|
||||
clear_download_progress(download.id)
|
||||
error = result.stderr.decode()
|
||||
|
||||
if result.return_code != 0:
|
||||
|
|
@ -1034,6 +1126,14 @@ class DownloadManager:
|
|||
curs.execute(stmt)
|
||||
pending_once_downloads = curs.fetchone()[0]
|
||||
|
||||
try:
|
||||
progress_data = dict(api_app.shared_ctx.download_manager_data.get('download_progress', {}))
|
||||
except Exception:
|
||||
progress_data = {}
|
||||
for download in once_downloads + recurring_downloads:
|
||||
if download['id'] in progress_data:
|
||||
download['progress'] = progress_data[download['id']]
|
||||
|
||||
data = dict(
|
||||
recurring_downloads=recurring_downloads,
|
||||
once_downloads=once_downloads,
|
||||
|
|
@ -1354,6 +1454,8 @@ async def signal_download_download(download_id: int, download_url: str):
|
|||
# Remove this domain from the running list.
|
||||
if download_domain:
|
||||
download_manager._delete_processing_domain(download_domain)
|
||||
# Ensure download progress is always cleaned up.
|
||||
clear_download_progress(download_id)
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -117,6 +117,7 @@ async def test_get_download_info_uuid_fallback():
|
|||
assert info.name == 'somefile.tar.gz'
|
||||
|
||||
|
||||
@skip_circleci
|
||||
async def test_file_downloader_aria2c_uses_output_flag(test_directory):
|
||||
"""The aria2c command includes -o with the predicted filename when not using metalink."""
|
||||
file_downloader = FileDownloader()
|
||||
|
|
@ -126,7 +127,7 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
|
|||
|
||||
captured_cmd = None
|
||||
|
||||
async def fake_process_runner(download, cmd, dest):
|
||||
async def fake_process_runner(download, cmd, dest, **kwargs):
|
||||
nonlocal captured_cmd
|
||||
captured_cmd = cmd
|
||||
# Create the expected output file so the downloader doesn't raise.
|
||||
|
|
@ -141,7 +142,8 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
|
|||
fake_info = DownloadFileInfo(name='somefile.tar.gz', size=1024, status=200)
|
||||
|
||||
with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \
|
||||
mock.patch.object(file_downloader, 'process_runner', fake_process_runner):
|
||||
mock.patch.object(file_downloader, 'process_runner', fake_process_runner), \
|
||||
mock.patch('wrolpi.downloader.clear_download_progress'):
|
||||
download = mock.MagicMock()
|
||||
url = 'https://github.com/user/repo/releases/download/v1.0/somefile.tar.gz'
|
||||
await file_downloader.download_file(download, url, destination)
|
||||
|
|
@ -152,6 +154,7 @@ async def test_file_downloader_aria2c_uses_output_flag(test_directory):
|
|||
assert captured_cmd[o_index + 1] == 'somefile.tar.gz'
|
||||
|
||||
|
||||
@skip_circleci
|
||||
async def test_file_downloader_meta4_no_output_flag(test_directory):
|
||||
"""The aria2c command should NOT include -o when using a metalink file."""
|
||||
file_downloader = FileDownloader()
|
||||
|
|
@ -161,7 +164,7 @@ async def test_file_downloader_meta4_no_output_flag(test_directory):
|
|||
|
||||
captured_cmd = None
|
||||
|
||||
async def fake_process_runner(download, cmd, dest):
|
||||
async def fake_process_runner(download, cmd, dest, **kwargs):
|
||||
nonlocal captured_cmd
|
||||
captured_cmd = cmd
|
||||
# Create the expected output file.
|
||||
|
|
@ -177,7 +180,8 @@ async def test_file_downloader_meta4_no_output_flag(test_directory):
|
|||
|
||||
with mock.patch('wrolpi.downloader.get_download_info', mock.AsyncMock(return_value=fake_info)), \
|
||||
mock.patch.object(file_downloader, 'get_meta4_contents', mock.AsyncMock(return_value=b'<metalink/>')), \
|
||||
mock.patch.object(file_downloader, 'process_runner', fake_process_runner):
|
||||
mock.patch.object(file_downloader, 'process_runner', fake_process_runner), \
|
||||
mock.patch('wrolpi.downloader.clear_download_progress'):
|
||||
download = mock.MagicMock()
|
||||
url = 'https://example.com/somefile.tar.gz'
|
||||
await file_downloader.download_file(download, url, destination)
|
||||
|
|
|
|||
|
|
@ -67,3 +67,44 @@ async def test_run_command_cancel(test_directory):
|
|||
assert result.stdout == b''
|
||||
assert result.stderr == b''
|
||||
assert result.elapsed == 1
|
||||
|
||||
|
||||
STREAMING_SCRIPT = '''#!/bin/sh
|
||||
echo "line one"
|
||||
echo "line two"
|
||||
echo "line three"
|
||||
echo stderr >&2
|
||||
'''
|
||||
|
||||
|
||||
async def test_run_command_streaming(test_directory):
|
||||
"""run_command_streaming captures stdout line-by-line and calls callback."""
|
||||
(test_directory / 'stream.sh').write_text(STREAMING_SCRIPT)
|
||||
lines = []
|
||||
result = await cmd.run_command(
|
||||
('sh', 'stream.sh'),
|
||||
cwd=test_directory,
|
||||
stdout_callback=lambda line: lines.append(line),
|
||||
)
|
||||
assert result.return_code == 0
|
||||
assert result.cancelled is False
|
||||
assert result.stdout == b'line one\nline two\nline three\n'
|
||||
assert result.stderr == b'stderr\n'
|
||||
assert lines == ['line one', 'line two', 'line three']
|
||||
|
||||
|
||||
async def test_run_command_streaming_no_callback(test_directory):
|
||||
"""run_command_streaming works without a callback."""
|
||||
(test_directory / 'stream.sh').write_text(STREAMING_SCRIPT)
|
||||
result = await cmd.run_command(('sh', 'stream.sh'), cwd=test_directory)
|
||||
assert result.return_code == 0
|
||||
assert result.stdout == b'line one\nline two\nline three\n'
|
||||
assert result.stderr == b'stderr\n'
|
||||
|
||||
|
||||
async def test_run_command_streaming_timeout(test_directory):
|
||||
"""Timeout is obeyed for streaming."""
|
||||
result = await cmd.run_command(('sleep', '10'), timeout=2)
|
||||
assert result.return_code == -9
|
||||
assert result.cancelled is False
|
||||
assert result.elapsed >= 2
|
||||
|
|
|
|||
|
|
@ -15,11 +15,131 @@ from wrolpi.common import get_wrolpi_config
|
|||
from wrolpi.dates import Seconds
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
|
||||
get_download_manager_config, RSSDownloader
|
||||
get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \
|
||||
set_download_progress, clear_download_progress, make_progress_callback, download_manager
|
||||
from wrolpi.errors import InvalidDownload, WROLModeEnabled
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
||||
def test_parse_size():
|
||||
"""Size strings are parsed to bytes."""
|
||||
assert _parse_size('0B') == 0
|
||||
assert _parse_size('1024B') == 1024
|
||||
assert _parse_size('1.0KiB') == 1024
|
||||
assert _parse_size('4.5MiB') == int(4.5 * 1024 ** 2)
|
||||
assert _parse_size('1.0GiB') == 1024 ** 3
|
||||
assert _parse_size('2.5TiB') == int(2.5 * 1024 ** 4)
|
||||
|
||||
|
||||
def test_parse_aria2c_progress():
|
||||
"""aria2c progress lines are parsed correctly."""
|
||||
line = '[#c52705 1.0MiB/1.0GiB(0%) CN:1 DL:4.5MiB ETA:3m44s]'
|
||||
result = parse_aria2c_progress(line)
|
||||
assert result is not None
|
||||
assert result['bytes_downloaded'] == int(1.0 * 1024 ** 2)
|
||||
assert result['total_bytes'] == 1024 ** 3
|
||||
assert result['percent'] == 0
|
||||
assert result['speed'] == int(4.5 * 1024 ** 2)
|
||||
assert result['eta'] == '3m44s'
|
||||
|
||||
|
||||
def test_parse_aria2c_progress_no_eta():
|
||||
"""aria2c progress lines without ETA are parsed."""
|
||||
line = '[#abc123 500.0MiB/1.0GiB(48%) CN:3 DL:10.0MiB]'
|
||||
result = parse_aria2c_progress(line)
|
||||
assert result is not None
|
||||
assert result['percent'] == 48
|
||||
assert result['speed'] == int(10.0 * 1024 ** 2)
|
||||
assert result['eta'] is None
|
||||
|
||||
|
||||
def test_parse_aria2c_progress_not_progress_line():
|
||||
"""Non-progress lines return None."""
|
||||
assert parse_aria2c_progress('03/12 10:00:00 [NOTICE] Download complete') is None
|
||||
assert parse_aria2c_progress('') is None
|
||||
assert parse_aria2c_progress('some random text') is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_and_clear_download_progress(async_client):
|
||||
"""set_download_progress stores progress in shared_ctx and clear_download_progress removes it."""
|
||||
from wrolpi.api_utils import api_app
|
||||
# No progress initially.
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
assert data.get('download_progress', {}) == {}
|
||||
|
||||
# Set progress for a download.
|
||||
progress = dict(bytes_downloaded=1024, total_bytes=2048, percent=50, speed=512, eta='2s')
|
||||
set_download_progress(42, progress)
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
assert data['download_progress'][42] == progress
|
||||
|
||||
# Clear it.
|
||||
clear_download_progress(42)
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
assert 42 not in data.get('download_progress', {})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_make_progress_callback(async_client):
|
||||
"""make_progress_callback creates a callback that throttles and updates shared_ctx."""
|
||||
from wrolpi.api_utils import api_app
|
||||
|
||||
def fake_parse(line):
|
||||
if 'progress' in line:
|
||||
return dict(percent=50)
|
||||
return None
|
||||
|
||||
callback = make_progress_callback(99, fake_parse)
|
||||
|
||||
# Non-matching line does nothing.
|
||||
callback('some other line')
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
assert 99 not in data.get('download_progress', {})
|
||||
|
||||
# Matching line updates shared_ctx.
|
||||
with mock.patch('wrolpi.downloader.time') as mock_time:
|
||||
mock_time.monotonic.return_value = 100.0
|
||||
callback('progress line')
|
||||
data = dict(api_app.shared_ctx.download_manager_data)
|
||||
assert data['download_progress'][99] == dict(percent=50)
|
||||
|
||||
# A call within 1 second is throttled (uses a different progress value to verify).
|
||||
mock_time.monotonic.return_value = 100.5
|
||||
callback('progress line') # Would update, but throttled.
|
||||
|
||||
# Clean up.
|
||||
clear_download_progress(99)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cancel_wrapper_outer_cancellation(test_session, test_download_manager, test_downloader):
|
||||
"""cancel_wrapper propagates outer CancelledError and cancels the inner task."""
|
||||
download = test_download_manager.create_download(test_session, 'https://example.com/cancel', test_downloader.name)
|
||||
test_session.commit()
|
||||
|
||||
inner_cancelled = asyncio.Event()
|
||||
|
||||
async def slow_coro():
|
||||
try:
|
||||
await asyncio.sleep(60)
|
||||
except asyncio.CancelledError:
|
||||
inner_cancelled.set()
|
||||
raise
|
||||
|
||||
wrapper_task = asyncio.create_task(Downloader.cancel_wrapper(slow_coro(), download))
|
||||
# Give the wrapper time to start polling.
|
||||
await asyncio.sleep(0.3)
|
||||
|
||||
# Cancel the wrapper from outside (simulating parent cancellation).
|
||||
wrapper_task.cancel()
|
||||
with pytest.raises(asyncio.CancelledError):
|
||||
await wrapper_task
|
||||
|
||||
# The inner coroutine should have been cancelled too.
|
||||
assert inner_cancelled.is_set()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_old_once_downloads(test_session, test_download_manager, test_downloader):
|
||||
"""Once-downloads over a month old should be deleted."""
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue