From 23551d56ac3265e9c6406ec36d18689b1dcb964b Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 14 May 2025 18:58:22 -0600 Subject: [PATCH 01/43] Installing and enabling iperf3 for terminal speed testing of the network. --- debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot | 3 +++ debian-live-config/config/package-lists/wrolpi.list.chroot | 3 ++- pi-gen/stage2/04-wrolpi/01-packages | 3 ++- pi-gen/stage2/04-wrolpi/03-run-chroot.sh | 3 +++ 4 files changed, 10 insertions(+), 2 deletions(-) diff --git a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot index 6895a90d..1b5f5d14 100644 --- a/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot +++ b/debian-live-config/config/hooks/normal/9999-wrolpi.hook.chroot @@ -65,6 +65,9 @@ cp /opt/wrolpi/etc/raspberrypios/motd_first /etc/motd # NetworkManager for the hotspot. systemctl enable NetworkManager +# Enable iperf3 for speed testing. +systemctl enable iperf3 + set +x echo diff --git a/debian-live-config/config/package-lists/wrolpi.list.chroot b/debian-live-config/config/package-lists/wrolpi.list.chroot index 4869ac3c..ef10efa6 100644 --- a/debian-live-config/config/package-lists/wrolpi.list.chroot +++ b/debian-live-config/config/package-lists/wrolpi.list.chroot @@ -32,6 +32,7 @@ gpsd hostapd htop iotop +iperf3 kiwix kiwix-tools libagg-dev @@ -101,4 +102,4 @@ vim vlc wget wpasupplicant -zim-tools \ No newline at end of file +zim-tools diff --git a/pi-gen/stage2/04-wrolpi/01-packages b/pi-gen/stage2/04-wrolpi/01-packages index e3020583..1f96d62c 100644 --- a/pi-gen/stage2/04-wrolpi/01-packages +++ b/pi-gen/stage2/04-wrolpi/01-packages @@ -32,6 +32,7 @@ gpsd gpsd-tools hostapd iotop +iperf3 kiwix kiwix-tools libagg-dev @@ -98,4 +99,4 @@ unzip vim vlc wget -zim-tools \ No newline at end of file +zim-tools diff --git a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh index ea9c8eab..06e6e4c4 100755 --- a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh @@ -58,6 +58,9 @@ cp /opt/wrolpi/etc/raspberrypios/motd_first /etc/motd # NetworkManager for the hotspot. systemctl enable NetworkManager +# Enable iperf3 for speed testing. +systemctl enable iperf3 + set +x echo From 5712f17d8bb05477eadb03744bf4f2bdebc2c215 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 6 Jun 2025 19:02:05 -0600 Subject: [PATCH 02/43] Bumping yt-dlp version for subtitles fix. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index fbcf3f41..61f7ced0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,4 +29,4 @@ tqdm==4.67.1 vininfo[cli]==1.8.0 websockets==13.1 webvtt-py==0.5.1 -yt-dlp==2025.04.30 +yt-dlp==2025.05.22 From 2dd182a10ed16cebf24825eb7f5427656997e4da Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 20 Jul 2025 16:17:43 -0600 Subject: [PATCH 03/43] Adding vertical video formats to video downloader. --- modules/videos/downloader.py | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 042329f3..0cd752e1 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -41,21 +41,27 @@ from .video.lib import download_video_info_json logger = logger.getChild(__name__) VIDEO_RESOLUTION_MAP = { - '360p': ['134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360', - 'hls-175+bestaudio', # 360p avc1 + best audio - ], - '480p': ['135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480', - 'hls-312+bestaudio', # 480p avc1 + best audio - ], + '360p': [ + '134+bestaudio', '230+bestaudio', 'mp4-360p', 'res:360', + 'hls-175+bestaudio', # 360p avc1 + best audio + '396+bestaudio', '243+bestaudio', # Vertical + ], + '480p': [ + '135+bestaudio', '231+bestaudio', '135+139', 'mp4-480p', 'res:480', + 'hls-312+bestaudio', # 480p avc1 + best audio + '788+bestaudio', '780+bestaudio', '397+bestaudio', # Vertical + ], '720p': [ '298+bestaudio', '232+bestaudio', # 720@60 avc1 + best audio '136+bestaudio', '22', '311+bestaudio', 'res:720', 'hls-517+bestaudio', 'hls-637+bestaudio', # 720p avc1 + best audio + '398+bestaudio', '247+bestaudio', # Vertical ], '1080p': [ '299+bestaudio', '312+bestaudio', '270+bestaudio', # 1080@60 avc1 + best audio '137+bestaudio', '614+bestaudio', 'res:1080', 'hls-899+bestaudio', 'hls-1271+bestaudio', # 1080p avc1 + best audio + '399+bestaudio', '616+bestaudio', # Vertical ], '1440p': [ '639+bestaudio', # vp9, best audio, hdr From 1f030daf754faee6f35ce994cc2eda312289d3f7 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 20 Jul 2025 16:26:45 -0600 Subject: [PATCH 04/43] Disabling status worker for all tests. --- wrolpi/conftest.py | 11 ++++++++++- wrolpi/status.py | 22 +++++++++++++++------- 2 files changed, 25 insertions(+), 8 deletions(-) diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index fe57acb0..11a1cb3b 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -9,6 +9,7 @@ import logging import multiprocessing import pathlib import shutil +import sys import tempfile import threading import zipfile @@ -18,7 +19,7 @@ from itertools import zip_longest from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional from typing import Tuple, Set from unittest import mock -from unittest.mock import MagicMock, AsyncMock +from unittest.mock import MagicMock, AsyncMock, patch from uuid import uuid1, uuid4 import pytest @@ -185,6 +186,7 @@ async def async_client(test_directory) -> SanicASGITestClient: try: yield client finally: + api_app.stop() logger.debug('Destroying async_client') @@ -841,3 +843,10 @@ def mock_downloader_download_file(): contents = contents_ yield set_contents + + +@pytest.fixture +def start_status_worker(): + """Enable the status worker for testing.""" + with mock.patch('wrolpi.status.DISABLE_STATUS_WORKER', False): + yield diff --git a/wrolpi/status.py b/wrolpi/status.py index 508f166e..78d36a3a 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -18,6 +18,7 @@ from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.cmd import which, run_command from wrolpi.common import logger, get_warn_once, unique_by_predicate, partition, TRACE_LEVEL from wrolpi.dates import now +from wrolpi.vars import PYTEST try: import psutil @@ -565,11 +566,18 @@ async def get_power_stats() -> PowerStats: return PowerStats() +DISABLE_STATUS_WORKER = bool(PYTEST) # Do not run status worker while testing, unless explicitly needed. + + @perpetual_signal(sleep=5) async def status_worker(count: int = None, sleep_time: int = 5): """A background process which will gather historical data about system statistics.""" shared_status = api_app.shared_ctx.status + if PYTEST and DISABLE_STATUS_WORKER: + # Tests are running, status worker has not been enabled by `start_status_worker` fixture. + return + if count is not None and count <= 0: logger.debug('Status worker ran out of count') return dict(**shared_status) @@ -579,13 +587,13 @@ async def status_worker(count: int = None, sleep_time: int = 5): # Update global `status` dict with stats that are gathered instantly. cpu_stats, load_stats, drives_stats, memory_stats, processes_stats, iostat_stats, power_stats = \ await asyncio.gather( - get_cpu_stats(), - get_load_stats(), - get_drives_stats(), - get_memory_stats(), - get_processes_stats(), - get_iostat_stats(), - get_power_stats(), + asyncio.create_task(get_cpu_stats(), name='get_cpu_stats'), + asyncio.create_task(get_load_stats(), name='get_load_stats'), + asyncio.create_task(get_drives_stats(), name='get_drives_stats'), + asyncio.create_task(get_memory_stats(), name='get_memory_stats'), + asyncio.create_task(get_processes_stats(), name='get_processes_stats'), + asyncio.create_task(get_iostat_stats(), name='get_iostat_stats'), + asyncio.create_task(get_power_stats(), name='get_power_stats'), ) shared_status.update({ 'cpu_stats': cpu_stats.__json__(), From 5820b864808c7be2f0abaa7b46cdd6fd6fa92db0 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 20 Jul 2025 16:37:00 -0600 Subject: [PATCH 05/43] Disabling all perpetual signals during testing, except switch_worker. --- wrolpi/api_utils.py | 9 ++++++++- wrolpi/switches.py | 2 +- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/wrolpi/api_utils.py b/wrolpi/api_utils.py index f6623d8a..78aceec1 100644 --- a/wrolpi/api_utils.py +++ b/wrolpi/api_utils.py @@ -1,6 +1,8 @@ import asyncio import json import logging +import multiprocessing +import sys from asyncio import CancelledError from datetime import datetime, timezone, date from decimal import Decimal @@ -13,6 +15,7 @@ from sanic import response, HTTPResponse, Request, Sanic, SanicException from wrolpi.common import Base, get_media_directory, logger, LOGGING_CONFIG, TRACE_LEVEL from wrolpi.errors import APIError +from wrolpi.vars import PYTEST logger = logger.getChild(__name__) @@ -141,11 +144,15 @@ async def start_perpetual_tasks(app: Sanic): logger.debug('start_perpetual_tasks completed') -def perpetual_signal(event: str = None, sleep: int | float = 1): +def perpetual_signal(event: str = None, sleep: int | float = 1, run_while_testing: bool = False): """Use Sanic signals to continually call the wrapped function. The wrapped function will continually be called, even if it has errors. If the function is long-running, it will only be called again after it has finished.""" def wrapper(func: callable): + if PYTEST and not run_while_testing: + # Do not run perpetual signal worker while testing, unless explicitly required. + return func + # Create a Sanic "signal" for the provided function. event_ = event or f'wrolpi.perpetual.{func.__name__}' diff --git a/wrolpi/switches.py b/wrolpi/switches.py index 908ca4f6..e9d438ff 100644 --- a/wrolpi/switches.py +++ b/wrolpi/switches.py @@ -85,7 +85,7 @@ def register_switch_handler(switch_name: str): DEBUG_LOGGED = False -@perpetual_signal(sleep=0.1) +@perpetual_signal(sleep=0.1, run_while_testing=True) async def switch_worker(): """Watches `api_app.shared_ctx.switches` for new activated switches, handles each one at a time.""" global DEBUG_LOGGED From bda63ab244be210a5aa22b830a1b5ec0c8203c7c Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 20 Jul 2025 16:37:45 -0600 Subject: [PATCH 06/43] Requiring status worker for status test. --- wrolpi/test/test_status.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/wrolpi/test/test_status.py b/wrolpi/test/test_status.py index 4931fa7f..7f21affb 100644 --- a/wrolpi/test/test_status.py +++ b/wrolpi/test/test_status.py @@ -67,7 +67,7 @@ async def test_get_drives_stats(async_client): @skip_circleci @pytest.mark.asyncio -async def test_status_worker(async_client): +async def test_status_worker(async_client, start_status_worker): """Status worker calls itself perpetually, but can be limited during testing.""" from wrolpi.api_utils import api_app assert async_client.sanic_app.shared_ctx.status['cpu_stats'] == dict() From f08790af05f0ec0c52d0c97642e40b8f62cd31b7 Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 20 Jul 2025 16:40:24 -0600 Subject: [PATCH 07/43] Updating test_get_channel_videos_pagination to use channel_factory. --- modules/videos/channel/test/test_api.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py index ec4d206c..2f7aa300 100644 --- a/modules/videos/channel/test/test_api.py +++ b/modules/videos/channel/test/test_api.py @@ -202,7 +202,7 @@ async def test_channel_download_requires_refresh( assert channel.refreshed == expected assert bool(channel.info_json) == expected - await async_client.sanic_app.dispatch('wrolpi.download.') + await async_client.sanic_app.dispatch('wrolpi.download.download') assert_refreshed(False) test_session.commit() @@ -373,16 +373,14 @@ def test_search_videos_channel(test_client, test_session, video_factory): @pytest.mark.asyncio -async def test_get_channel_videos_pagination(test_session, simple_channel, video_factory, assert_video_search): +async def test_get_channel_videos_pagination(test_session, video_factory, assert_video_search, channel_factory): + # Create two channels, one with 50 videos... + channel1, channel2 = channel_factory(), channel_factory() for i in range(50): - video_factory(channel_id=simple_channel.id) - - channel2 = Channel(name='Bar') - test_session.add(channel2) - test_session.flush() - test_session.refresh(channel2) + video_factory(channel_id=channel1.id) + # The other channel only has one video. video_factory(channel_id=channel2.id) - video_factory(title='vid2', channel_id=channel2.id, with_video_file=True) + video_factory(title='vid2', channel_id=channel2.id) test_session.commit() # Get first, second, third, and empty pages of videos. @@ -395,10 +393,11 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video ] last_ids = [] for offset, video_count in tests: - _, response = await assert_video_search(channel_id=simple_channel.id, order_by='published_datetime', + _, response = await assert_video_search(channel_id=channel1.id, order_by='published_datetime', offset=offset, limit=20) - assert len(response.json['file_groups']) == video_count, 'Returned videos does not match' + assert len(response.json['file_groups']) == video_count, \ + f'Returned videos does not match for ({offset}, {video_count})' current_ids = [i['id'] for i in response.json['file_groups']] assert current_ids != last_ids, f'IDs are unchanged current_ids={current_ids}' last_ids = current_ids From 66392909e7fab25783c9e53c3b3b0b446d8396b3 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 22 Jul 2025 09:43:20 -0600 Subject: [PATCH 08/43] Bumping yt-dlp version. --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 61f7ced0..5335ffb7 100644 --- a/requirements.txt +++ b/requirements.txt @@ -29,4 +29,4 @@ tqdm==4.67.1 vininfo[cli]==1.8.0 websockets==13.1 webvtt-py==0.5.1 -yt-dlp==2025.05.22 +yt-dlp==2025.07.21 From 2cca0dc778f769cb318e93f57e055a6f73a60571 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 25 Jul 2025 10:10:32 -0600 Subject: [PATCH 09/43] Skipping archive test on circleci because it parses files differently. --- modules/archive/test/test_lib.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index f6ed8d30..101d74ea 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -276,6 +276,7 @@ def test_get_domains(test_session, archive_factory): assert [i['domain'] for i in get_domains()] == [] +@skip_circleci @pytest.mark.asyncio async def test_new_archive(test_session, fake_now): singlefile, readability, screenshot = make_fake_archive_result() From 3725f0f39d839bb7b65e66c6d8879188397af61e Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 7 Jun 2025 17:36:00 -0600 Subject: [PATCH 10/43] Adding edit button so user can edit downloads, currently supports videos and archives. --- app/src/components/Download.js | 236 +++++++++++++++++++++----- app/src/components/Videos.js | 1 + app/src/components/admin/Downloads.js | 84 ++++++++- app/src/hooks/useForm.js | 83 +++++---- modules/videos/downloader.py | 3 + modules/videos/test/test_api.py | 37 ++++ modules/videos/video/lib.py | 4 + scripts/dev_build.sh | 10 +- wrolpi/downloader.py | 3 + wrolpi/files/lib.py | 1 - wrolpi/test/test_root_api.py | 15 ++ 11 files changed, 396 insertions(+), 81 deletions(-) diff --git a/app/src/components/Download.js b/app/src/components/Download.js index e045fa84..5ce4de7d 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -271,8 +271,13 @@ function VideoDownloadCountLimit({form, name = 'video_count_limit', path = 'sett /> } -export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions'}) { - const [inputProps, inputAttrs] = form.getSelectionProps({name, path, type: 'array'}); +export function VideoResolutionSelectorForm({form, name = 'video_resolutions', path = 'settings.video_resolutions', onChange}) { + const [inputProps, inputAttrs] = form.getSelectionProps({ + name, + path, + type: 'array', + afterChange: onChange + }); return <> } -export function VideosDownloadForm({singleDownload = true, onCancel}) { +export function VideosDownloadForm({singleDownload = true, onCancel, onSuccess: propOnSuccess, download, submitter: propSubmitter, actions}) { const [showMessage, setShowMessage] = React.useState(false); + const [userChangedResolutions, setUserChangedResolutions] = React.useState(false); + const [config, setConfig] = React.useState(null); - // Keep video settings in session to help user start downloads consistently. - const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions); + // Keep video format in session to help user start downloads consistently. const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption); + // Use config video resolutions if available, otherwise use default + const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions; + const defaultFormData = { urls: '', // Textarea, one URL per line. destination: '', @@ -397,11 +406,11 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) { settings: { use_browser_profile: false, video_format: defaultVideoFormat, - video_resolutions: defaultVideoResolutions, + video_resolutions: configResolutions, } } - const submitter = async (formData) => { + const submitter = propSubmitter || (async (formData) => { const downloadData = { destination: formData.destination, downloader: formData.downloader, @@ -410,24 +419,45 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) { urls: formData.urls.split(/r?\n/), } await postDownload(downloadData); - } + }); const onSuccess = () => { setShowMessage(true); - form.reset(); + if (propOnSuccess) { + propOnSuccess(); + } else { + form.reset(); + } } const form = useForm({ submitter, - defaultFormData, + defaultFormData: download ? mergeDeep(defaultFormData, download) : defaultFormData, onSuccess, }); + // Fetch video downloader config when the modal opens React.useEffect(() => { - const {video_resolutions, video_format} = form.formData.settings; - if (video_resolutions && video_resolutions !== defaultVideoResolutions) { - setDefaultVideoResolutions(video_resolutions); - } + const fetchConfig = async () => { + try { + const result = await fetchVideoDownloaderConfig(); + setConfig(result); + + // Update form with video resolutions from config only if user hasn't changed them + // and if no download object was provided (this is a new download, not an edit) + if (result && result.video_resolutions && !userChangedResolutions && !download) { + form.setValue('settings.video_resolutions', result.video_resolutions); + } + } catch (error) { + console.error('Failed to fetch video downloader config:', error); + } + }; + + fetchConfig(); + }, []); + + React.useEffect(() => { + const {video_format} = form.formData.settings; if (video_format && video_format !== defaultVideoFormat) { setDefaultVideoFormat(video_format); } @@ -473,7 +503,10 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) { - + setUserChangedResolutions(true)} + /> @@ -492,7 +525,7 @@ export function VideosDownloadForm({singleDownload = true, onCancel}) { {showMessage && } - + {actions ? actions({onCancel: localOnCancel, form}) : } @@ -511,6 +544,40 @@ export function VideoMinimumDurationForm({form}) { /> } +export function EditVideosDownloadForm({ + download, + onCancel, + onSuccess, + onDelete, + actions = EditDownloadFormButtons, +}) { + const submitter = async (formData) => { + // Create a copy of settings without channel_url and channel_id + const settings = {...formData.settings}; + // These settings are created when downloading the video, we can't submit them again. + delete settings.channel_url; + delete settings.channel_id; + + const downloadData = { + destination: formData.destination, + downloader: formData.downloader, + settings: settings, + tag_names: formData.tag_names, + urls: formData.urls.split(/r?\n/), + } + await putDownload(download.id, downloadData); + } + + return actions({...props, onDelete})} + /> +} + export function VideoMaximumDurationForm({form}) { return { @@ -557,10 +627,12 @@ export function ChannelDownloadForm({ } } - // Keep video settings in session to help user start downloads consistently. - const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions); + // Keep video format in session to help user start downloads consistently. const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption); + // Use config video resolutions if available, otherwise use default + const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions; + const emptyFormData = { destination: '', downloader: Downloaders.VideoChannel, @@ -577,7 +649,7 @@ export function ChannelDownloadForm({ use_browser_profile: false, video_count_limit: null, video_format: defaultVideoFormat, - video_resolutions: defaultVideoResolutions, + video_resolutions: configResolutions, }, sub_downloader: Downloaders.Video, tag_names: [], @@ -591,11 +663,30 @@ export function ChannelDownloadForm({ clearOnSuccess, }); + // Fetch video downloader config when the modal opens React.useEffect(() => { - const {video_resolutions, video_format} = form.formData.settings; - if (video_resolutions && video_resolutions !== defaultVideoResolutions) { - setDefaultVideoResolutions(video_resolutions); - } + const fetchConfig = async () => { + try { + const result = await fetchVideoDownloaderConfig(); + setConfig(result); + setIsConfigLoaded(true); + + // Update form with video resolutions from config only if user hasn't changed them + // and if no download object was provided (this is a new download, not an edit) + if (result && result.video_resolutions && !userChangedResolutions && !download) { + form.setValue('settings.video_resolutions', result.video_resolutions); + } + } catch (error) { + console.error('Failed to fetch video downloader config:', error); + setIsConfigLoaded(true); + } + }; + + fetchConfig(); + }, []); + + React.useEffect(() => { + const {video_format} = form.formData.settings; if (video_format && video_format !== defaultVideoFormat) { setDefaultVideoFormat(video_format); } @@ -656,7 +747,10 @@ export function ChannelDownloadForm({ - + setUserChangedResolutions(true)} + /> @@ -730,17 +824,42 @@ function SuccessfulDownloadSubmitMessage() { } -export function ArchiveDownloadForm({download, onCancel}) { - const [showMessage, setShowMessage] = React.useState(false); - +export function EditArchiveDownloadForm({ + download, + onCancel, + onSuccess, + onDelete, + actions = EditDownloadFormButtons, +}) { const submitter = async (formData) => { + const downloadData = { + downloader: formData.downloader, + tag_names: formData.tag_names, + urls: formData.urls.split(/\r?\n/), + } + await putDownload(download.id, downloadData); + } + + return actions({...props, onDelete})} + /> +} + +export function ArchiveDownloadForm({download, onCancel, onSuccess: propOnSuccess, submitter: propSubmitter, actions}) { + const [showMessage, setShowMessage] = React.useState(false); + + const submitter = propSubmitter || (async (formData) => { const downloadData = { downloader: formData.downloader, tag_names: formData.tag_names, urls: formData.urls.split(/\r?\n/), } await postDownload(downloadData); - }; + }); const emptyFormData = { downloader: Downloaders.Archive, @@ -748,14 +867,28 @@ export function ArchiveDownloadForm({download, onCancel}) { tag_names: [], }; + const onSuccess = () => { + setShowMessage(true); + if (propOnSuccess) { + propOnSuccess(); + } + } + const form = useForm({ submitter, defaultFormData: mergeDeep(emptyFormData, download), emptyFormData, - clearOnSuccess: true, - onSuccess: async () => setShowMessage(true), + clearOnSuccess: !propOnSuccess, + onSuccess, }); + const localOnCancel = (e) => { + if (e) e.preventDefault(); + if (onCancel) { + onCancel(); + } + } + return
Archives

Create a Singlefile Archive for each of the URLs provided below.

@@ -774,7 +907,7 @@ export function ArchiveDownloadForm({download, onCancel}) { {showMessage && } - + {actions ? actions({onCancel: localOnCancel, form}) : } @@ -783,8 +916,10 @@ export function ArchiveDownloadForm({download, onCancel}) { export function RSSDownloadForm({download, submitter, onDelete, onCancel, actions, clearOnSuccess = true}) { const [showMessage, setShowMessage] = React.useState(false); + const [config, setConfig] = React.useState(null); + const [isConfigLoaded, setIsConfigLoaded] = React.useState(false); + const [userChangedResolutions, setUserChangedResolutions] = React.useState(false); - const [defaultVideoResolutions, setDefaultVideoResolutions] = useLocalStorage('video_resolutions', defaultVideoResolutionOptions); const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption); submitter = submitter || (async (formData) => { @@ -800,6 +935,9 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action await postDownload(downloadData); }); + // Use config video resolutions if available, otherwise use default + const configResolutions = config && config.video_resolutions ? config.video_resolutions : defaultVideoResolutionOptions; + const emptyFormData = { destination: null, downloader: Downloaders.RSS, @@ -809,7 +947,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action excluded_urls: null, title_exclude: null, title_include: null, - video_resolutions: defaultVideoResolutions, + video_resolutions: configResolutions, video_format: defaultVideoFormat, }, tag_names: [], @@ -824,11 +962,30 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action onSuccess: async () => setShowMessage(true), }); + // Fetch video downloader config when the modal opens React.useEffect(() => { - const {video_resolutions, video_format} = form.formData.settings; - if (video_resolutions && video_resolutions !== defaultVideoResolutions) { - setDefaultVideoResolutions(video_resolutions); - } + const fetchConfig = async () => { + try { + const result = await fetchVideoDownloaderConfig(); + setConfig(result); + setIsConfigLoaded(true); + + // Update form with video resolutions from config only if user hasn't changed them + // and if no download object was provided (this is a new download, not an edit) + if (result && result.video_resolutions && !userChangedResolutions && !download) { + form.setValue('settings.video_resolutions', result.video_resolutions); + } + } catch (error) { + console.error('Failed to fetch video downloader config:', error); + setIsConfigLoaded(true); + } + }; + + fetchConfig(); + }, []); + + React.useEffect(() => { + const {video_format} = form.formData.settings; if (video_format && video_format !== defaultVideoFormat) { setDefaultVideoFormat(video_format); } @@ -859,7 +1016,10 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action - + setUserChangedResolutions(true)} + /> diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index cd594f3a..c2145e6a 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -93,6 +93,7 @@ function VideosPage() { {value: 'published_datetime', text: 'Published Date', short: 'P.Date'}, {value: 'length', text: 'Length'}, {value: 'size', text: 'Size'}, + {value: 'size_to_duration', text: 'Size to Duration', short: 'S/Duration'}, {value: 'view_count', text: 'Views'}, {value: 'viewed', text: 'Recently Viewed', short: 'R.Viewed'}, {value: 'download_datetime', text: 'Download Date', short: 'D.Date'}, diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 9723eab3..6bee85c4 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -18,7 +18,7 @@ import { WROLModeMessage } from "../Common"; import { - Button as SButton, + Button as SButton, ButtonGroup, Label, Loader, PlaceholderLine, @@ -35,7 +35,9 @@ import { EditChannelDownloadForm, EditRSSDownloadForm, EditScrapeFilesDownloadForm, - EditZimDownloadForm + EditZimDownloadForm, + EditVideosDownloadForm, + EditArchiveDownloadForm } from "../Download"; import {Downloaders} from "../Vars"; @@ -246,6 +248,9 @@ function RecurringDownloadRow({download, fetchDownloads, onDelete}) { class OnceDownloadRow extends React.Component { constructor(props) { super(props); + this.state = { + editModalOpen: false + }; } handleDelete = async () => { @@ -272,8 +277,21 @@ class OnceDownloadRow extends React.Component { } }; + handleEditOpen = () => { + this.setState({ editModalOpen: true }); + }; + + handleEditClose = () => { + this.setState({ editModalOpen: false }); + }; + + handleEditSuccess = async () => { + await this.props.fetchDownloads(); + this.handleEditClose(); + }; + render() { - let {url, last_successful_download, status, location, error} = this.props; + let {url, last_successful_download, status, location, error, downloader, settings, id} = this.props; // Open downloads (/download), or external links in an anchor. const link = location && !location.startsWith('/download') ? @@ -297,6 +315,8 @@ class OnceDownloadRow extends React.Component { } else if (status === 'failed' || status === 'deferred') { buttonCell = ( + + + {(downloader === Downloaders.Video || downloader === Downloaders.Archive) && status !== 'pending' && ( +