mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 16:06:19 -05:00
fix: a video download requests one format, not every matching format
The yt-dlp `-f` selector was built by joining every selector of every requested resolution with a comma. In yt-dlp a comma means "download all of these formats"; `/` is the fallback operator. The resolution map is ordered best-to-worst and its entries already use `/` internally, so the comma was never the intent. With the default resolutions this handed yt-dlp ~45 comma-separated selectors and asked it to download every one that matched. A 720p video matched 8 of them (including duplicates, which yt-dlp does not dedupe), so the whole per-download pipeline ran 8 times against the same output path: subtitles downloaded, thumbnail downloaded, info.json written, subtitles converted, thumbnail converted, existing subtitles deleted, repeat. The duplicate work was invisible because each pass overwrites the last. Join with `/` so the resolution list behaves as the preference chain it reads as.
This commit is contained in:
parent
b9a892b203
commit
4b9593580d
2 changed files with 64 additions and 2 deletions
|
|
@ -159,6 +159,16 @@ DEFAULT_POSTER_FORMAT = 'jpg'
|
|||
DEFAULT_CHANNEL_DOWNLOAD_ORDER = 'newest'
|
||||
|
||||
|
||||
def format_selector(video_resolutions: List[str]) -> str:
|
||||
"""Flatten the requested resolutions into a single yt-dlp `-f` format selector.
|
||||
|
||||
The selectors are joined with "/" so yt-dlp downloads only the first one that matches.
|
||||
Joining with "," would instead ask yt-dlp to download *every* matching selector, which
|
||||
downloads (and post-processes) the same video once per match.
|
||||
"""
|
||||
return '/'.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i])
|
||||
|
||||
|
||||
def get_youtube_video_id(url: str) -> Optional[str]:
|
||||
"""Extract YouTube video ID from various URL formats."""
|
||||
parsed = urlparse(url)
|
||||
|
|
@ -933,7 +943,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
|
||||
try:
|
||||
video_resolutions = effective['video_resolutions']
|
||||
video_resolutions_str = ','.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i])
|
||||
video_resolutions_str = format_selector(video_resolutions)
|
||||
video_format = effective['video_format']
|
||||
video_path, entry = prepare_video_filename(url, out_dir, video_resolutions_str, video_format,
|
||||
audio_only=audio_only, audio_format=audio_format)
|
||||
|
|
|
|||
|
|
@ -10,7 +10,7 @@ import pytest
|
|||
from modules.videos.downloader import VideoDownloader, \
|
||||
get_or_create_channel, channel_downloader, video_downloader, preview_filename, \
|
||||
prepare_filename, convert_wrolpi_filename_format, _bot_blocked, _skip_download, \
|
||||
parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url
|
||||
parse_ytdlp_progress, is_youtube_rss_feed_url, is_youtube_url, format_selector, VIDEO_RESOLUTION_MAP
|
||||
from modules.videos.lib import get_videos_downloader_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.conftest import test_directory, await_switches
|
||||
|
|
@ -1119,6 +1119,58 @@ async def test_video_download_uses_effective_settings(test_session, test_directo
|
|||
assert cmd[sleep_idx + 1] == '5.0', f'Expected sleep_requests=5.0, got {cmd[sleep_idx + 1]}'
|
||||
|
||||
|
||||
@pytest.mark.parametrize('video_resolutions', [
|
||||
['1080p', '720p', '480p', 'maximum'], # The default.
|
||||
['480p'],
|
||||
['2160p', 'maximum'],
|
||||
])
|
||||
def test_format_selector_contains_no_comma(video_resolutions):
|
||||
"""`format_selector` joins selectors with "/" so yt-dlp downloads only the first match.
|
||||
|
||||
A "," would tell yt-dlp to download *every* matching selector, which downloads and
|
||||
post-processes the same video once per match.
|
||||
"""
|
||||
selector = format_selector(video_resolutions)
|
||||
|
||||
assert ',' not in selector, f'A comma makes yt-dlp download every matching format: {selector}'
|
||||
# Every selector of every requested resolution is present, in order, as a "/" fallback.
|
||||
expected = [j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i]]
|
||||
assert selector == '/'.join(expected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_download_requests_one_format(test_session, test_directory, mock_video_extract_info,
|
||||
video_download_manager, mock_video_process_runner,
|
||||
image_file, simple_channel, test_videos_downloader_config):
|
||||
"""The `-f` passed to yt-dlp is a fallback chain, so only one format is downloaded."""
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
simple_channel.directory.mkdir(parents=True)
|
||||
|
||||
video_path = simple_channel.directory / 'a video.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
||||
image_file.rename(video_path.with_suffix('.jpg'))
|
||||
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
|
||||
with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
|
||||
mock_video_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
|
||||
video_download_manager.create_download(test_session, url, video_downloader.name)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
mock_video_process_runner.assert_called_once()
|
||||
download, cmd, out_dir = mock_video_process_runner.call_args[0]
|
||||
|
||||
assert '-f' in cmd, 'A video download must request formats'
|
||||
format_arg = cmd[cmd.index('-f') + 1]
|
||||
assert ',' not in format_arg, \
|
||||
f'A comma makes yt-dlp download the video once per matching format: {format_arg}'
|
||||
# The multiple resolutions the config requests are all still offered, as fallbacks.
|
||||
assert format_arg == format_selector(get_videos_downloader_config().video_resolutions)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_live_video_retry_seconds(test_session, test_directory, mock_video_extract_info,
|
||||
video_download_manager, mock_video_process_runner,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue