diff --git a/app/src/components/CodecSettings.test.js b/app/src/components/CodecSettings.test.js new file mode 100644 index 00000000..2575711f --- /dev/null +++ b/app/src/components/CodecSettings.test.js @@ -0,0 +1,53 @@ +import React from 'react'; +import {screen} from '@testing-library/react'; +import userEvent from '@testing-library/user-event'; +import {CodecSettingsForm} from './Download'; +import {createTestForm, renderUI} from '../test-utils'; + +/** createTestForm implements getCustomProps but not getSelectionProps (used by + * MultiSelectField); the shapes are compatible for these tests. */ +const createCodecForm = (initialData) => { + const form = createTestForm(initialData); + form.getSelectionProps = form.getCustomProps; + return form; +}; + +describe('CodecSettingsForm', () => { + it('renders codec selectors and toggles', () => { + const form = createCodecForm({settings: {video_codecs: ['h264'], audio_codecs: []}}); + + renderUI(); + + expect(screen.getByText('Video Codecs')).toBeInTheDocument(); + expect(screen.getByText('Audio Codecs')).toBeInTheDocument(); + expect(screen.getByText('Transcode to preferred codecs')).toBeInTheDocument(); + expect(screen.getByText('Fail if codecs unavailable')).toBeInTheDocument(); + }); + + it('keeps the strict toggle usable while transcode is enabled', () => { + // Strict still matters with transcode on: a preference with no transcode target + // (e.g. av1) cannot be fixed by transcoding, so the backend honors strict then. + const form = createCodecForm({settings: {transcode: true, strict_codecs: true}}); + + renderUI(); + + const toggles = screen.getAllByTestId('toggle'); + const [transcodeToggle, strictToggle] = toggles.map(i => i.querySelector('input') || i); + expect(transcodeToggle).toBeChecked(); + expect(strictToggle).toBeChecked(); + expect(strictToggle).not.toBeDisabled(); + }); + + it('toggling strict updates the form', async () => { + const form = createCodecForm({settings: {transcode: false, strict_codecs: false}}); + + renderUI(); + + const toggles = screen.getAllByTestId('toggle'); + const [, strictToggle] = toggles.map(i => i.querySelector('input') || i); + expect(strictToggle).not.toBeDisabled(); + + await userEvent.click(strictToggle); + expect(form.formData.settings.strict_codecs).toBe(true); + }); +}); diff --git a/app/src/components/Download.cy.js b/app/src/components/Download.cy.js index cf13faa9..cf61eedc 100644 --- a/app/src/components/Download.cy.js +++ b/app/src/components/Download.cy.js @@ -186,7 +186,10 @@ describe('', () => { "720p", "480p", "maximum" - ] + ], + video_codecs: ["h264"], + audio_codecs: ["aac"], + transcode: true, }, sub_downloader: "video", tag_names: ['Automotive'], @@ -265,6 +268,9 @@ describe('', () => { video_format: 'mp4', minimum_duration: null, maximum_duration: null, + video_codecs: ['h264'], + audio_codecs: ['aac'], + transcode: true, }); }); }); diff --git a/app/src/components/Download.js b/app/src/components/Download.js index 356044cf..f8f4bc5e 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -26,11 +26,13 @@ import { defaultAudioFormatOption, defaultVideoFormatOption, defaultVideoResolutionOptions, + downloadAudioCodecOptions, downloadAudioFormatOptions, Downloaders, downloadFormatOptions, downloadOrderOptions, downloadResolutionOptions, + downloadVideoCodecOptions, extendedFrequencyOptions, frequencyOptions, weeklyOption @@ -434,6 +436,71 @@ export function AudioFormatSelectorForm({form, name = 'audio_format', path = 'se } +export function VideoCodecSelectorForm({form, name = 'video_codecs', path = 'settings.video_codecs'}) { + return
+ + +
+} + +export function AudioCodecSelectorForm({form, name = 'audio_codecs', path = 'settings.audio_codecs'}) { + return
+ + +
+} + +/** Codec preference selectors with the transcode/strict toggles. */ +export function CodecSettingsForm({form, codecsPathPrefix = 'settings.'}) { + const [transcodeProps] = form.getCustomProps({name: 'transcode', path: `${codecsPathPrefix}transcode`}); + const [strictProps] = form.getCustomProps({name: 'strict_codecs', path: `${codecsPathPrefix}strict_codecs`}); + return + + + + + + + + + +} + function VideoDurationLimit({form, name, path, label, helpContent, placeholder, helpPosition}) { return + diff --git a/app/src/components/Vars.js b/app/src/components/Vars.js index 08890d96..02d293c3 100644 --- a/app/src/components/Vars.js +++ b/app/src/components/Vars.js @@ -143,6 +143,21 @@ export const downloadAudioFormatOptions = [ export const defaultAudioFormatOption = 'mp3'; +export const downloadVideoCodecOptions = [ + {key: 'h264', text: 'h264 (avc1)', value: 'h264'}, + {key: 'vp9', text: 'vp9', value: 'vp9'}, + {key: 'av1', text: 'av1', value: 'av1'}, + {key: 'vp8', text: 'vp8', value: 'vp8'}, + {key: 'hevc', text: 'hevc (h265)', value: 'hevc'}, +]; + +export const downloadAudioCodecOptions = [ + {key: 'aac', text: 'aac', value: 'aac'}, + {key: 'opus', text: 'opus', value: 'opus'}, + {key: 'mp3', text: 'mp3', value: 'mp3'}, + {key: 'vorbis', text: 'vorbis', value: 'vorbis'}, +]; + export const Downloaders = { Archive: 'archive', File: 'file', diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 2e0625a8..4f4afdea 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -64,7 +64,7 @@ import {QueryContext} from "../contexts/contexts"; import _ from "lodash"; import {defaultFileOrder, defaultSearchOrder} from "./Vars"; import {InputForm, ToggleForm, useForm} from "../hooks/useForm"; -import {VideoFormatSelectorForm, VideoResolutionSelectorForm} from "./Download"; +import {CodecSettingsForm, VideoFormatSelectorForm, VideoResolutionSelectorForm} from "./Download"; import {BatchReorganizeModal} from "./collections/BatchReorganizeModal"; export function VideoWrapper() { @@ -350,6 +350,10 @@ export function VideosSettingsPage() { download_missing_video_comments: false, user_agent: '', sleep_requests: 0.75, + video_codecs: [], + audio_codecs: [], + transcode: false, + strict_codecs: false, }; const configSubmitter = async () => { @@ -396,6 +400,7 @@ export function VideosSettingsPage() { /> +
. DEFAULT_CAPTION_FORMAT = 'vtt' DEFAULT_POSTER_FORMAT = 'jpg' DEFAULT_CHANNEL_DOWNLOAD_ORDER = 'newest' -def format_selector(video_resolutions: List[str]) -> str: - """Flatten the requested resolutions into a single yt-dlp `-f` format selector. +def format_selector(video_resolutions: List[str], video_codecs: List[str] = None, + audio_codecs: List[str] = None, strict_codecs: bool = False) -> str: + """Flatten the requested resolutions (and codec preferences) into a single yt-dlp `-f` + format selector. The selectors are joined with "/" so yt-dlp downloads only the first one that matches. Joining with "," would instead ask yt-dlp to download *every* matching selector, which downloads (and post-processes) the same video once per match. + + Codec preference is codec-major: a preferred codec at a lower resolution beats a + non-preferred codec at a higher resolution. Unless `strict_codecs`, the legacy + resolution selectors are appended as a fallback so a video is still downloaded when no + preferred codec is available. """ - return '/'.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i]) + legacy = '/'.join(j for i in video_resolutions for j in VIDEO_RESOLUTION_MAP[i]) + if not video_codecs and not audio_codecs: + return legacy + + audio_selectors = [f"bestaudio[acodec~='{AUDIO_CODEC_YT_DLP_PATTERNS[i]}']" for i in audio_codecs or []] + if not strict_codecs or not audio_codecs: + audio_selectors.append('bestaudio') + + video_filters = [f"[vcodec~='{VIDEO_CODEC_YT_DLP_PATTERNS[i]}']" for i in video_codecs] \ + if video_codecs else [''] + selectors = [] + for video_filter in video_filters: + for resolution in video_resolutions: + height = RESOLUTION_HEIGHTS[resolution] + dimensions = [f'[height={height}]', f'[width={height}]'] if height else [''] + for dimension in dimensions: + for audio_selector in audio_selectors: + selectors.append(f'bestvideo{dimension}{video_filter}+{audio_selector}') + if not strict_codecs: + selectors.append(legacy) + return '/'.join(selectors) + + +async def enforce_codecs(video_path: pathlib.Path, video_paths: List[pathlib.Path], effective: dict) \ + -> Tuple[pathlib.Path, List[pathlib.Path]]: + """Verify the downloaded file's codecs against the effective codec preferences. + + The format selector already prefers matching sources, but the legacy fallback selectors (and + lying extractor metadata) can still deliver a non-preferred codec. On mismatch: transcode when + enabled, fail permanently when strict, otherwise keep the file (best-effort preference). + + Runs before the Video record is created and holds no DB session, so ffmpeg may run for a long + time without blocking writers. + """ + video_codecs = effective.get('video_codecs') or [] + audio_codecs = effective.get('audio_codecs') or [] + transcode = effective.get('transcode', False) + strict_codecs = effective.get('strict_codecs', False) + + data, _ = await get_or_create_ffprobe_json(video_path) + video_match, audio_match = codecs_match(data, video_codecs, audio_codecs) + if video_match and audio_match: + return video_path, video_paths + + if transcode: + target_vcodec = None if video_match else get_transcode_target(video_codecs, TRANSCODE_VIDEO_TARGETS) + target_acodec = None if audio_match else get_transcode_target(audio_codecs, TRANSCODE_AUDIO_TARGETS) + # Transcode only when it can fix every mismatched stream: converting just the audio of a + # video whose codec preference has no target would report success while leaving the video + # codec wrong. An unfixable mismatch falls through to strict/keep below. + if (video_match or target_vcodec) and (audio_match or target_acodec): + new_path = await transcode_video_file(video_path, target_vcodec, target_acodec, + container=effective.get('video_format') or 'mp4') + await get_or_create_ffprobe_json(new_path) # Refresh the sidecar for the new streams. + video_paths = [new_path if i == video_path else i for i in video_paths] + return new_path, video_paths + logger.warning(f'Cannot transcode {video_path}: a mismatched preference' + f' (video={video_codecs}, audio={audio_codecs}) has no supported transcode target') + + if strict_codecs: + actual_video = [i.get('codec_name') for i in data.get('streams', []) if i.get('codec_type') == 'video'] + actual_audio = [i.get('codec_name') for i in data.get('streams', []) if i.get('codec_type') == 'audio'] + raise UnrecoverableDownloadError( + f'Downloaded video has codecs (video={actual_video}, audio={actual_audio}) but strict codecs' + f' (video={video_codecs}, audio={audio_codecs}) was required and transcoding cannot produce them') + + logger.warning(f'{video_path} does not match preferred codecs' + f' (video={video_codecs}, audio={audio_codecs}); keeping it as downloaded') + return video_path, video_paths def get_youtube_video_id(url: str) -> Optional[str]: @@ -936,6 +1032,13 @@ class VideoDownloader(Downloader, ABC): effective = get_effective_video_settings(settings) audio_only = effective.get('audio_only', False) audio_format = effective.get('audio_format', 'mp3') + video_codecs = effective.get('video_codecs') or [] + audio_codecs = effective.get('audio_codecs') or [] + transcode = effective.get('transcode', False) + # Strict is suppressed only when transcoding could actually produce every preferred + # codec; a preference like av1-only has no transcode target, so strict must hold. + strict_codecs = effective.get('strict_codecs', False) \ + and not (transcode and transcode_can_satisfy_codecs(video_codecs, audio_codecs)) # download_for_log carries .id (for progress) and .url (for log messages). # Falling back to a stub keeps unit tests viable. @@ -943,7 +1046,8 @@ class VideoDownloader(Downloader, ABC): try: video_resolutions = effective['video_resolutions'] - video_resolutions_str = format_selector(video_resolutions) + video_resolutions_str = format_selector(video_resolutions, video_codecs=video_codecs, + audio_codecs=audio_codecs, strict_codecs=strict_codecs) video_format = effective['video_format'] video_path, entry = prepare_video_filename(url, out_dir, video_resolutions_str, video_format, audio_only=audio_only, audio_format=audio_format) @@ -971,7 +1075,14 @@ class VideoDownloader(Downloader, ABC): '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top', ) if audio_only: - cmd = (*cmd, '-f', 'bestaudio', '-x', '--audio-format', audio_format) + audio_selector = 'bestaudio' + if audio_codecs: + audio_selectors = [f"bestaudio[acodec~='{AUDIO_CODEC_YT_DLP_PATTERNS[i]}']" + for i in audio_codecs] + if not strict_codecs: + audio_selectors.append('bestaudio') + audio_selector = '/'.join(audio_selectors) + cmd = (*cmd, '-f', audio_selector, '-x', '--audio-format', audio_format) else: cmd = (*cmd, '-f', video_resolutions_str, @@ -1022,6 +1133,11 @@ class VideoDownloader(Downloader, ABC): error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {result.return_code}' if _bot_blocked(error): raise BotBlockedDownloadError(f'Bot detection or invalid cookies: {error}') + if strict_codecs and 'Requested format is not available' in error: + # The source will not grow new formats; do not retry. + raise UnrecoverableDownloadError( + f'None of the requested codecs (video={video_codecs}, audio={audio_codecs}) are' + f' available for this video, and transcoding cannot produce them: {url}') return DownloadResult(success=False, error=error, location=location) output_format = audio_output_extension(audio_format) if audio_only else video_format @@ -1061,6 +1177,9 @@ class VideoDownloader(Downloader, ABC): video_paths = self._delete_part_files(video_path, video_paths) video_paths = self.normalize_video_file_names(video_path, video_paths) + if not audio_only and (video_codecs or audio_codecs): + video_path, video_paths = await enforce_codecs(video_path, video_paths, effective) + if is_youtube_url(url): # YouTube auto-captions pin the text to the bottom-left; center them at the source. Runs # before the FileGroup is created, so the first index reads the corrected file. diff --git a/modules/videos/lib.py b/modules/videos/lib.py index c06989e2..0f16648e 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -569,6 +569,10 @@ class VideoDownloaderConfigValidator: sleep_requests: float = 0.75 audio_only: bool = False audio_format: str = 'mp3' + video_codecs: List[str] = field(default_factory=list) + audio_codecs: List[str] = field(default_factory=list) + transcode: bool = False + strict_codecs: bool = False def __post_init__(self): allowed_fields = {i.name for i in dataclasses.fields(VideoDownloaderConfigYtDlpOptionsValidator)} @@ -600,6 +604,10 @@ class VideoDownloaderConfig(ConfigFile): sleep_requests=0.75, audio_only=False, audio_format='mp3', + video_codecs=[], + audio_codecs=[], + transcode=False, + strict_codecs=False, ) validator = VideoDownloaderConfigValidator @@ -715,6 +723,38 @@ class VideoDownloaderConfig(ConfigFile): def audio_format(self, value: str): self.update({**self._config, 'audio_format': value}) + @property + def video_codecs(self) -> List[str]: + return self._config.get('video_codecs', []) + + @video_codecs.setter + def video_codecs(self, value: List[str]): + self.update({**self._config, 'video_codecs': value}) + + @property + def audio_codecs(self) -> List[str]: + return self._config.get('audio_codecs', []) + + @audio_codecs.setter + def audio_codecs(self, value: List[str]): + self.update({**self._config, 'audio_codecs': value}) + + @property + def transcode(self) -> bool: + return self._config.get('transcode', False) + + @transcode.setter + def transcode(self, value: bool): + self.update({**self._config, 'transcode': value}) + + @property + def strict_codecs(self) -> bool: + return self._config.get('strict_codecs', False) + + @strict_codecs.setter + def strict_codecs(self, value: bool): + self.update({**self._config, 'strict_codecs': value}) + def import_config(self, file: pathlib.Path = None, send_events=False): super().import_config(file, send_events) self.successful_import = True @@ -744,6 +784,7 @@ INHERITABLE_VIDEO_SETTINGS = { 'video_resolutions', 'video_format', 'writesubtitles', 'writeautomaticsub', 'writethumbnail', 'writeinfojson', 'yt_dlp_extra_args', 'sleep_requests', 'user_agent', 'continue_dl', 'nooverwrites', 'audio_only', 'audio_format', + 'video_codecs', 'audio_codecs', 'transcode', 'strict_codecs', } # Subset of inheritable settings allowed at channel/RSS level. @@ -767,6 +808,10 @@ def _extract_global_inheritable(config: VideoDownloaderConfig) -> dict: 'nooverwrites': config.nooverwrites, 'audio_only': config.audio_only, 'audio_format': config.audio_format, + 'video_codecs': config.video_codecs, + 'audio_codecs': config.audio_codecs, + 'transcode': config.transcode, + 'strict_codecs': config.strict_codecs, } diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index cc8438ff..31fa25aa 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1027,6 +1027,10 @@ async def test_channel_passes_all_inheritable_settings_to_video(test_session, ch 'writesubtitles': False, 'sleep_requests': 3.0, 'user_agent': 'TestBot/1.0', + 'video_codecs': ['h264'], + 'audio_codecs': ['aac'], + 'transcode': True, + 'strict_codecs': True, } download = Download( @@ -1056,7 +1060,8 @@ async def test_channel_passes_all_inheritable_settings_to_video(test_session, ch result = channel_downloader.finalize_download(test_session, download, executed) # Verify all channel-level inheritable settings are passed through - for key in ('video_resolutions', 'video_format', 'writesubtitles', 'sleep_requests', 'user_agent'): + for key in ('video_resolutions', 'video_format', 'writesubtitles', 'sleep_requests', 'user_agent', + 'video_codecs', 'audio_codecs', 'transcode', 'strict_codecs'): assert key in result.settings, f'{key} missing from DownloadResult.settings' assert result.settings[key] == channel_settings[key], \ f'{key}: expected {channel_settings[key]}, got {result.settings[key]}' @@ -1499,3 +1504,288 @@ def test_prepare_uses_existing_video_location(test_session, video_factory): def test_is_youtube_url(url, expected): """is_youtube_url matches YouTube hosts (and subdomains) but rejects lookalike hosts.""" assert is_youtube_url(url) is expected + + +def test_format_selector_with_codecs(): + """Codec preferences generate codec-major selectors: a preferred codec at a lower resolution + beats a non-preferred codec at a higher resolution. The legacy resolution selectors are + appended as a fallback unless strict.""" + from modules.videos.downloader import VIDEO_CODEC_YT_DLP_PATTERNS, AUDIO_CODEC_YT_DLP_PATTERNS + + h264 = VIDEO_CODEC_YT_DLP_PATTERNS['h264'] + aac = AUDIO_CODEC_YT_DLP_PATTERNS['aac'] + legacy = format_selector(['1080p', '720p']) + + selector = format_selector(['1080p', '720p'], video_codecs=['h264'], audio_codecs=['aac']) + expected = '/'.join([ + f"bestvideo[height=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[height=1080][vcodec~='{h264}']+bestaudio", + f"bestvideo[width=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[width=1080][vcodec~='{h264}']+bestaudio", + f"bestvideo[height=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[height=720][vcodec~='{h264}']+bestaudio", + f"bestvideo[width=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[width=720][vcodec~='{h264}']+bestaudio", + legacy, + ]) + assert selector == expected + assert ',' not in selector + + # Strict drops the any-codec audio fallbacks and the legacy tail. + strict = format_selector(['1080p', '720p'], video_codecs=['h264'], audio_codecs=['aac'], + strict_codecs=True) + expected_strict = '/'.join([ + f"bestvideo[height=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[width=1080][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[height=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + f"bestvideo[width=720][vcodec~='{h264}']+bestaudio[acodec~='{aac}']", + ]) + assert strict == expected_strict + assert ',' not in strict + + # Codec-major: every h264 selector comes before the first vp9 selector. + vp9 = VIDEO_CODEC_YT_DLP_PATTERNS['vp9'] + two = format_selector(['1080p'], video_codecs=['h264', 'vp9'], strict_codecs=True) + assert two.rindex(h264) < two.index(vp9) + assert two.count('bestvideo') == 4 # (height + width) x 2 codecs, one audio selector each. + + # No codecs at all: byte-identical to the legacy selector. + assert format_selector(['1080p', '720p']) == legacy + + # Only audio codecs: video is unfiltered but audio is preferred. + audio_only_pref = format_selector(['720p'], audio_codecs=['opus']) + assert "bestvideo[height=720]+bestaudio[acodec~='^opus']" in audio_only_pref + assert audio_only_pref.endswith(format_selector(['720p'])) + + # 'maximum' has no dimension filter. + maximum = format_selector(['maximum'], video_codecs=['h264'], strict_codecs=True) + assert maximum == f"bestvideo[vcodec~='{h264}']+bestaudio" + + +def test_format_selector_codec_patterns_are_safe(): + """Codec regex patterns are embedded in `-f` bracket filters; a comma would make yt-dlp + download every matching format, and a slash would break the fallback chain.""" + from modules.videos.downloader import VIDEO_CODEC_YT_DLP_PATTERNS, AUDIO_CODEC_YT_DLP_PATTERNS + + for pattern in (*VIDEO_CODEC_YT_DLP_PATTERNS.values(), *AUDIO_CODEC_YT_DLP_PATTERNS.values()): + assert ',' not in pattern + assert '/' not in pattern + + +@pytest.mark.asyncio +async def test_video_download_requests_codec_format(test_session, test_directory, mock_video_extract_info, + video_download_manager, mock_video_process_runner, + image_file, simple_channel, test_videos_downloader_config): + """Codec settings change the `-f` selector; the legacy selectors remain as fallbacks.""" + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + + video_path = simple_channel.directory / 'a video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) + image_file.rename(video_path.with_suffix('.jpg')) + + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + settings = {'video_codecs': ['h264'], 'audio_codecs': ['aac']} + + with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename: + mock_video_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + + video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings) + await video_download_manager.wait_for_all_downloads() + + mock_video_process_runner.assert_called_once() + download, cmd, out_dir = mock_video_process_runner.call_args[0] + + format_arg = cmd[cmd.index('-f') + 1] + resolutions = get_videos_downloader_config().video_resolutions + assert format_arg == format_selector(resolutions, video_codecs=['h264'], audio_codecs=['aac']) + assert "vcodec~=" in format_arg + # The legacy selectors are still offered as a fallback (strict is off). + assert format_arg.endswith(format_selector(resolutions)) + + +@pytest.mark.asyncio +async def test_strict_selector_when_transcode_cannot_satisfy(test_session, test_directory, + mock_video_extract_info, + video_download_manager, + mock_video_process_runner, image_file, + simple_channel, + test_videos_downloader_config): + """Transcode suppresses strict only when it can produce a preferred codec; av1 has no + transcode target, so strict still removes the fallback selectors.""" + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + + video_path = simple_channel.directory / 'a video.mp4' + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path) + image_file.rename(video_path.with_suffix('.jpg')) + + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + settings = {'video_codecs': ['av1'], 'strict_codecs': True, 'transcode': True} + + with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename, \ + mock.patch('modules.videos.downloader.enforce_codecs', new_callable=mock.AsyncMock) as mock_enforce: + mock_video_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + mock_enforce.side_effect = lambda path, paths, effective: (path, paths) + + video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings) + await video_download_manager.wait_for_all_downloads() + + mock_video_process_runner.assert_called_once() + download, cmd, out_dir = mock_video_process_runner.call_args[0] + + format_arg = cmd[cmd.index('-f') + 1] + resolutions = get_videos_downloader_config().video_resolutions + assert format_arg == format_selector(resolutions, video_codecs=['av1'], strict_codecs=True) + # No fallback to the legacy selectors. + assert not format_arg.endswith(format_selector(resolutions)) + + +@pytest.mark.asyncio +async def test_strict_codecs_unavailable_fails_download(test_session, test_directory, mock_video_extract_info, + video_download_manager, mock_video_process_runner, + image_file, simple_channel, + test_videos_downloader_config): + """With strict_codecs, yt-dlp's "Requested format is not available" fails the download + permanently (the source will never grow new formats).""" + from wrolpi.cmd import CommandResult + + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + video_path = simple_channel.directory / 'a video.mp4' + + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + settings = {'video_codecs': ['h264'], 'strict_codecs': True} + + mock_video_process_runner.return_value = CommandResult( + return_code=1, cancelled=False, stdout=b'', + stderr=b'ERROR: [youtube] 31jPEBiAC3c: Requested format is not available.', elapsed=0) + + with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename: + mock_video_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + + video_download_manager.create_download(test_session, url, video_downloader.name, settings=settings) + await video_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.is_failed, f'Download should have failed permanently, got {download.status}' + assert 'transcoding cannot produce them' in download.error + + +@pytest.mark.asyncio +async def test_codecs_unavailable_defers_without_strict(test_session, test_directory, mock_video_extract_info, + video_download_manager, mock_video_process_runner, + image_file, simple_channel, + test_videos_downloader_config): + """The same yt-dlp failure without strict_codecs is deferred (retried), as before.""" + from wrolpi.cmd import CommandResult + + simple_channel.source_id = example_video_json['channel_id'] + simple_channel.directory = test_directory / 'videos/channel name' + simple_channel.directory.mkdir(parents=True) + video_path = simple_channel.directory / 'a video.mp4' + + url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' + + mock_video_process_runner.return_value = CommandResult( + return_code=1, cancelled=False, stdout=b'', + stderr=b'ERROR: [youtube] 31jPEBiAC3c: Requested format is not available.', elapsed=0) + + with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename: + mock_video_extract_info.return_value = example_video_json + mock_prepare_filename.return_value = (video_path, {'id': 'foo'}) + + video_download_manager.create_download(test_session, url, video_downloader.name, + settings={'video_codecs': ['h264']}) + await video_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.is_deferred, f'Download should defer without strict_codecs, got {download.status}' + + +@pytest.mark.asyncio +async def test_enforce_codecs(): + """enforce_codecs transcodes, fails, or keeps the file depending on the effective settings.""" + import pathlib + from modules.videos.downloader import enforce_codecs + + video_path = pathlib.Path('/tmp/video.webm') + poster_path = pathlib.Path('/tmp/video.jpg') + video_paths = [video_path, poster_path] + vp9_ffprobe = {'streams': [ + {'codec_type': 'video', 'codec_name': 'vp9'}, + {'codec_type': 'audio', 'codec_name': 'opus'}, + ]} + + with mock.patch('modules.videos.downloader.get_or_create_ffprobe_json', + new_callable=mock.AsyncMock) as mock_ffprobe, \ + mock.patch('modules.videos.downloader.transcode_video_file', + new_callable=mock.AsyncMock) as mock_transcode: + mock_ffprobe.return_value = (vp9_ffprobe, None) + + # Codecs match: nothing happens. + effective = {'video_codecs': ['vp9'], 'audio_codecs': ['opus']} + result = await enforce_codecs(video_path, video_paths, effective) + assert result == (video_path, video_paths) + mock_transcode.assert_not_called() + + # Mismatch with neither transcode nor strict: file is kept. + effective = {'video_codecs': ['h264']} + result = await enforce_codecs(video_path, video_paths, effective) + assert result == (video_path, video_paths) + mock_transcode.assert_not_called() + + # Mismatch with strict: permanent failure. + effective = {'video_codecs': ['h264'], 'strict_codecs': True} + with pytest.raises(UnrecoverableDownloadError): + await enforce_codecs(video_path, video_paths, effective) + mock_transcode.assert_not_called() + + # Mismatch with transcode: the file is transcoded and the paths updated. + new_path = video_path.with_suffix('.mp4') + mock_transcode.return_value = new_path + effective = {'video_codecs': ['h264'], 'audio_codecs': ['aac'], 'transcode': True, + 'video_format': 'mp4'} + result = await enforce_codecs(video_path, video_paths, effective) + assert result == (new_path, [new_path, poster_path]) + mock_transcode.assert_called_once_with(video_path, 'h264', 'aac', container='mp4') + + # Transcode wins over strict when it can produce the preferred codec. + mock_transcode.reset_mock() + mock_transcode.return_value = new_path + effective = {'video_codecs': ['h264'], 'transcode': True, 'strict_codecs': True, + 'video_format': 'mp4'} + result = await enforce_codecs(video_path, video_paths, effective) + assert result[0] == new_path + mock_transcode.assert_called_once() + + # No supported transcode target: file is kept, no error. + mock_transcode.reset_mock() + effective = {'video_codecs': ['av1'], 'transcode': True} + result = await enforce_codecs(video_path, video_paths, effective) + assert result == (video_path, video_paths) + mock_transcode.assert_not_called() + + # Regression: strict must hold when transcode cannot produce the preferred codec. + effective = {'video_codecs': ['av1'], 'transcode': True, 'strict_codecs': True} + with pytest.raises(UnrecoverableDownloadError): + await enforce_codecs(video_path, video_paths, effective) + mock_transcode.assert_not_called() + + # No partial transcode: an unfixable video mismatch (av1) blocks the audio-only + # conversion; the file is kept as-is (or fails when strict). + effective = {'video_codecs': ['av1'], 'audio_codecs': ['aac'], 'transcode': True} + result = await enforce_codecs(video_path, video_paths, effective) + assert result == (video_path, video_paths) + mock_transcode.assert_not_called() + effective = {'video_codecs': ['av1'], 'audio_codecs': ['aac'], 'transcode': True, + 'strict_codecs': True} + with pytest.raises(UnrecoverableDownloadError): + await enforce_codecs(video_path, video_paths, effective) + mock_transcode.assert_not_called() diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 4206ca1f..27765f41 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -542,6 +542,54 @@ async def test_get_download_defaults_endpoint(async_client, test_directory, test assert 'writesubtitles' in data assert 'continue_dl' in data assert 'nooverwrites' in data + assert 'video_codecs' in data + assert 'audio_codecs' in data + assert 'transcode' in data + assert 'strict_codecs' in data # Should NOT contain global-only settings assert 'file_name_format' not in data assert 'quiet' not in data + + +def test_codec_config_properties(test_directory, test_videos_downloader_config): + """The codec settings default to off and round-trip through the config.""" + config = lib.get_videos_downloader_config() + assert config.video_codecs == [] + assert config.audio_codecs == [] + assert config.transcode is False + assert config.strict_codecs is False + + config.video_codecs = ['h264', 'vp9'] + config.audio_codecs = ['aac'] + config.transcode = True + config.strict_codecs = True + assert config.video_codecs == ['h264', 'vp9'] + assert config.audio_codecs == ['aac'] + assert config.transcode is True + assert config.strict_codecs is True + + effective = lib.get_effective_video_settings() + assert effective['video_codecs'] == ['h264', 'vp9'] + assert effective['transcode'] is True + + # A download-level override wins. + effective = lib.get_effective_video_settings({'video_codecs': ['av1'], 'transcode': False}) + assert effective['video_codecs'] == ['av1'] + assert effective['transcode'] is False + assert effective['audio_codecs'] == ['aac'] + + +def test_download_settings_codec_validation(): + """DownloadSettings accepts only known codec names.""" + from wrolpi.schema import DownloadSettings + from wrolpi.errors import ValidationError + + settings = DownloadSettings(video_codecs=['h264', 'vp9'], audio_codecs=['aac'], + transcode=True, strict_codecs=True) + assert settings.video_codecs == ['h264', 'vp9'] + + with pytest.raises(ValidationError): + DownloadSettings(video_codecs=['divx']) + + with pytest.raises(ValidationError): + DownloadSettings(audio_codecs=['wma']) diff --git a/modules/videos/test/test_transcode.py b/modules/videos/test/test_transcode.py new file mode 100644 index 00000000..e7967e53 --- /dev/null +++ b/modules/videos/test/test_transcode.py @@ -0,0 +1,216 @@ +import json +import pathlib +from unittest import mock + +import pytest + +from modules.videos.transcode import codecs_match, get_stream_codec_names, get_transcode_target, \ + transcode_can_satisfy_codecs, transcode_video_file, TRANSCODE_VIDEO_TARGETS, TRANSCODE_AUDIO_TARGETS +from wrolpi.cmd import CommandResult + + +def test_get_stream_codec_names(): + """Embedded thumbnail streams (mjpeg/png) are not video.""" + data = {'streams': [ + {'codec_type': 'video', 'codec_name': 'vp9'}, + {'codec_type': 'video', 'codec_name': 'mjpeg'}, + {'codec_type': 'audio', 'codec_name': 'opus'}, + ]} + assert get_stream_codec_names(data, 'video') == ['vp9'] + assert get_stream_codec_names(data, 'audio') == ['opus'] + assert get_stream_codec_names({}, 'video') == [] + assert get_stream_codec_names(None, 'video') == [] + + +@pytest.mark.parametrize('video_codecs,audio_codecs,expected', [ + ([], [], (True, True)), # No preference always matches. + (['vp9'], ['opus'], (True, True)), + (['h264'], [], (False, True)), + (['h264', 'vp9'], [], (True, True)), # Any preferred codec matches. + ([], ['aac'], (True, False)), + (['h264'], ['aac'], (False, False)), +]) +def test_codecs_match(video_codecs, audio_codecs, expected): + data = {'streams': [ + {'codec_type': 'video', 'codec_name': 'vp9'}, + {'codec_type': 'audio', 'codec_name': 'opus'}, + ]} + assert codecs_match(data, video_codecs, audio_codecs) == expected + + +def test_codecs_match_missing_stream(): + """A missing stream matches: an audio-only file has no video stream to enforce.""" + data = {'streams': [{'codec_type': 'audio', 'codec_name': 'mp3'}]} + assert codecs_match(data, ['h264'], ['mp3']) == (True, True) + + +def test_get_transcode_target(): + """The first preferred codec which is a supported target is the target.""" + assert get_transcode_target(['h264', 'vp9'], TRANSCODE_VIDEO_TARGETS) == 'h264' + assert get_transcode_target(['vp9', 'h264'], TRANSCODE_VIDEO_TARGETS) == 'h264' + assert get_transcode_target(['vp9', 'av1'], TRANSCODE_VIDEO_TARGETS) is None + assert get_transcode_target([], TRANSCODE_VIDEO_TARGETS) is None + assert get_transcode_target(['opus', 'aac'], TRANSCODE_AUDIO_TARGETS) == 'opus' + + +@pytest.mark.parametrize('video_codecs,audio_codecs,expected', [ + ([], [], True), # Nothing to satisfy. + (['h264'], [], True), + (['av1', 'h264'], [], True), # h264 is a target. + (['av1'], [], False), # No target can produce av1. + (['h264'], ['aac'], True), + (['h264'], ['vorbis'], False), # No target can produce vorbis. + ([], ['aac'], True), + (['av1'], ['aac'], False), # Both lists must be satisfiable. +]) +def test_transcode_can_satisfy_codecs(video_codecs, audio_codecs, expected): + """Strict is only suppressed by transcode when every preference list has a transcode target.""" + assert transcode_can_satisfy_codecs(video_codecs, audio_codecs) is expected + + +@pytest.mark.asyncio +async def test_transcode_video_file(test_directory, async_client): + """A webm is transcoded to an mp4 with the same stem; the original file and its stale + ffprobe sidecar are removed.""" + video_path = test_directory / 'video.webm' + video_path.write_bytes(b'fake video data') + sidecar = test_directory / 'video.ffprobe.json' + sidecar.write_text(json.dumps({'streams': []})) + + async def fake_run_command(cmd, **kwargs): + # ffmpeg writes the temporary output file. + pathlib.Path(cmd[-1]).write_bytes(b'transcoded data') + return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1) + + with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command) as mock_run: + result = await transcode_video_file(video_path, target_vcodec='h264', target_acodec='aac', + container='mp4') + + assert result == test_directory / 'video.mp4' + assert result.is_file() and result.read_bytes() == b'transcoded data' + assert not video_path.exists(), 'The original webm should be deleted' + assert not sidecar.exists(), 'The stale ffprobe sidecar should be deleted' + assert not (test_directory / 'video.transcode.mp4').exists(), 'The temporary file should be renamed' + + cmd = mock_run.call_args[0][0] + assert '-c:v' in cmd and cmd[cmd.index('-c:v') + 1] == 'libx264' + assert '-c:a' in cmd and cmd[cmd.index('-c:a') + 1] == 'aac' + # Attached-picture streams are excluded; a missing audio stream is tolerated. + assert '0:V:0' in cmd and '0:a:0?' in cmd + + +@pytest.mark.asyncio +async def test_transcode_video_file_copies_matching_stream(test_directory, async_client): + """Only the mismatching stream is re-encoded; the other stream is copied.""" + video_path = test_directory / 'video.mp4' + video_path.write_bytes(b'fake video data') + + async def fake_run_command(cmd, **kwargs): + pathlib.Path(cmd[-1]).write_bytes(b'transcoded data') + return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1) + + with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command) as mock_run: + result = await transcode_video_file(video_path, target_vcodec='h264', container='mp4') + + assert result == video_path, 'Same container: the file is replaced in place' + cmd = mock_run.call_args[0][0] + assert cmd[cmd.index('-c:v') + 1] == 'libx264' + assert cmd[cmd.index('-c:a') + 1] == 'copy' + + +@pytest.mark.asyncio +async def test_transcode_video_file_forces_supported_container(test_directory, async_client): + """webm cannot contain h264/aac; the container is forced to mp4.""" + video_path = test_directory / 'video.webm' + video_path.write_bytes(b'fake video data') + + async def fake_run_command(cmd, **kwargs): + pathlib.Path(cmd[-1]).write_bytes(b'transcoded data') + return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1) + + with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command): + result = await transcode_video_file(video_path, target_vcodec='h264', container='webm') + + assert result == test_directory / 'video.mp4' + + +@pytest.mark.asyncio +async def test_transcode_video_file_failure_removes_tmp(test_directory, async_client): + """A failed ffmpeg leaves the original file untouched and no temporary file behind.""" + video_path = test_directory / 'video.webm' + video_path.write_bytes(b'fake video data') + + async def fake_run_command(cmd, **kwargs): + pathlib.Path(cmd[-1]).write_bytes(b'partial data') + return CommandResult(return_code=1, cancelled=False, stdout=b'', stderr=b'boom', elapsed=1) + + with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command): + with pytest.raises(RuntimeError): + await transcode_video_file(video_path, target_vcodec='h264') + + assert video_path.is_file(), 'The original file must survive a failed transcode' + assert not (test_directory / 'video.transcode.mp4').exists() + assert not (test_directory / 'video.mp4').exists() + + +@pytest.mark.asyncio +async def test_transcode_video_file_disk_space_guard(test_directory, async_client): + """Transcoding is refused when the disk is nearly full.""" + video_path = test_directory / 'video.webm' + video_path.write_bytes(b'fake video data' * 1000) + + fake_usage = mock.Mock(free=video_path.stat().st_size) # Less than 1.5x the source size. + with mock.patch('modules.videos.transcode.shutil.disk_usage', return_value=fake_usage), \ + mock.patch('modules.videos.transcode.run_command') as mock_run: + with pytest.raises(RuntimeError, match='free space'): + await transcode_video_file(video_path, target_vcodec='h264') + mock_run.assert_not_called() + + +@pytest.mark.asyncio +async def test_transcode_lock_serializes(test_directory, async_client): + """A transcode waits while `shared_ctx.transcode_lock` is held and releases it when done. + + This exercises the wait/release logic in-process; cross-process sharing of the lock is + provided by Sanic's shared_ctx (created before workers fork) and is not proven here.""" + import asyncio + from wrolpi.api_utils import api_app + + video_path = test_directory / 'video.webm' + video_path.write_bytes(b'fake video data') + + async def fake_run_command(cmd, **kwargs): + pathlib.Path(cmd[-1]).write_bytes(b'transcoded data') + return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1) + + lock = api_app.shared_ctx.transcode_lock + assert lock.acquire(block=False), 'Test could not take the transcode lock' + released = False + with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command) as mock_run: + task = asyncio.create_task(transcode_video_file(video_path, target_vcodec='h264')) + try: + await asyncio.sleep(0.5) + mock_run.assert_not_called() # Still waiting for the lock. + assert not task.done() + + lock.release() + released = True + result = await asyncio.wait_for(task, timeout=10) + finally: + if not released: + lock.release() + if not task.done(): + task.cancel() + + assert result == test_directory / 'video.mp4' + mock_run.assert_called_once() + + # The lock was released after the transcode. + assert lock.acquire(block=False), 'transcode_video_file did not release the lock' + lock.release() + + +@pytest.mark.asyncio +async def test_transcode_video_file_requires_target(): + with pytest.raises(RuntimeError, match='target codec'): + await transcode_video_file(pathlib.Path('/tmp/video.mp4')) diff --git a/modules/videos/transcode.py b/modules/videos/transcode.py new file mode 100644 index 00000000..2a97e8e9 --- /dev/null +++ b/modules/videos/transcode.py @@ -0,0 +1,209 @@ +import asyncio +import os +import pathlib +import shutil +from typing import List, Optional, Tuple + +from wrolpi.captions import FFMPEG_BIN +from wrolpi.cmd import run_command +from wrolpi.common import logger +from wrolpi.vars import DEFAULT_FILE_PERMISSIONS, PYTEST + +logger = logger.getChild(__name__) + +# How often a waiting worker re-checks the machine-wide transcode lock. +TRANSCODE_LOCK_POLL_SECONDS = 0.1 if PYTEST else 10 + +# ffprobe `codec_name` values that satisfy each user-selectable codec preference. +FFPROBE_VIDEO_CODEC_NAMES = { + 'h264': {'h264'}, + 'hevc': {'hevc', 'h265'}, + 'vp9': {'vp9'}, + 'av1': {'av1'}, + 'vp8': {'vp8'}, +} +FFPROBE_AUDIO_CODEC_NAMES = { + 'aac': {'aac'}, + 'opus': {'opus'}, + 'mp3': {'mp3'}, + 'vorbis': {'vorbis'}, +} + +# Codecs which can be a transcode *target*, mapped to their ffmpeg encoder args. VP9/AV1/HEVC are +# omitted because software encoding them is impractically slow on a Raspberry Pi. +TRANSCODE_VIDEO_TARGETS = { + 'h264': ('-c:v', 'libx264', '-preset', 'medium', '-crf', '23'), +} +TRANSCODE_AUDIO_TARGETS = { + 'aac': ('-c:a', 'aac', '-b:a', '192k'), + 'opus': ('-c:a', 'libopus', '-b:a', '128k'), + 'mp3': ('-c:a', 'libmp3lame', '-q:a', '2'), +} + +# Containers that can hold an h264/aac transcode result. webm cannot. +TRANSCODE_CONTAINERS = ('mp4', 'mkv') + +TRANSCODE_TIMEOUT = 4 * 60 * 60 # A long video on a Raspberry Pi can take hours. + +# Transcoding writes a whole new copy of the video next to the original. +MINIMUM_FREE_SPACE_RATIO = 1.5 + + +def get_stream_codec_names(ffprobe_data: dict, codec_type: str) -> List[str]: + """Return the codec_name of every stream of the given type ('video' or 'audio'). + + Embedded thumbnails appear as mjpeg/png video streams; they are not real video.""" + streams = (ffprobe_data or {}).get('streams') or [] + names = [s.get('codec_name') for s in streams if s.get('codec_type') == codec_type] + if codec_type == 'video': + names = [i for i in names if i not in ('mjpeg', 'png')] + return [i for i in names if i] + + +def codecs_match(ffprobe_data: dict, video_codecs: List[str], audio_codecs: List[str]) \ + -> Tuple[bool, bool]: + """Return (video matches, audio matches) for the codec preferences. + + An empty preference list always matches. A missing stream also matches (an audio-only file + has no video stream to enforce).""" + video_match = True + if video_codecs: + names = get_stream_codec_names(ffprobe_data, 'video') + if names: + acceptable = set().union(*(FFPROBE_VIDEO_CODEC_NAMES.get(i, {i}) for i in video_codecs)) + video_match = any(i in acceptable for i in names) + audio_match = True + if audio_codecs: + names = get_stream_codec_names(ffprobe_data, 'audio') + if names: + acceptable = set().union(*(FFPROBE_AUDIO_CODEC_NAMES.get(i, {i}) for i in audio_codecs)) + audio_match = any(i in acceptable for i in names) + return video_match, audio_match + + +def get_transcode_target(preferences: List[str], targets: dict) -> Optional[str]: + """The first preferred codec that is a supported transcode target, or None.""" + for codec in preferences or []: + if codec in targets: + return codec + return None + + +def transcode_can_satisfy_codecs(video_codecs: List[str], audio_codecs: List[str]) -> bool: + """True only if transcoding could produce a preferred codec for every non-empty preference + list. When False, enabling transcode cannot guarantee the preferences (e.g. av1-only), so + strict_codecs must still be honored.""" + return ((not video_codecs or get_transcode_target(video_codecs, TRANSCODE_VIDEO_TARGETS) is not None) + and (not audio_codecs or get_transcode_target(audio_codecs, TRANSCODE_AUDIO_TARGETS) is not None)) + + +async def _acquire_transcode_lock(): + """Block (asynchronously) until this process holds the machine-wide transcode lock. + + Polled with sleep instead of a blocking acquire so a multi-hour ffmpeg hold elsewhere does + not stall this worker's event loop.""" + from wrolpi.api_utils import api_app + lock = api_app.shared_ctx.transcode_lock + if not lock.acquire(block=False): + logger.info('Waiting for another transcode to finish') + while not lock.acquire(block=False): + await asyncio.sleep(TRANSCODE_LOCK_POLL_SECONDS) + return lock + + +async def transcode_video_file(video_path: pathlib.Path, + target_vcodec: Optional[str] = None, + target_acodec: Optional[str] = None, + container: str = 'mp4') -> pathlib.Path: + """Transcode a video file in place (same stem, possibly a new container extension). + + Only one transcode runs at a time machine-wide (across all Sanic workers); this call waits + for its turn. Only the stream(s) with a target are re-encoded; the other stream is copied. + The output is written to a temporary file in the same directory, then atomically renamed over + the final path. The original file is deleted if the extension changed. Returns the final + path. + + @raise RuntimeError: when ffmpeg fails, or there is not enough free disk space. + """ + if not FFMPEG_BIN: + raise RuntimeError('ffmpeg was not found') + if not target_vcodec and not target_acodec: + raise RuntimeError('Refusing to transcode without a target codec') + if container not in TRANSCODE_CONTAINERS: + logger.info(f'Forcing mp4 container for transcode of {video_path} ({container} is not supported)') + container = 'mp4' + + lock = await _acquire_transcode_lock() + try: + return await _transcode_video_file(video_path, target_vcodec, target_acodec, container) + finally: + lock.release() + + +async def _transcode_video_file(video_path: pathlib.Path, target_vcodec: Optional[str], + target_acodec: Optional[str], container: str) -> pathlib.Path: + """The ffmpeg work of `transcode_video_file`; the caller holds the transcode lock.""" + # Notify here, after the lock: the wait for another transcode can last hours, and the user + # should hear "Transcoding" only when this file's work actually begins. + try: + from wrolpi.events import Events + Events.send_user_notify(f'Transcoding {video_path.name}') + except Exception: + # Events are best-effort; never let them break a download. + logger.debug(f'Failed to send transcode event for {video_path}', exc_info=True) + + source_size = video_path.stat().st_size + free = shutil.disk_usage(video_path.parent).free + if free < MINIMUM_FREE_SPACE_RATIO * source_size: + raise RuntimeError( + f'Not enough free space to transcode {video_path} ({free} bytes free, source is {source_size} bytes)') + + video_args = TRANSCODE_VIDEO_TARGETS[target_vcodec] if target_vcodec else ('-c:v', 'copy') + audio_args = TRANSCODE_AUDIO_TARGETS[target_acodec] if target_acodec else ('-c:a', 'copy') + + final_path = video_path.with_suffix(f'.{container}') + tmp_path = video_path.with_suffix(f'.transcode.{container}') + # 0:V:0 excludes attached-picture streams (embedded thumbnails); 0:a:0? tolerates a video + # with no audio stream. + cmd = (FFMPEG_BIN, '-y', + '-i', video_path, + '-map', '0:V:0', '-map', '0:a:0?', + *video_args, + *audio_args, + '-movflags', '+faststart', + str(tmp_path)) + logger.warning(f'Transcoding {video_path} to {final_path} (video={target_vcodec}, audio={target_acodec})') + try: + result = await run_command(cmd, cwd=video_path.parent, timeout=TRANSCODE_TIMEOUT) + if result.return_code != 0: + raise RuntimeError( + f'ffmpeg exited with {result.return_code} while transcoding {video_path}:' + f'\n{result.stderr.decode()[-2000:]}') + except Exception: + tmp_path.unlink(missing_ok=True) + raise + + # fsync before the rename: a Pi losing power after the rename must not be left with a + # truncated final file. + fd = os.open(tmp_path, os.O_RDONLY) + try: + os.fsync(fd) + finally: + os.close(fd) + + # The old ffprobe sidecar describes the old streams. + video_path.with_suffix('.ffprobe.json').unlink(missing_ok=True) + tmp_path.rename(final_path) + if final_path != video_path: + video_path.unlink() + final_path.chmod(DEFAULT_FILE_PERMISSIONS) + + # fsync the directory so the rename itself is durable. + fd = os.open(video_path.parent, os.O_RDONLY) + try: + os.fsync(fd) + except OSError: + pass # Some filesystems refuse directory fsync; the rename is still atomic. + finally: + os.close(fd) + return final_path diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index 8d65b821..ac80ad10 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -68,6 +68,12 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.perpetual_tasks_heartbeat = multiprocessing.Value(ctypes.c_double, 0.0) app.shared_ctx.perpetual_tasks_lock = multiprocessing.Lock() + # Transcode + # Transcoding saturates every core; only one ffmpeg transcode may run machine-wide, + # across all Sanic workers. If the holding worker dies (SIGKILL/crash) the lock stays + # held until the API restarts; that is accepted, transcoding is not critical work. + app.shared_ctx.transcode_lock = multiprocessing.Lock() + # Switches app.shared_ctx.switches = manager.dict() app.shared_ctx.switches_lock = multiprocessing.Lock() diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 6894d6dc..533ce38b 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -323,6 +323,10 @@ class DownloadSettings: nooverwrites: Optional[bool] = None audio_only: Optional[bool] = None audio_format: Optional[str] = None + video_codecs: Optional[List[str]] = None + audio_codecs: Optional[List[str]] = None + transcode: Optional[bool] = None + strict_codecs: Optional[bool] = None def __post_init__(self): if self.excluded_urls and self.excluded_urls.endswith(','): @@ -360,6 +364,14 @@ class DownloadSettings: if not all(i in valid_resolutions for i in self.video_resolutions): raise ValidationError(f'Download order must be one of 360p, 480p, 720p, 1080p, 1440p, 2160p, or maximum.') + valid_video_codecs = ('h264', 'vp9', 'av1', 'vp8', 'hevc') + if not all(i in valid_video_codecs for i in self.video_codecs or []): + raise ValidationError(f'Video codecs must be of: {", ".join(valid_video_codecs)}') + + valid_audio_codecs = ('aac', 'opus', 'mp3', 'vorbis') + if not all(i in valid_audio_codecs for i in self.audio_codecs or []): + raise ValidationError(f'Audio codecs must be of: {", ".join(valid_audio_codecs)}') + @dataclass class DownloadRequest: