From 387ef6dfe81af671037b30de029f89d32c37efee Mon Sep 17 00:00:00 2001 From: Roland Date: Sun, 8 Feb 2026 12:29:25 -0700 Subject: [PATCH] Saving video ffprobe json when video file is uploaded. --- modules/videos/common.py | 44 ++++++++++++++++++++++++++ modules/videos/lib.py | 28 +++++++++++++++-- modules/videos/test/test_lib.py | 12 ++++++-- wrolpi/files/api.py | 2 -- wrolpi/files/test/test_api.py | 53 +++++++++++++++++++++++++++++++- wrolpi/files/test/test_lib.py | 8 +++-- wrolpi/files/test/test_models.py | 6 +++- wrolpi/test/test_tags.py | 5 ++- 8 files changed, 146 insertions(+), 12 deletions(-) diff --git a/modules/videos/common.py b/modules/videos/common.py index 31532af0..15c16144 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -244,6 +244,50 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict: return content +def ffprobe_json_sync(video_path: Union[Path, str]) -> dict: + """Synchronous version of ffprobe_json for use in sync validation code. + + Extract video file metadata using ffprobe. + + >>> ffprobe_json_sync('video.mp4') + { + 'chapters': [], + 'format': {}, + 'streams': [ + {}, + ], + } + """ + if not FFPROBE_BIN: + raise RuntimeError('ffprobe was not found') + + cmd = [FFPROBE_BIN, + '-print_format', 'json', + '-loglevel', 'quiet', + '-show_streams', + '-show_format', + '-show_chapters', + str(Path(video_path).absolute())] + + proc = subprocess.run(cmd, capture_output=True, timeout=60) + + if proc.returncode != 0: + raise RuntimeError(f'Got non-zero exit code: {proc.returncode}') + + try: + content = json.loads(proc.stdout.decode().strip()) + except Exception as e: + logger.debug(proc.stdout.decode()) + logger.error(f'Failed to load ffprobe json', exc_info=e) + raise + + if content == dict(): + # Data is empty, video may be corrupt. + raise RuntimeError('ffprobe data was empty') + + return content + + def read_ffprobe_json_file(path: Path) -> Optional[dict]: """Read ffprobe data from .ffprobe.json file if it exists.""" if not path or not path.is_file(): diff --git a/modules/videos/lib.py b/modules/videos/lib.py index f14ee4aa..2bae63e0 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -27,7 +27,7 @@ from wrolpi.switches import register_switch_handler, ActivateSwitchMethod from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST, WROLPI_HOME from .common import is_valid_poster, convert_image, \ generate_video_poster, ConfigError, \ - extract_video_duration + extract_video_duration, ffprobe_json_sync, read_ffprobe_json_file, write_ffprobe_json_file from .errors import UnknownChannel from .models import Channel @@ -153,15 +153,37 @@ def validate_video(session: Session, video: Video, channel_generate_poster: bool if duration: video.file_group.length = duration + if video_path and not video.ffprobe_json: + # Extract ffprobe data if not already present (e.g., from .ffprobe.json cache file) + ffprobe_path = video.ffprobe_json_path + + # Try to read from cache file first + if ffprobe_path and ffprobe_path.is_file(): + video.ffprobe_json = read_ffprobe_json_file(ffprobe_path) + + # If still no data, run ffprobe + if not video.ffprobe_json: + try: + video.ffprobe_json = ffprobe_json_sync(video_path) + # Write cache file if enabled + if ffprobe_path and get_wrolpi_config().save_ffprobe_json: + try: + write_ffprobe_json_file(ffprobe_path, video.ffprobe_json) + video.file_group.append_files(ffprobe_path) + except IOError as e: + logger.warning(f'Failed to write ffprobe json cache file {ffprobe_path}', exc_info=e) + except Exception as e: + logger.warning(f'Failed to extract ffprobe data for {video}', exc_info=e) + + # Extract duration from ffprobe data if video_path and not video.file_group.length: - # Duration was not retrieved during poster generation. if video.ffprobe_json: if duration := video.ffprobe_json['format'].get('duration'): video.file_group.length = float(duration) elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]: video.file_group.length = float(video_streams[0]['duration']) else: - # Slowest method. + # Fallback if ffprobe extraction failed video.file_group.length = extract_video_duration(video_path) if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES: diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 6c1bd613..4bc09afd 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -72,8 +72,13 @@ async def test_video_factory(test_session, video_factory, channel_factory): assert video.source_id == 'some id' -def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file): +def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file, + test_wrolpi_config): """A video poster will be generated only if the channel permits.""" + # Disable ffprobe json file creation to avoid unexpected files in test assertions + # (Directly modify _config to avoid locking issues in sync tests) + get_wrolpi_config()._config['save_ffprobe_json'] = False + video_file = video_file.rename(test_directory / 'Channel Name_20050607_1234567890_The Title.mp4') vid1 = video_factory(with_video_file=video_file, with_info_json=True) # Clear source_id so source_id in the file will be extracted. @@ -142,7 +147,10 @@ async def test_get_statistics(test_session, video_factory, channel_factory): @pytest.mark.asyncio async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory, - refresh_files): + refresh_files, test_wrolpi_config): + # Disable ffprobe json file creation to avoid unexpected files in test assertions + get_wrolpi_config().save_ffprobe_json = False + # A Video without associated files is not orphaned. vid1 = video_factory(title='vid1', with_video_file=True) # The video files will be removed... diff --git a/wrolpi/files/api.py b/wrolpi/files/api.py index c886e0f9..5ad08ee6 100644 --- a/wrolpi/files/api.py +++ b/wrolpi/files/api.py @@ -353,13 +353,11 @@ async def post_upload(request: Request): mkdir = request.form['mkdir'][0] mkdir = True if mkdir.strip().lower() == 'true' else False except Exception as e: - logger.error('Failed to parse mkdir form data', exc_info=e) mkdir = False try: destination_str = request.form['destination'][0] except Exception as e: - logger.error(f'Failed to get upload destination', exc_info=e) raise UnknownDirectory('Must provide destination') from e destination = get_media_directory() / destination_str diff --git a/wrolpi/files/test/test_api.py b/wrolpi/files/test/test_api.py index a529c47f..0c712cd9 100644 --- a/wrolpi/files/test/test_api.py +++ b/wrolpi/files/test/test_api.py @@ -560,8 +560,11 @@ async def test_post_upload_directory(test_session, async_client, test_directory, @pytest.mark.asyncio async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form, - tag_factory, video_bytes): + tag_factory, video_bytes, test_wrolpi_config): """A file can be uploaded in chunks directly to the destination.""" + # Disable ffprobe json file creation to avoid unexpected files in test assertions + get_wrolpi_config().save_ffprobe_json = False + make_files_structure(['uploads/']) tag1, tag2 = await tag_factory(), await tag_factory() @@ -691,6 +694,54 @@ async def test_post_upload(test_session, async_client, test_directory, make_file assert not video.channel and not video.channel_id +@pytest.mark.asyncio +async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure, + make_multipart_form, video_bytes): + """Uploading a video file should extract and store ffprobe data.""" + make_files_structure(['uploads/']) + + # Upload video in chunks (same pattern as test_post_upload) + part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:] + + # First chunk + forms = [ + dict(name='chunkNumber', value='0'), + dict(name='filename', value='test_ffprobe.mp4'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value=1_000_000), + dict(name='chunk', value=part1, filename='chunk'), + ] + body = make_multipart_form(forms) + headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'} + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.OK + + # Second chunk (final) + forms = [ + dict(name='chunkNumber', value='1'), + dict(name='filename', value='test_ffprobe.mp4'), + dict(name='totalChunks', value='1'), + dict(name='destination', value='uploads'), + dict(name='chunkSize', value=len(part2)), + dict(name='chunk', value=part2, filename='chunk'), + ] + body = make_multipart_form(forms) + request, response = await async_client.post('/api/files/upload', content=body, headers=headers) + assert response.status_code == HTTPStatus.CREATED + + # Verify video was created + video: Video = test_session.query(Video).one() + assert video.video_path + + # THIS IS THE BUG - ffprobe_json should be populated but it's not + assert video.ffprobe_json is not None, 'ffprobe_json should be extracted on upload' + assert 'streams' in video.ffprobe_json, 'ffprobe_json should contain streams' + assert 'format' in video.ffprobe_json, 'ffprobe_json should contain format' + assert video.file_group.length, 'Duration should be extracted from ffprobe' + assert video.ffprobe_json_path + + @pytest.mark.asyncio async def test_post_upload_text(test_session, async_client, test_directory, make_files_structure, make_multipart_form): """A file that cannot be modeled can still be uploaded, and is indexed.""" diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 102a5b88..b40cb5c4 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -15,7 +15,7 @@ from PIL import Image import wrolpi.common from modules.videos import Video -from wrolpi.common import timer +from wrolpi.common import timer, get_wrolpi_config from wrolpi.conftest import await_switches from wrolpi.dates import now from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile @@ -1108,8 +1108,12 @@ def test_get_bulk_tag_preview_empty(test_session, test_directory): ('multi_file_filegroup', 1), ]) async def test_process_bulk_tag_job_add_tags(test_case, expected_fg_count, async_client, test_session, - make_files_structure, tag_factory, video_bytes, srt_text): + make_files_structure, tag_factory, video_bytes, srt_text, + test_wrolpi_config): """_process_bulk_tag_job adds tags to files in various scenarios.""" + # Disable ffprobe json file creation to avoid unexpected files in test assertions + get_wrolpi_config().save_ffprobe_json = False + if test_case == 'existing_filegroups': foo, bar = make_files_structure({'foo.txt': 'foo', 'bar.txt': 'bar'}) FileGroup.from_paths(test_session, foo) diff --git a/wrolpi/files/test/test_models.py b/wrolpi/files/test/test_models.py index f9cff48e..8ac7fdeb 100644 --- a/wrolpi/files/test/test_models.py +++ b/wrolpi/files/test/test_models.py @@ -1,5 +1,6 @@ import pytest +from wrolpi.common import get_wrolpi_config from wrolpi.errors import UnknownFile from wrolpi.files.models import FileGroup @@ -25,8 +26,11 @@ def test_file_group_find_by_id(test_session, make_files_structure): @pytest.mark.asyncio -async def test_file_group_move(async_client, test_session, test_directory, video_factory): +async def test_file_group_move(async_client, test_session, test_directory, video_factory, test_wrolpi_config): """Any new or renamed files are adopted during FileGroup.move.""" + # Disable ffprobe json file creation to avoid unexpected files in test assertions + get_wrolpi_config().save_ffprobe_json = False + video = video_factory(title='video', with_poster_ext='.webp') test_session.commit() diff --git a/wrolpi/test/test_tags.py b/wrolpi/test/test_tags.py index 90b7e972..149271fd 100644 --- a/wrolpi/test/test_tags.py +++ b/wrolpi/test/test_tags.py @@ -352,8 +352,11 @@ async def test_invalid_tag(async_client, test_session, test_directory): @pytest.mark.asyncio async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches, - refresh_files): + refresh_files, test_wrolpi_config): """Test that Tag Directory is synchronized with database.""" + # Disable ffprobe json file creation to avoid unexpected files in test assertions + get_wrolpi_config().save_ffprobe_json = False + readme = test_directory / 'tags/README.txt' # Should be deleted on sync.