Saving video ffprobe json when video file is uploaded.

This commit is contained in:
Roland 2026-02-08 12:29:25 -07:00
parent 6b939bb195
commit 387ef6dfe8
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
8 changed files with 146 additions and 12 deletions

View file

@ -244,6 +244,50 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict:
return content
def ffprobe_json_sync(video_path: Union[Path, str]) -> dict:
"""Synchronous version of ffprobe_json for use in sync validation code.
Extract video file metadata using ffprobe.
>>> ffprobe_json_sync('video.mp4')
{
'chapters': [],
'format': {},
'streams': [
{},
],
}
"""
if not FFPROBE_BIN:
raise RuntimeError('ffprobe was not found')
cmd = [FFPROBE_BIN,
'-print_format', 'json',
'-loglevel', 'quiet',
'-show_streams',
'-show_format',
'-show_chapters',
str(Path(video_path).absolute())]
proc = subprocess.run(cmd, capture_output=True, timeout=60)
if proc.returncode != 0:
raise RuntimeError(f'Got non-zero exit code: {proc.returncode}')
try:
content = json.loads(proc.stdout.decode().strip())
except Exception as e:
logger.debug(proc.stdout.decode())
logger.error(f'Failed to load ffprobe json', exc_info=e)
raise
if content == dict():
# Data is empty, video may be corrupt.
raise RuntimeError('ffprobe data was empty')
return content
def read_ffprobe_json_file(path: Path) -> Optional[dict]:
"""Read ffprobe data from .ffprobe.json file if it exists."""
if not path or not path.is_file():

View file

@ -27,7 +27,7 @@ from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST, WROLPI_HOME
from .common import is_valid_poster, convert_image, \
generate_video_poster, ConfigError, \
extract_video_duration
extract_video_duration, ffprobe_json_sync, read_ffprobe_json_file, write_ffprobe_json_file
from .errors import UnknownChannel
from .models import Channel
@ -153,15 +153,37 @@ def validate_video(session: Session, video: Video, channel_generate_poster: bool
if duration:
video.file_group.length = duration
if video_path and not video.ffprobe_json:
# Extract ffprobe data if not already present (e.g., from .ffprobe.json cache file)
ffprobe_path = video.ffprobe_json_path
# Try to read from cache file first
if ffprobe_path and ffprobe_path.is_file():
video.ffprobe_json = read_ffprobe_json_file(ffprobe_path)
# If still no data, run ffprobe
if not video.ffprobe_json:
try:
video.ffprobe_json = ffprobe_json_sync(video_path)
# Write cache file if enabled
if ffprobe_path and get_wrolpi_config().save_ffprobe_json:
try:
write_ffprobe_json_file(ffprobe_path, video.ffprobe_json)
video.file_group.append_files(ffprobe_path)
except IOError as e:
logger.warning(f'Failed to write ffprobe json cache file {ffprobe_path}', exc_info=e)
except Exception as e:
logger.warning(f'Failed to extract ffprobe data for {video}', exc_info=e)
# Extract duration from ffprobe data
if video_path and not video.file_group.length:
# Duration was not retrieved during poster generation.
if video.ffprobe_json:
if duration := video.ffprobe_json['format'].get('duration'):
video.file_group.length = float(duration)
elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]:
video.file_group.length = float(video_streams[0]['duration'])
else:
# Slowest method.
# Fallback if ffprobe extraction failed
video.file_group.length = extract_video_duration(video_path)
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:

View file

@ -72,8 +72,13 @@ async def test_video_factory(test_session, video_factory, channel_factory):
assert video.source_id == 'some id'
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file):
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file,
test_wrolpi_config):
"""A video poster will be generated only if the channel permits."""
# Disable ffprobe json file creation to avoid unexpected files in test assertions
# (Directly modify _config to avoid locking issues in sync tests)
get_wrolpi_config()._config['save_ffprobe_json'] = False
video_file = video_file.rename(test_directory / 'Channel Name_20050607_1234567890_The Title.mp4')
vid1 = video_factory(with_video_file=video_file, with_info_json=True)
# Clear source_id so source_id in the file will be extracted.
@ -142,7 +147,10 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
@pytest.mark.asyncio
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory,
refresh_files):
refresh_files, test_wrolpi_config):
# Disable ffprobe json file creation to avoid unexpected files in test assertions
get_wrolpi_config().save_ffprobe_json = False
# A Video without associated files is not orphaned.
vid1 = video_factory(title='vid1', with_video_file=True)
# The video files will be removed...

View file

@ -353,13 +353,11 @@ async def post_upload(request: Request):
mkdir = request.form['mkdir'][0]
mkdir = True if mkdir.strip().lower() == 'true' else False
except Exception as e:
logger.error('Failed to parse mkdir form data', exc_info=e)
mkdir = False
try:
destination_str = request.form['destination'][0]
except Exception as e:
logger.error(f'Failed to get upload destination', exc_info=e)
raise UnknownDirectory('Must provide destination') from e
destination = get_media_directory() / destination_str

View file

@ -560,8 +560,11 @@ async def test_post_upload_directory(test_session, async_client, test_directory,
@pytest.mark.asyncio
async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form,
tag_factory, video_bytes):
tag_factory, video_bytes, test_wrolpi_config):
"""A file can be uploaded in chunks directly to the destination."""
# Disable ffprobe json file creation to avoid unexpected files in test assertions
get_wrolpi_config().save_ffprobe_json = False
make_files_structure(['uploads/'])
tag1, tag2 = await tag_factory(), await tag_factory()
@ -691,6 +694,54 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
assert not video.channel and not video.channel_id
@pytest.mark.asyncio
async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure,
make_multipart_form, video_bytes):
"""Uploading a video file should extract and store ffprobe data."""
make_files_structure(['uploads/'])
# Upload video in chunks (same pattern as test_post_upload)
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
# First chunk
forms = [
dict(name='chunkNumber', value='0'),
dict(name='filename', value='test_ffprobe.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=1_000_000),
dict(name='chunk', value=part1, filename='chunk'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
# Second chunk (final)
forms = [
dict(name='chunkNumber', value='1'),
dict(name='filename', value='test_ffprobe.mp4'),
dict(name='totalChunks', value='1'),
dict(name='destination', value='uploads'),
dict(name='chunkSize', value=len(part2)),
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED
# Verify video was created
video: Video = test_session.query(Video).one()
assert video.video_path
# THIS IS THE BUG - ffprobe_json should be populated but it's not
assert video.ffprobe_json is not None, 'ffprobe_json should be extracted on upload'
assert 'streams' in video.ffprobe_json, 'ffprobe_json should contain streams'
assert 'format' in video.ffprobe_json, 'ffprobe_json should contain format'
assert video.file_group.length, 'Duration should be extracted from ffprobe'
assert video.ffprobe_json_path
@pytest.mark.asyncio
async def test_post_upload_text(test_session, async_client, test_directory, make_files_structure, make_multipart_form):
"""A file that cannot be modeled can still be uploaded, and is indexed."""

View file

@ -15,7 +15,7 @@ from PIL import Image
import wrolpi.common
from modules.videos import Video
from wrolpi.common import timer
from wrolpi.common import timer, get_wrolpi_config
from wrolpi.conftest import await_switches
from wrolpi.dates import now
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
@ -1108,8 +1108,12 @@ def test_get_bulk_tag_preview_empty(test_session, test_directory):
('multi_file_filegroup', 1),
])
async def test_process_bulk_tag_job_add_tags(test_case, expected_fg_count, async_client, test_session,
make_files_structure, tag_factory, video_bytes, srt_text):
make_files_structure, tag_factory, video_bytes, srt_text,
test_wrolpi_config):
"""_process_bulk_tag_job adds tags to files in various scenarios."""
# Disable ffprobe json file creation to avoid unexpected files in test assertions
get_wrolpi_config().save_ffprobe_json = False
if test_case == 'existing_filegroups':
foo, bar = make_files_structure({'foo.txt': 'foo', 'bar.txt': 'bar'})
FileGroup.from_paths(test_session, foo)

View file

@ -1,5 +1,6 @@
import pytest
from wrolpi.common import get_wrolpi_config
from wrolpi.errors import UnknownFile
from wrolpi.files.models import FileGroup
@ -25,8 +26,11 @@ def test_file_group_find_by_id(test_session, make_files_structure):
@pytest.mark.asyncio
async def test_file_group_move(async_client, test_session, test_directory, video_factory):
async def test_file_group_move(async_client, test_session, test_directory, video_factory, test_wrolpi_config):
"""Any new or renamed files are adopted during FileGroup.move."""
# Disable ffprobe json file creation to avoid unexpected files in test assertions
get_wrolpi_config().save_ffprobe_json = False
video = video_factory(title='video', with_poster_ext='.webp')
test_session.commit()

View file

@ -352,8 +352,11 @@ async def test_invalid_tag(async_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches,
refresh_files):
refresh_files, test_wrolpi_config):
"""Test that Tag Directory is synchronized with database."""
# Disable ffprobe json file creation to avoid unexpected files in test assertions
get_wrolpi_config().save_ffprobe_json = False
readme = test_directory / 'tags/README.txt'
# Should be deleted on sync.