mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
Saving video ffprobe json when video file is uploaded.
This commit is contained in:
parent
6b939bb195
commit
387ef6dfe8
8 changed files with 146 additions and 12 deletions
|
|
@ -244,6 +244,50 @@ async def ffprobe_json(video_path: Union[Path, str]) -> dict:
|
|||
return content
|
||||
|
||||
|
||||
def ffprobe_json_sync(video_path: Union[Path, str]) -> dict:
|
||||
"""Synchronous version of ffprobe_json for use in sync validation code.
|
||||
|
||||
Extract video file metadata using ffprobe.
|
||||
|
||||
>>> ffprobe_json_sync('video.mp4')
|
||||
{
|
||||
'chapters': [],
|
||||
'format': {},
|
||||
'streams': [
|
||||
{},
|
||||
],
|
||||
}
|
||||
"""
|
||||
if not FFPROBE_BIN:
|
||||
raise RuntimeError('ffprobe was not found')
|
||||
|
||||
cmd = [FFPROBE_BIN,
|
||||
'-print_format', 'json',
|
||||
'-loglevel', 'quiet',
|
||||
'-show_streams',
|
||||
'-show_format',
|
||||
'-show_chapters',
|
||||
str(Path(video_path).absolute())]
|
||||
|
||||
proc = subprocess.run(cmd, capture_output=True, timeout=60)
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f'Got non-zero exit code: {proc.returncode}')
|
||||
|
||||
try:
|
||||
content = json.loads(proc.stdout.decode().strip())
|
||||
except Exception as e:
|
||||
logger.debug(proc.stdout.decode())
|
||||
logger.error(f'Failed to load ffprobe json', exc_info=e)
|
||||
raise
|
||||
|
||||
if content == dict():
|
||||
# Data is empty, video may be corrupt.
|
||||
raise RuntimeError('ffprobe data was empty')
|
||||
|
||||
return content
|
||||
|
||||
|
||||
def read_ffprobe_json_file(path: Path) -> Optional[dict]:
|
||||
"""Read ffprobe data from .ffprobe.json file if it exists."""
|
||||
if not path or not path.is_file():
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
|||
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST, WROLPI_HOME
|
||||
from .common import is_valid_poster, convert_image, \
|
||||
generate_video_poster, ConfigError, \
|
||||
extract_video_duration
|
||||
extract_video_duration, ffprobe_json_sync, read_ffprobe_json_file, write_ffprobe_json_file
|
||||
from .errors import UnknownChannel
|
||||
from .models import Channel
|
||||
|
||||
|
|
@ -153,15 +153,37 @@ def validate_video(session: Session, video: Video, channel_generate_poster: bool
|
|||
if duration:
|
||||
video.file_group.length = duration
|
||||
|
||||
if video_path and not video.ffprobe_json:
|
||||
# Extract ffprobe data if not already present (e.g., from .ffprobe.json cache file)
|
||||
ffprobe_path = video.ffprobe_json_path
|
||||
|
||||
# Try to read from cache file first
|
||||
if ffprobe_path and ffprobe_path.is_file():
|
||||
video.ffprobe_json = read_ffprobe_json_file(ffprobe_path)
|
||||
|
||||
# If still no data, run ffprobe
|
||||
if not video.ffprobe_json:
|
||||
try:
|
||||
video.ffprobe_json = ffprobe_json_sync(video_path)
|
||||
# Write cache file if enabled
|
||||
if ffprobe_path and get_wrolpi_config().save_ffprobe_json:
|
||||
try:
|
||||
write_ffprobe_json_file(ffprobe_path, video.ffprobe_json)
|
||||
video.file_group.append_files(ffprobe_path)
|
||||
except IOError as e:
|
||||
logger.warning(f'Failed to write ffprobe json cache file {ffprobe_path}', exc_info=e)
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to extract ffprobe data for {video}', exc_info=e)
|
||||
|
||||
# Extract duration from ffprobe data
|
||||
if video_path and not video.file_group.length:
|
||||
# Duration was not retrieved during poster generation.
|
||||
if video.ffprobe_json:
|
||||
if duration := video.ffprobe_json['format'].get('duration'):
|
||||
video.file_group.length = float(duration)
|
||||
elif (video_streams := video.get_streams_by_codec_type('video')) and 'duration' in video_streams[0]:
|
||||
video.file_group.length = float(video_streams[0]['duration'])
|
||||
else:
|
||||
# Slowest method.
|
||||
# Fallback if ffprobe extraction failed
|
||||
video.file_group.length = extract_video_duration(video_path)
|
||||
|
||||
if video_path and not video.caption_paths and video.file_group.d_text and not EXTRACT_SUBTITLES:
|
||||
|
|
|
|||
|
|
@ -72,8 +72,13 @@ async def test_video_factory(test_session, video_factory, channel_factory):
|
|||
assert video.source_id == 'some id'
|
||||
|
||||
|
||||
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file):
|
||||
def test_validate_video(test_session, test_directory, video_factory, image_bytes_factory, video_file,
|
||||
test_wrolpi_config):
|
||||
"""A video poster will be generated only if the channel permits."""
|
||||
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
||||
# (Directly modify _config to avoid locking issues in sync tests)
|
||||
get_wrolpi_config()._config['save_ffprobe_json'] = False
|
||||
|
||||
video_file = video_file.rename(test_directory / 'Channel Name_20050607_1234567890_The Title.mp4')
|
||||
vid1 = video_factory(with_video_file=video_file, with_info_json=True)
|
||||
# Clear source_id so source_id in the file will be extracted.
|
||||
|
|
@ -142,7 +147,10 @@ async def test_get_statistics(test_session, video_factory, channel_factory):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_orphaned_files(async_client, test_session, make_files_structure, test_directory, video_factory,
|
||||
refresh_files):
|
||||
refresh_files, test_wrolpi_config):
|
||||
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
||||
get_wrolpi_config().save_ffprobe_json = False
|
||||
|
||||
# A Video without associated files is not orphaned.
|
||||
vid1 = video_factory(title='vid1', with_video_file=True)
|
||||
# The video files will be removed...
|
||||
|
|
|
|||
|
|
@ -353,13 +353,11 @@ async def post_upload(request: Request):
|
|||
mkdir = request.form['mkdir'][0]
|
||||
mkdir = True if mkdir.strip().lower() == 'true' else False
|
||||
except Exception as e:
|
||||
logger.error('Failed to parse mkdir form data', exc_info=e)
|
||||
mkdir = False
|
||||
|
||||
try:
|
||||
destination_str = request.form['destination'][0]
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to get upload destination', exc_info=e)
|
||||
raise UnknownDirectory('Must provide destination') from e
|
||||
|
||||
destination = get_media_directory() / destination_str
|
||||
|
|
|
|||
|
|
@ -560,8 +560,11 @@ async def test_post_upload_directory(test_session, async_client, test_directory,
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form,
|
||||
tag_factory, video_bytes):
|
||||
tag_factory, video_bytes, test_wrolpi_config):
|
||||
"""A file can be uploaded in chunks directly to the destination."""
|
||||
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
||||
get_wrolpi_config().save_ffprobe_json = False
|
||||
|
||||
make_files_structure(['uploads/'])
|
||||
tag1, tag2 = await tag_factory(), await tag_factory()
|
||||
|
||||
|
|
@ -691,6 +694,54 @@ async def test_post_upload(test_session, async_client, test_directory, make_file
|
|||
assert not video.channel and not video.channel_id
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_upload_video_extracts_ffprobe(test_session, async_client, test_directory, make_files_structure,
|
||||
make_multipart_form, video_bytes):
|
||||
"""Uploading a video file should extract and store ffprobe data."""
|
||||
make_files_structure(['uploads/'])
|
||||
|
||||
# Upload video in chunks (same pattern as test_post_upload)
|
||||
part1, part2 = video_bytes[:1_000_000], video_bytes[1_000_000:]
|
||||
|
||||
# First chunk
|
||||
forms = [
|
||||
dict(name='chunkNumber', value='0'),
|
||||
dict(name='filename', value='test_ffprobe.mp4'),
|
||||
dict(name='totalChunks', value='1'),
|
||||
dict(name='destination', value='uploads'),
|
||||
dict(name='chunkSize', value=1_000_000),
|
||||
dict(name='chunk', value=part1, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
||||
# Second chunk (final)
|
||||
forms = [
|
||||
dict(name='chunkNumber', value='1'),
|
||||
dict(name='filename', value='test_ffprobe.mp4'),
|
||||
dict(name='totalChunks', value='1'),
|
||||
dict(name='destination', value='uploads'),
|
||||
dict(name='chunkSize', value=len(part2)),
|
||||
dict(name='chunk', value=part2, filename='chunk'),
|
||||
]
|
||||
body = make_multipart_form(forms)
|
||||
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
|
||||
assert response.status_code == HTTPStatus.CREATED
|
||||
|
||||
# Verify video was created
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_path
|
||||
|
||||
# THIS IS THE BUG - ffprobe_json should be populated but it's not
|
||||
assert video.ffprobe_json is not None, 'ffprobe_json should be extracted on upload'
|
||||
assert 'streams' in video.ffprobe_json, 'ffprobe_json should contain streams'
|
||||
assert 'format' in video.ffprobe_json, 'ffprobe_json should contain format'
|
||||
assert video.file_group.length, 'Duration should be extracted from ffprobe'
|
||||
assert video.ffprobe_json_path
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_upload_text(test_session, async_client, test_directory, make_files_structure, make_multipart_form):
|
||||
"""A file that cannot be modeled can still be uploaded, and is indexed."""
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from PIL import Image
|
|||
|
||||
import wrolpi.common
|
||||
from modules.videos import Video
|
||||
from wrolpi.common import timer
|
||||
from wrolpi.common import timer, get_wrolpi_config
|
||||
from wrolpi.conftest import await_switches
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
|
||||
|
|
@ -1108,8 +1108,12 @@ def test_get_bulk_tag_preview_empty(test_session, test_directory):
|
|||
('multi_file_filegroup', 1),
|
||||
])
|
||||
async def test_process_bulk_tag_job_add_tags(test_case, expected_fg_count, async_client, test_session,
|
||||
make_files_structure, tag_factory, video_bytes, srt_text):
|
||||
make_files_structure, tag_factory, video_bytes, srt_text,
|
||||
test_wrolpi_config):
|
||||
"""_process_bulk_tag_job adds tags to files in various scenarios."""
|
||||
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
||||
get_wrolpi_config().save_ffprobe_json = False
|
||||
|
||||
if test_case == 'existing_filegroups':
|
||||
foo, bar = make_files_structure({'foo.txt': 'foo', 'bar.txt': 'bar'})
|
||||
FileGroup.from_paths(test_session, foo)
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import pytest
|
||||
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.errors import UnknownFile
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
||||
|
|
@ -25,8 +26,11 @@ def test_file_group_find_by_id(test_session, make_files_structure):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_move(async_client, test_session, test_directory, video_factory):
|
||||
async def test_file_group_move(async_client, test_session, test_directory, video_factory, test_wrolpi_config):
|
||||
"""Any new or renamed files are adopted during FileGroup.move."""
|
||||
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
||||
get_wrolpi_config().save_ffprobe_json = False
|
||||
|
||||
video = video_factory(title='video', with_poster_ext='.webp')
|
||||
test_session.commit()
|
||||
|
||||
|
|
|
|||
|
|
@ -352,8 +352,11 @@ async def test_invalid_tag(async_client, test_session, test_directory):
|
|||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tags_directory(test_session, test_directory, tag_factory, video_factory, example_pdf, await_switches,
|
||||
refresh_files):
|
||||
refresh_files, test_wrolpi_config):
|
||||
"""Test that Tag Directory is synchronized with database."""
|
||||
# Disable ffprobe json file creation to avoid unexpected files in test assertions
|
||||
get_wrolpi_config().save_ffprobe_json = False
|
||||
|
||||
readme = test_directory / 'tags/README.txt'
|
||||
|
||||
# Should be deleted on sync.
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue