mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Transcode: verify the output before replacing the original
After ffmpeg exits, the output is probed and compared with the source (container, presence of video/audio streams, codec per stream, duration within 1s or 1%) and its last 30 seconds are decoded to a null output. Any failure removes the output and keeps the original. The tail decode replaces the download-time completeness check, which grabbed a single frame near the end: one ffmpeg_video_complete (sync for the downloader, async for Jobs) now serves both, and is stricter, since every frame of the tail plus the audio must decode. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
d5917749a7
commit
f2d15c2ffe
5 changed files with 386 additions and 29 deletions
|
|
@ -5,10 +5,51 @@ from unittest import mock
|
|||
import pytest
|
||||
|
||||
from modules.videos.transcode import codecs_match, get_stream_codec_names, get_transcode_target, \
|
||||
transcode_can_satisfy_codecs, transcode_video_file, TRANSCODE_VIDEO_TARGETS, TRANSCODE_AUDIO_TARGETS
|
||||
transcode_can_satisfy_codecs, transcode_video_file, TRANSCODE_VIDEO_TARGETS, TRANSCODE_AUDIO_TARGETS, \
|
||||
verify_transcode_output
|
||||
from wrolpi.cmd import CommandResult
|
||||
|
||||
|
||||
def make_probe(vcodec='vp9', acodec='opus', duration=10.0, format_name='matroska,webm'):
|
||||
"""An ffprobe result with one video and one audio stream."""
|
||||
streams = []
|
||||
if vcodec:
|
||||
streams.append({'codec_type': 'video', 'codec_name': vcodec})
|
||||
if acodec:
|
||||
streams.append({'codec_type': 'audio', 'codec_name': acodec})
|
||||
return {'streams': streams, 'format': {'format_name': format_name, 'duration': str(duration)}}
|
||||
|
||||
|
||||
MP4 = 'mov,mp4,m4a,3gp,3g2,mj2'
|
||||
|
||||
|
||||
class FakeProbes:
|
||||
"""What ffprobe reports for the source (a vp9/opus webm) and for the transcode output.
|
||||
|
||||
The default output is a full h264/aac transcode; a test that copies a stream sets `output`
|
||||
to what ffmpeg would really produce (a copied stream keeps the source codec)."""
|
||||
|
||||
def __init__(self):
|
||||
self.source = make_probe('vp9', 'opus', 10.0)
|
||||
self.output = make_probe('h264', 'aac', 10.0, MP4)
|
||||
|
||||
def __call__(self, path: pathlib.Path):
|
||||
# Sync: mock.patch wraps the async target in an AsyncMock, which awaits for us.
|
||||
if '.transcode.' in path.name:
|
||||
output = dict(self.output)
|
||||
if path.suffix != '.mp4' and output.get('format'):
|
||||
output['format'] = {**output['format'], 'format_name': 'matroska,webm'}
|
||||
return output
|
||||
return self.source
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_probe() -> FakeProbes:
|
||||
probes = FakeProbes()
|
||||
with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes):
|
||||
yield probes
|
||||
|
||||
|
||||
def test_get_stream_codec_names():
|
||||
"""Embedded thumbnail streams (mjpeg/png) are not video."""
|
||||
data = {'streams': [
|
||||
|
|
@ -79,6 +120,9 @@ async def test_transcode_video_file(test_directory, async_client):
|
|||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
# ffmpeg writes the temporary output file.
|
||||
if cmd[-1] == '-':
|
||||
# The tail-decode check: nothing to write.
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
|
|
@ -92,7 +136,7 @@ async def test_transcode_video_file(test_directory, async_client):
|
|||
assert not sidecar.exists(), 'The stale ffprobe sidecar should be deleted'
|
||||
assert not (test_directory / 'video.transcode.mp4').exists(), 'The temporary file should be renamed'
|
||||
|
||||
cmd = mock_run.call_args[0][0]
|
||||
cmd = mock_run.call_args_list[0][0][0]
|
||||
assert '-c:v' in cmd and cmd[cmd.index('-c:v') + 1] == 'libx264'
|
||||
assert '-c:a' in cmd and cmd[cmd.index('-c:a') + 1] == 'aac'
|
||||
# Attached-picture streams are excluded; a missing audio stream is tolerated.
|
||||
|
|
@ -100,12 +144,16 @@ async def test_transcode_video_file(test_directory, async_client):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcode_video_file_copies_matching_stream(test_directory, async_client):
|
||||
async def test_transcode_video_file_copies_matching_stream(test_directory, async_client, mock_probe):
|
||||
"""Only the mismatching stream is re-encoded; the other stream is copied."""
|
||||
video_path = test_directory / 'video.mp4'
|
||||
video_path.write_bytes(b'fake video data')
|
||||
mock_probe.output = make_probe('h264', 'opus', 10.0, MP4) # Audio was copied.
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] == '-':
|
||||
# The tail-decode check: nothing to write.
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
|
|
@ -113,18 +161,22 @@ async def test_transcode_video_file_copies_matching_stream(test_directory, async
|
|||
result = await transcode_video_file(video_path, target_vcodec='h264', container='mp4')
|
||||
|
||||
assert result == video_path, 'Same container: the file is replaced in place'
|
||||
cmd = mock_run.call_args[0][0]
|
||||
cmd = mock_run.call_args_list[0][0][0]
|
||||
assert cmd[cmd.index('-c:v') + 1] == 'libx264'
|
||||
assert cmd[cmd.index('-c:a') + 1] == 'copy'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcode_video_file_forces_supported_container(test_directory, async_client):
|
||||
async def test_transcode_video_file_forces_supported_container(test_directory, async_client, mock_probe):
|
||||
"""webm cannot contain h264/aac; the container is forced to mp4."""
|
||||
video_path = test_directory / 'video.webm'
|
||||
video_path.write_bytes(b'fake video data')
|
||||
mock_probe.output = make_probe('h264', 'opus', 10.0, MP4) # Audio was copied.
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] == '-':
|
||||
# The tail-decode check: nothing to write.
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
|
|
@ -141,6 +193,9 @@ async def test_transcode_video_file_failure_removes_tmp(test_directory, async_cl
|
|||
video_path.write_bytes(b'fake video data')
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] == '-':
|
||||
# The tail-decode check: nothing to write.
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'partial data')
|
||||
return CommandResult(return_code=1, cancelled=False, stdout=b'', stderr=b'boom', elapsed=1)
|
||||
|
||||
|
|
@ -168,7 +223,7 @@ async def test_transcode_video_file_disk_space_guard(test_directory, async_clien
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcode_lock_serializes(test_directory, async_client):
|
||||
async def test_transcode_lock_serializes(test_directory, async_client, mock_probe):
|
||||
"""A transcode waits while `shared_ctx.transcode_lock` is held and releases it when done.
|
||||
|
||||
This exercises the wait/release logic in-process; cross-process sharing of the lock is
|
||||
|
|
@ -178,8 +233,12 @@ async def test_transcode_lock_serializes(test_directory, async_client):
|
|||
|
||||
video_path = test_directory / 'video.webm'
|
||||
video_path.write_bytes(b'fake video data')
|
||||
mock_probe.output = make_probe('h264', 'opus', 10.0, MP4) # Audio was copied.
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] == '-':
|
||||
# The tail-decode check: nothing to write.
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
|
|
@ -203,7 +262,9 @@ async def test_transcode_lock_serializes(test_directory, async_client):
|
|||
task.cancel()
|
||||
|
||||
assert result == test_directory / 'video.mp4'
|
||||
mock_run.assert_called_once()
|
||||
# One encode (plus the tail-decode verification of its output); the second waiter ran nothing yet.
|
||||
encodes = [c for c in mock_run.call_args_list if '-progress' in c[0][0]]
|
||||
assert len(encodes) == 1
|
||||
|
||||
# The lock was released after the transcode.
|
||||
assert lock.acquire(block=False), 'transcode_video_file did not release the lock'
|
||||
|
|
@ -211,12 +272,16 @@ async def test_transcode_lock_serializes(test_directory, async_client):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcode_video_file_remux(test_directory, async_client):
|
||||
async def test_transcode_video_file_remux(test_directory, async_client, mock_probe):
|
||||
"""No target codec is a remux: both streams are copied into the requested container."""
|
||||
video_path = test_directory / 'video.mkv'
|
||||
video_path.write_bytes(b'fake video data')
|
||||
mock_probe.output = make_probe('vp9', 'opus', 10.0, MP4) # Both copied.
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] == '-':
|
||||
# The tail-decode check: nothing to write.
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'remuxed data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
|
|
@ -225,6 +290,114 @@ async def test_transcode_video_file_remux(test_directory, async_client):
|
|||
|
||||
assert result == test_directory / 'video.mp4' and result.is_file()
|
||||
assert not video_path.exists()
|
||||
cmd = mock_run.call_args[0][0]
|
||||
cmd = mock_run.call_args_list[0][0][0]
|
||||
assert cmd[cmd.index('-c:v') + 1] == 'copy' and cmd[cmd.index('-c:a') + 1] == 'copy'
|
||||
assert '+faststart' in cmd
|
||||
|
||||
|
||||
|
||||
# ------------------------------------------------------------------ output verification
|
||||
|
||||
def test_verify_transcode_output_ok():
|
||||
source = make_probe('vp9', 'opus', 100.0)
|
||||
output = make_probe('h264', 'opus', 99.6, MP4) # Audio copied; a frame trimmed.
|
||||
verify_transcode_output(source, output, target_vcodec='h264', target_acodec=None, container='mp4')
|
||||
|
||||
|
||||
def test_verify_transcode_output_remux_ok():
|
||||
source = make_probe('vp9', 'opus', 100.0)
|
||||
output = make_probe('vp9', 'opus', 100.0, MP4)
|
||||
verify_transcode_output(source, output, target_vcodec=None, target_acodec=None, container='mp4')
|
||||
|
||||
|
||||
def test_verify_transcode_output_no_audio_ok():
|
||||
"""A silent source has no audio to demand of the output."""
|
||||
source = make_probe('vp9', None, 100.0)
|
||||
output = make_probe('h264', None, 100.0, MP4)
|
||||
verify_transcode_output(source, output, target_vcodec='h264', target_acodec=None, container='mp4')
|
||||
|
||||
|
||||
@pytest.mark.parametrize('output,message', [
|
||||
(make_probe('h264', 'aac', 50.0, MP4), 'duration'), # Truncated.
|
||||
(make_probe('h264', None, 100.0, MP4), 'audio'), # Lost the audio stream.
|
||||
(make_probe(None, 'aac', 100.0, MP4), 'video'), # Lost the video stream.
|
||||
(make_probe('vp9', 'aac', 100.0, MP4), 'video codec'), # Not re-encoded.
|
||||
(make_probe('h264', 'opus', 100.0, MP4), 'audio codec'), # Not re-encoded.
|
||||
(make_probe('h264', 'aac', 100.0, 'matroska,webm'), 'container'), # Wrong container.
|
||||
({}, 'probe'), # ffprobe could not read it.
|
||||
])
|
||||
def test_verify_transcode_output_rejects(output, message):
|
||||
source = make_probe('vp9', 'opus', 100.0)
|
||||
with pytest.raises(RuntimeError, match=message):
|
||||
verify_transcode_output(source, output, target_vcodec='h264', target_acodec='aac', container='mp4')
|
||||
|
||||
|
||||
def test_verify_transcode_output_copied_stream_must_match_source():
|
||||
"""A copied stream must come out with the source's codec."""
|
||||
source = make_probe('vp9', 'opus', 100.0)
|
||||
output = make_probe('h264', 'aac', 100.0, MP4)
|
||||
with pytest.raises(RuntimeError, match='audio codec'):
|
||||
verify_transcode_output(source, output, target_vcodec='h264', target_acodec=None, container='mp4')
|
||||
|
||||
|
||||
def test_verify_transcode_output_duration_tolerance():
|
||||
"""One second, or one percent, whichever is larger."""
|
||||
source = make_probe('vp9', 'opus', 10.0)
|
||||
verify_transcode_output(source, make_probe('h264', 'aac', 9.1, MP4), 'h264', 'aac', 'mp4')
|
||||
with pytest.raises(RuntimeError, match='duration'):
|
||||
verify_transcode_output(source, make_probe('h264', 'aac', 8.9, MP4), 'h264', 'aac', 'mp4')
|
||||
long_source = make_probe('vp9', 'opus', 40000.0)
|
||||
verify_transcode_output(long_source, make_probe('h264', 'aac', 39700.0, MP4), 'h264', 'aac', 'mp4')
|
||||
with pytest.raises(RuntimeError, match='duration'):
|
||||
verify_transcode_output(long_source, make_probe('h264', 'aac', 39500.0, MP4), 'h264', 'aac', 'mp4')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcode_video_file_rejects_truncated_output(test_directory, async_client, mock_probe):
|
||||
"""ffmpeg exits 0 but the output is short: the original is kept, the output is removed."""
|
||||
video_path = test_directory / 'video.webm'
|
||||
video_path.write_bytes(b'fake video data')
|
||||
|
||||
mock_probe.output = make_probe('h264', 'aac', 4.0, MP4) # Source is 10s.
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] != '-':
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'partial data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command):
|
||||
with pytest.raises(RuntimeError, match='duration'):
|
||||
await transcode_video_file(video_path, target_vcodec='h264', target_acodec='aac')
|
||||
|
||||
assert video_path.read_bytes() == b'fake video data', 'The original must survive'
|
||||
assert not (test_directory / 'video.mp4').exists()
|
||||
assert not (test_directory / 'video.transcode.mp4').exists(), 'The bad output is removed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_transcode_video_file_rejects_undecodable_tail(test_directory, async_client):
|
||||
"""The last seconds of the output must decode cleanly, or the original is kept."""
|
||||
video_path = test_directory / 'video.webm'
|
||||
video_path.write_bytes(b'fake video data')
|
||||
tail_cmds = []
|
||||
|
||||
async def fake_run_command(cmd, **kwargs):
|
||||
if cmd[-1] == '-':
|
||||
tail_cmds.append(cmd)
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'',
|
||||
stderr=b'[h264 @ 0x1] Invalid NAL unit size', elapsed=1)
|
||||
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
||||
|
||||
with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command):
|
||||
with pytest.raises(RuntimeError, match='decode'):
|
||||
await transcode_video_file(video_path, target_vcodec='h264', target_acodec='aac')
|
||||
|
||||
assert video_path.read_bytes() == b'fake video data'
|
||||
assert not (test_directory / 'video.transcode.mp4').exists()
|
||||
# The check decodes only the tail of the temporary output, discarding the frames.
|
||||
assert len(tail_cmds) == 1
|
||||
cmd = tail_cmds[0]
|
||||
assert '-sseof' in cmd and cmd[cmd.index('-sseof') + 1].startswith('-')
|
||||
assert str(cmd[cmd.index('-i') + 1]).endswith('video.transcode.mp4')
|
||||
assert cmd[cmd.index('-f') + 1] == 'null'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue