diff --git a/app/src/api.js b/app/src/api.js
index 2d57e456..b06ef557 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -236,6 +236,65 @@ export async function getVideo(fileGroupId) {
return [data['file_group'], data['prev'], data['next']];
}
+export async function updateVideo(fileGroupId, {title, description}) {
+ // Edits the Video's details (written to its info json). Only the fields given are changed.
+ // Returns the updated video file group.
+ const response = await apiPut(`${VIDEOS_API}/${fileGroupId}`, {title, description});
+ if (!response.ok) {
+ const message = await getErrorMessage(response, 'Unable to save the video. See server logs.');
+ toast({type: 'error', title: 'Failed to save', description: message, time: 5000});
+ throw new Error(message);
+ }
+ const data = await response.json();
+ return data['file_group'];
+}
+
+export async function transcodeVideo(fileGroupId, {
+ video_codec = null, audio_codec = null, container = 'mp4', fragmented = false,
+}) {
+ // Queues a transcode Job; returns the job id. Watch it with `getJob`.
+ const body = {video_codec, audio_codec, container, fragmented};
+ const response = await apiPost(`${VIDEOS_API}/${fileGroupId}/transcode`, body);
+ if (!response.ok) {
+ const message = await getErrorMessage(response, 'Unable to queue the transcode. See server logs.');
+ toast({type: 'error', title: 'Failed to transcode', description: message, time: 5000});
+ throw new Error(message);
+ }
+ const data = await response.json();
+ return data['job_id'];
+}
+
+export async function getJobs() {
+ const response = await apiGet(`${API_URI}/jobs`);
+ if (!response.ok) {
+ throw new Error(await getErrorMessage(response, 'Unable to get the jobs'));
+ }
+ const data = await response.json();
+ return data['jobs'];
+}
+
+export async function getJob(jobId) {
+ const response = await apiGet(`${API_URI}/jobs/${jobId}`);
+ if (!response.ok) {
+ const error = new Error(await getErrorMessage(response, 'Unable to get the job'));
+ error.status = response.status;
+ throw error;
+ }
+ const data = await response.json();
+ return data['job'];
+}
+
+export async function cancelJob(jobId) {
+ const response = await apiDelete(`${API_URI}/jobs/${jobId}`);
+ if (!response.ok) {
+ const message = await getErrorMessage(response, 'Unable to cancel the job');
+ toast({type: 'error', title: 'Failed to cancel', description: message, time: 5000});
+ throw new Error(message);
+ }
+ const data = await response.json();
+ return data['job'];
+}
+
export async function getVideoComments(fileGroupId) {
const response = await apiGet(`${VIDEOS_API}/${fileGroupId}/comments`);
let data = await response.json();
diff --git a/app/src/components/ThemeSamplePage.js b/app/src/components/ThemeSamplePage.js
index 7767a317..3f95df25 100644
--- a/app/src/components/ThemeSamplePage.js
+++ b/app/src/components/ThemeSamplePage.js
@@ -16,6 +16,7 @@ import {
IconStack,
MediaFilterToggle,
MediaGate,
+ Menu,
Message,
MultiSelect,
NumberInput,
@@ -382,6 +383,22 @@ export function ThemeSamplePage() {
)}
+
+ A dropdown of actions
+ {/* Mirrors the video page's Edit control. */}
+
+
+
+
A row of actions
{/* `.wrolpi-button-row` is how a page lists the actions on a thing -- the
diff --git a/app/src/components/Vars.js b/app/src/components/Vars.js
index 02d293c3..01631236 100644
--- a/app/src/components/Vars.js
+++ b/app/src/components/Vars.js
@@ -158,6 +158,41 @@ export const downloadAudioCodecOptions = [
{key: 'vorbis', text: 'vorbis', value: 'vorbis'},
];
+// Codecs WROLPi's ffmpeg step can produce. Mirrors TRANSCODE_VIDEO_TARGETS / TRANSCODE_AUDIO_TARGETS /
+// TRANSCODE_CONTAINERS in modules/videos/transcode.py; the API rejects anything else.
+export const TRANSCODE_COPY = 'copy';
+// REMOVE_VIDEO in modules/videos/transcode.py: drop the video stream, the result is an audio file.
+export const TRANSCODE_REMOVE_VIDEO = 'none';
+export const transcodeVideoCodecOptions = [
+ {value: TRANSCODE_COPY, label: 'Keep (copy)'},
+ {value: 'h264', label: 'h264 (avc1)'},
+ {value: TRANSCODE_REMOVE_VIDEO, label: 'Remove video (audio only)'},
+];
+export const transcodeAudioCodecOptions = [
+ {value: TRANSCODE_COPY, label: 'Keep (copy)'},
+ {value: 'aac', label: 'aac'},
+ {value: 'opus', label: 'opus'},
+ {value: 'mp3', label: 'mp3'},
+];
+// Containers for output with a video stream ...
+export const transcodeContainerOptions = [
+ {value: 'mp4', label: 'mp4'},
+ {value: 'mkv', label: 'mkv'},
+];
+// ... and for audio-only output. AUDIO_CONTAINER_CODECS in transcode.py says what each holds.
+export const transcodeAudioContainerOptions = [
+ {value: 'm4a', label: 'm4a (aac)'},
+ {value: 'ogg', label: 'ogg (opus)'},
+ {value: 'mp3', label: 'mp3'},
+];
+
+/** The audio container a player expects for a codec (default_audio_container in transcode.py). */
+export function audioContainerForCodec(codec) {
+ if (codec === 'mp3') return 'mp3';
+ if (['opus', 'vorbis', 'flac'].includes(codec)) return 'ogg';
+ return 'm4a';
+}
+
export const Downloaders = {
Archive: 'archive',
File: 'file',
diff --git a/app/src/components/VideoEditMenu.js b/app/src/components/VideoEditMenu.js
new file mode 100644
index 00000000..5fa2eee6
--- /dev/null
+++ b/app/src/components/VideoEditMenu.js
@@ -0,0 +1,612 @@
+import React, {useEffect, useRef, useState} from 'react';
+import {cancelJob, fetchVideoDownloadDefaults, getJob, transcodeVideo, updateVideo} from '../api';
+import {useWROLMode} from '../hooks/customHooks';
+import {
+ Button, Confirm, Group, Icon, Menu, Modal, Progress, Select, Stack, Status, Text, Textarea, TextInput, Toggle,
+ toast,
+} from './ui';
+import {
+ audioContainerForCodec,
+ TRANSCODE_COPY,
+ TRANSCODE_REMOVE_VIDEO,
+ transcodeAudioCodecOptions,
+ transcodeAudioContainerOptions,
+ transcodeContainerOptions,
+ transcodeVideoCodecOptions,
+} from './Vars';
+
+const JOB_POLL_MS = 2000;
+// Consecutive failed polls before a Job is given up as lost (the API restarted and forgot it).
+const JOB_POLL_MAX_FAILURES = 5;
+export const FINISHED_STATUSES = ['complete', 'failed', 'cancelled'];
+// ffprobe calls an embedded thumbnail a video stream.
+const THUMBNAIL_CODECS = ['mjpeg', 'png', 'bmp', 'gif'];
+
+export const isJobFinished = (job) => !!job && FINISHED_STATUSES.includes(job.status);
+
+/** The codec names of a Video's real streams, from the `codec_names`/`codec_types` the API sends. */
+export function currentCodecs(video) {
+ const names = video?.codec_names || [];
+ const types = video?.codec_types || [];
+ const video_ = names.filter((name, i) => types[i] === 'video' && !THUMBNAIL_CODECS.includes(name));
+ const audio = names.filter((name, i) => types[i] === 'audio');
+ return {video: video_, audio};
+}
+
+/**
+ * The transcode target implied by the user's ordered codec preferences: the first preferred codec
+ * that ffmpeg can produce (the same rule the downloader uses). `copy` when none can be.
+ */
+export function preferredTarget(preferences, options) {
+ const producible = options.map(i => i.value).filter(i => i !== TRANSCODE_COPY);
+ for (const codec of preferences || []) {
+ if (producible.includes(codec)) {
+ return codec;
+ }
+ }
+ return TRANSCODE_COPY;
+}
+
+/** The container of a video (or audio) file, when it is one ffmpeg can write for us. */
+export function currentContainer(video) {
+ const suffix = String(video?.video_path || '').split('.').pop().toLowerCase();
+ const known = [...transcodeContainerOptions, ...transcodeAudioContainerOptions];
+ return known.some(i => i.value === suffix) ? suffix : null;
+}
+
+/**
+ * The container to offer for the chosen codecs: audio-only output gets the natural container of
+ * the audio codec that will be in it (the target, or the source's when copied); output with video
+ * keeps the file's own container when possible.
+ */
+export function containerFor({audioOutput, audioCodec, video}) {
+ if (audioOutput) {
+ const codec = audioCodec !== TRANSCODE_COPY ? audioCodec : currentCodecs(video).audio[0];
+ return audioContainerForCodec(codec);
+ }
+ const current = currentContainer(video);
+ return transcodeContainerOptions.some(i => i.value === current) ? current : 'mp4';
+}
+
+const statusKind = (status) => {
+ if (status === 'complete') return 'complete';
+ if (status === 'running') return 'active';
+ if (status === 'failed' || status === 'cancelled') return 'failed';
+ return 'pending';
+}
+
+/**
+ * Poll a Job until it finishes; `onFinished(job)` is called once. A failed request is retried;
+ * a 404 (the API restarted and forgot its Jobs) or `JOB_POLL_MAX_FAILURES` failures in a row stop
+ * the polling and call `onLost(error)` instead.
+ */
+export function useJob(jobId, onFinished, onLost) {
+ const [job, setJob] = useState(null);
+ const onFinishedRef = useRef(onFinished);
+ const onLostRef = useRef(onLost);
+ onFinishedRef.current = onFinished;
+ onLostRef.current = onLost;
+
+ useEffect(() => {
+ setJob(null);
+ if (!jobId) {
+ return;
+ }
+ let stopped = false;
+ let timer = null;
+ let failures = 0;
+
+ const poll = async () => {
+ let latest = null;
+ try {
+ latest = await getJob(jobId);
+ failures = 0;
+ } catch (e) {
+ console.error(e);
+ failures += 1;
+ if (!stopped && (e?.status === 404 || failures >= JOB_POLL_MAX_FAILURES)) {
+ if (onLostRef.current) onLostRef.current(e);
+ return;
+ }
+ }
+ if (stopped) return;
+ if (latest) {
+ setJob(latest);
+ if (isJobFinished(latest)) {
+ if (onFinishedRef.current) onFinishedRef.current(latest);
+ return;
+ }
+ }
+ timer = window.setTimeout(poll, JOB_POLL_MS);
+ };
+
+ poll();
+ return () => {
+ stopped = true;
+ clearTimeout(timer);
+ };
+ }, [jobId]);
+
+ const cancel = async () => {
+ const latest = await cancelJob(jobId);
+ setJob(latest);
+ return latest;
+ };
+
+ return {job, cancel};
+}
+
+/** Progress, status and log tail of one Job. */
+export function JobProgress({job, onCancel}) {
+ const [cancelling, setCancelling] = useState(false);
+
+ const handleCancel = async () => {
+ setCancelling(true);
+ try {
+ await onCancel();
+ } catch (e) {
+ console.error(e);
+ } finally {
+ setCancelling(false);
+ }
+ };
+
+ if (!job) {
+ return ;
+ }
+
+ const finished = isJobFinished(job);
+ const percent = job.progress ?? 0;
+ const logTail = (job.log || []).slice(-8);
+
+ return
+
+ {job.status}
+ {!finished && onCancel && }
+
+
+ {job.error && {job.error}}
+ {logTail.length > 0 &&
{logTail.join('\n')}
}
+
+}
+
+/**
+ * Choose target codecs for one Video and queue the transcode Job (`onQueued(jobId)`), or show the
+ * given `job`. The selects start at the user's preferred codecs from the Videos settings.
+ */
+export function TranscodeModal({
+ open, onClose, fileGroupId, video, audioOnly = false, job, jobId, onQueued,
+ onCancelJob,
+ }) {
+ const wrolModeEnabled = useWROLMode();
+ const [videoCodec, setVideoCodec] = useState(TRANSCODE_COPY);
+ const [audioCodec, setAudioCodec] = useState(TRANSCODE_COPY);
+ const [container, setContainer] = useState('mp4');
+ const [fragmented, setFragmented] = useState(false);
+ const [defaultsLoaded, setDefaultsLoaded] = useState(false);
+ const [submitting, setSubmitting] = useState(false);
+
+ const current = currentCodecs(video);
+ const removeVideo = videoCodec === TRANSCODE_REMOVE_VIDEO;
+ // An audio file has no video stream to keep or remove; removing the video makes an audio file.
+ const audioOutput = audioOnly || removeVideo;
+
+ useEffect(() => {
+ if (!open) {
+ return;
+ }
+ setDefaultsLoaded(false);
+ setFragmented(false);
+ let stale = false;
+ const load = async () => {
+ let nextVideo = TRANSCODE_COPY;
+ let nextAudio = TRANSCODE_COPY;
+ try {
+ const defaults = await fetchVideoDownloadDefaults();
+ if (stale) return;
+ nextVideo = audioOnly ? TRANSCODE_COPY
+ : preferredTarget(defaults?.video_codecs, transcodeVideoCodecOptions);
+ nextAudio = preferredTarget(defaults?.audio_codecs, transcodeAudioCodecOptions);
+ } catch (e) {
+ console.error(e);
+ } finally {
+ if (!stale) {
+ setVideoCodec(nextVideo);
+ setAudioCodec(nextAudio);
+ setContainer(containerFor({audioOutput: audioOnly, audioCodec: nextAudio, video}));
+ setDefaultsLoaded(true);
+ }
+ }
+ };
+ load();
+ return () => {
+ stale = true;
+ };
+ }, [open]);
+
+ const changeVideoCodec = (value) => {
+ const next = value || TRANSCODE_COPY;
+ setVideoCodec(next);
+ setContainer(containerFor({audioOutput: audioOnly || next === TRANSCODE_REMOVE_VIDEO, audioCodec, video}));
+ };
+ const changeAudioCodec = (value) => {
+ const next = value || TRANSCODE_COPY;
+ setAudioCodec(next);
+ setContainer(containerFor({audioOutput, audioCodec: next, video}));
+ };
+
+ const remuxOnly = !removeVideo && videoCodec === TRANSCODE_COPY && audioCodec === TRANSCODE_COPY;
+ const sameContainer = container === currentContainer(video);
+ const containerOptions = audioOutput ? transcodeAudioContainerOptions : transcodeContainerOptions;
+ // Fragments are an mp4 concept (m4a is an mp4).
+ const canFragment = container === 'mp4' || container === 'm4a';
+
+ const handleStart = async () => {
+ setSubmitting(true);
+ try {
+ const id = await transcodeVideo(fileGroupId, {
+ video_codec: videoCodec === TRANSCODE_COPY ? null : videoCodec,
+ audio_codec: audioCodec === TRANSCODE_COPY ? null : audioCodec,
+ container,
+ fragmented: canFragment && fragmented,
+ });
+ onQueued(id);
+ } catch (e) {
+ // transcodeVideo already toasted.
+ console.error(e);
+ } finally {
+ setSubmitting(false);
+ }
+ };
+
+ const describe = (codecs) => codecs.length ? codecs.join(', ') : 'unknown';
+ const showJob = !!jobId;
+
+ let hint = null;
+ if (defaultsLoaded && removeVideo) {
+ hint = `The video stream is removed; the result is an audio file (.${container}). `
+ + (audioCodec === TRANSCODE_COPY ? 'The audio is copied without re-encoding.' : `The audio is converted to ${audioCodec}.`);
+ } else if (defaultsLoaded && remuxOnly) {
+ const kept = audioOnly ? 'The audio is kept' : 'Both streams are kept';
+ if (sameContainer && canFragment && fragmented) {
+ hint = `${kept}: the file is rewritten as-is in fragments (a quick remux, no quality loss).`;
+ } else if (sameContainer) {
+ hint = `${kept}: the file is rewritten as-is` + (canFragment
+ ? ' with fast start (moves the index to the front for quicker playback start).' : '.');
+ } else {
+ hint = `${kept}: only the container changes to ${container} (a quick remux, no quality loss).`;
+ }
+ }
+ const buttonLabel = removeVideo ? 'Extract Audio' : remuxOnly ? 'Remux' : 'Transcode';
+
+ return
+ {audioOnly ? 'Transcode Audio' : 'Transcode Video'}
+
+
+
+ {audioOnly
+ ? `Current codec: audio ${describe(current.audio)}. `
+ : `Current codecs: video ${describe(current.video)}, audio ${describe(current.audio)}. `}
+ Streams set to Keep are copied without re-encoding. Transcoding is slow and slightly
+ reduces quality; the original file is replaced when it finishes.
+
+ {!showJob && <>
+ {!audioOnly && }
+
+
+
+
+
+ {!showJob && }
+
+
+}
+
+/**
+ * Edit a Video's details. The change is written to the Video's info json, so it survives a
+ * refresh; a re-download of the Video's metadata will overwrite it.
+ */
+export function EditVideoModal({open, onClose, fileGroupId, videoFile, description: currentDescription, onSaved}) {
+ const wrolModeEnabled = useWROLMode();
+ const [title, setTitle] = useState('');
+ const [description, setDescription] = useState('');
+ const [saving, setSaving] = useState(false);
+
+ useEffect(() => {
+ if (open) {
+ // Start from the current details on every open, discarding an abandoned edit.
+ setTitle(videoFile?.title || '');
+ setDescription(currentDescription || '');
+ }
+ }, [open, videoFile?.title, currentDescription]);
+
+ const trimmed = title.trim();
+ const titleChanged = trimmed !== (videoFile?.title || '');
+ const descriptionChanged = description !== (currentDescription || '');
+ const unchanged = !titleChanged && !descriptionChanged;
+
+ const handleSave = async () => {
+ setSaving(true);
+ try {
+ // Send only what changed, so an untouched field is never rewritten.
+ const changes = {};
+ if (titleChanged) changes.title = trimmed;
+ if (descriptionChanged) changes.description = description;
+ const updated = await updateVideo(fileGroupId, changes);
+ toast({type: 'success', title: 'Saved', description: 'The video was updated.', time: 3000});
+ if (onSaved) await onSaved(updated);
+ onClose();
+ } catch (e) {
+ // updateVideo already toasted.
+ console.error(e);
+ } finally {
+ setSaving(false);
+ }
+ };
+
+ return
+ Edit Video
+
+ {/* No submit-on-Enter: Enter belongs to the description's new lines. */}
+
+
+
+
+
+
+
+}
+
+/**
+ * The "Edit" dropdown on the Video page: Edit, Refresh, Transcode, Delete. Holds this video's
+ * active transcode Job; `onTranscodeComplete` fires whether or not the modal is open.
+ */
+export function VideoEditMenu({
+ videoFile, video, description, onRefresh, onTranscodeComplete, onDelete, onSaved,
+ initialJobId = null,
+ }) {
+ const wrolModeEnabled = useWROLMode();
+ const [editOpen, setEditOpen] = useState(false);
+ const [transcodeOpen, setTranscodeOpen] = useState(false);
+ const [refreshing, setRefreshing] = useState(false);
+ const [deleteOpen, setDeleteOpen] = useState(false);
+ const [deleting, setDeleting] = useState(false);
+ // `initialJobId`: resume watching a Job already queued for this video (tests use it too).
+ const [jobId, setJobId] = useState(initialJobId);
+
+ const handleJobFinished = (finished) => {
+ if (finished.status === 'complete') {
+ toast({type: 'success', title: 'Transcode complete', description: finished.description, time: 5000});
+ if (onTranscodeComplete) onTranscodeComplete(finished);
+ } else if (finished.status === 'failed') {
+ toast({
+ type: 'error', title: 'Transcode failed', time: 10000,
+ description: `${finished.description}: ${finished.error || 'unknown error'}. The original file was kept.`,
+ });
+ } else if (finished.status === 'cancelled') {
+ toast({type: 'info', title: 'Transcode cancelled', description: finished.description, time: 5000});
+ }
+ if (!transcodeOpen) {
+ // Nobody is looking at the result; make room for the next transcode.
+ setJobId(null);
+ }
+ };
+ const handleJobLost = (error) => {
+ toast({
+ type: 'warning', title: 'Lost track of the transcode', time: 10000,
+ description: 'The job is no longer known (the API may have restarted). Check the file before trying again.',
+ });
+ setJobId(null);
+ setTranscodeOpen(false);
+ };
+ const {job, cancel} = useJob(jobId, handleJobFinished, handleJobLost);
+
+ // Another video: its transcodes are its own.
+ const firstVideoId = useRef(videoFile?.id);
+ useEffect(() => {
+ if (videoFile?.id !== firstVideoId.current) {
+ firstVideoId.current = videoFile?.id;
+ setJobId(null);
+ }
+ }, [videoFile?.id]);
+
+ const handleTranscodeClose = () => {
+ setTranscodeOpen(false);
+ if (isJobFinished(job)) {
+ setJobId(null);
+ }
+ };
+
+ const handleDelete = async () => {
+ setDeleting(true);
+ try {
+ await onDelete();
+ setDeleteOpen(false);
+ } finally {
+ setDeleting(false);
+ }
+ };
+
+ const handleRefresh = async () => {
+ setRefreshing(true);
+ try {
+ await onRefresh();
+ } finally {
+ setRefreshing(false);
+ }
+ };
+
+ const isVideo = !!videoFile?.mimetype?.startsWith('video/');
+ const isAudio = !!videoFile?.mimetype?.startsWith('audio/');
+ const transcoding = !!jobId && !isJobFinished(job);
+
+ return <>
+
+
+ setEditOpen(false)}
+ fileGroupId={videoFile?.id}
+ videoFile={videoFile}
+ description={description}
+ onSaved={onSaved}
+ />
+ setDeleteOpen(false)}
+ >
+ Are you sure you want to delete this video? All files related to this video will be deleted.
+ It will not be downloaded again!
+
+ >
+}
diff --git a/app/src/components/VideoEditMenu.test.js b/app/src/components/VideoEditMenu.test.js
new file mode 100644
index 00000000..c74c4c59
--- /dev/null
+++ b/app/src/components/VideoEditMenu.test.js
@@ -0,0 +1,616 @@
+import React from 'react';
+import {act, fireEvent, screen, waitFor} from '@testing-library/react';
+import {renderWithProviders} from '../test-utils';
+import {
+ containerFor, currentCodecs, currentContainer, EditVideoModal, JobProgress, preferredTarget, TranscodeModal,
+ useJob, VideoEditMenu,
+} from './VideoEditMenu';
+import {
+ audioContainerForCodec, TRANSCODE_COPY, TRANSCODE_REMOVE_VIDEO, transcodeAudioCodecOptions,
+ transcodeVideoCodecOptions,
+} from './Vars';
+
+jest.mock('../api', () => ({
+ cancelJob: jest.fn(),
+ fetchVideoDownloadDefaults: jest.fn(),
+ getJob: jest.fn(),
+ transcodeVideo: jest.fn(),
+ updateVideo: jest.fn(),
+}));
+
+import {cancelJob, fetchVideoDownloadDefaults, getJob, transcodeVideo, updateVideo} from '../api';
+
+// The ui index re-exports `toast` from here; a spy lets tests see what the page tells the user.
+jest.mock('./ui/toast', () => ({...jest.requireActual('./ui/toast'), toast: jest.fn()}));
+import {toast} from './ui/toast';
+
+const video = {
+ codec_names: ['vp9', 'mjpeg', 'opus'],
+ codec_types: ['video', 'video', 'audio'],
+ video_path: 'videos/movie.webm',
+};
+const videoFile = {id: 7, url: 'https://example.com/watch?v=1', mimetype: 'video/webm', title: 'Old Title', video};
+const audio = {codec_names: ['aac'], codec_types: ['audio'], video_path: 'videos/song.m4a'};
+const audioFile = {id: 8, url: null, mimetype: 'audio/mp4', title: 'Song', video: audio};
+
+// Mantine's Select renders a hidden input next to the visible one, both tied to the label.
+const selectInput = (label) => screen.getAllByLabelText(label).find(i => i.type !== 'hidden');
+
+// `renderWithProviders`' `settings` option replaces the SettingsContext value; the WROL flag lives
+// one level down in its `settings`.
+const wrolMode = {settings: {settings: {wrol_mode: true}}};
+
+const pendingJob = {
+ id: 'transcode_video-abc', status: 'pending', progress: null, log: [], error: null, cancel_requested: false,
+ description: 'Transcode movie.webm',
+};
+const runningJob = {...pendingJob, status: 'running', progress: 40, log: ['out_time_us=1']};
+const completeJob = {...pendingJob, status: 'complete', progress: 100, log: ['done']};
+const failedJob = {...pendingJob, status: 'failed', progress: 12, error: 'ffmpeg exited with 1', log: ['boom']};
+const notFound = Object.assign(new Error('Job does not exist'), {status: 404});
+
+describe('helpers', () => {
+ test('currentCodecs ignores embedded thumbnail streams', () => {
+ expect(currentCodecs(video)).toEqual({video: ['vp9'], audio: ['opus']});
+ expect(currentCodecs(null)).toEqual({video: [], audio: []});
+ });
+
+ test('currentContainer reads the file suffix, only for containers we can write', () => {
+ expect(currentContainer({video_path: 'videos/a/movie.MP4'})).toBe('mp4');
+ expect(currentContainer({video_path: 'videos/a/movie.mkv'})).toBe('mkv');
+ expect(currentContainer({video_path: 'videos/a/song.m4a'})).toBe('m4a');
+ expect(currentContainer({video_path: 'videos/a/movie.webm'})).toBeNull();
+ expect(currentContainer(null)).toBeNull();
+ });
+
+ test('audioContainerForCodec mirrors the backend default', () => {
+ expect(audioContainerForCodec('aac')).toBe('m4a');
+ expect(audioContainerForCodec('mp3')).toBe('mp3');
+ expect(audioContainerForCodec('opus')).toBe('ogg');
+ expect(audioContainerForCodec('vorbis')).toBe('ogg');
+ expect(audioContainerForCodec(undefined)).toBe('m4a');
+ });
+
+ test('containerFor follows the audio codec for audio output, the file for video output', () => {
+ // Removing the video from a vp9/opus webm, copying the audio: opus lives in ogg.
+ expect(containerFor({audioOutput: true, audioCodec: TRANSCODE_COPY, video})).toBe('ogg');
+ expect(containerFor({audioOutput: true, audioCodec: 'aac', video})).toBe('m4a');
+ expect(containerFor({audioOutput: true, audioCodec: 'mp3', video})).toBe('mp3');
+ // Video output keeps a container we can write, else mp4.
+ expect(containerFor({audioOutput: false, audioCodec: 'aac', video})).toBe('mp4');
+ expect(containerFor({audioOutput: false, audioCodec: 'aac', video: {video_path: 'a.mkv'}})).toBe('mkv');
+ // An audio file's own suffix is not a video container.
+ expect(containerFor({audioOutput: false, audioCodec: TRANSCODE_COPY, video: audio})).toBe('mp4');
+ });
+
+ test('preferredTarget picks the first preference ffmpeg can produce', () => {
+ expect(preferredTarget(['vp9', 'h264'], transcodeVideoCodecOptions)).toBe('h264');
+ expect(preferredTarget(['vp9', 'av1'], transcodeVideoCodecOptions)).toBe(TRANSCODE_COPY);
+ expect(preferredTarget(['opus', 'aac'], transcodeAudioCodecOptions)).toBe('opus');
+ expect(preferredTarget([], transcodeAudioCodecOptions)).toBe(TRANSCODE_COPY);
+ expect(preferredTarget(undefined, transcodeAudioCodecOptions)).toBe(TRANSCODE_COPY);
+ });
+});
+
+describe('TranscodeModal', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: ['h264'], audio_codecs: ['vorbis', 'aac']});
+ transcodeVideo.mockResolvedValue('transcode_video-abc');
+ });
+
+ test('pre-populates from the preferred codecs and queues the Job', async () => {
+ const onQueued = jest.fn();
+ renderWithProviders(
+ ,
+ );
+
+ expect(screen.getByText(/Current codecs: video vp9, audio opus/)).toBeInTheDocument();
+
+ // The defaults arrive: h264 is producible, vorbis is not, so aac is the audio target.
+ await waitFor(() => expect(selectInput('Video codec')).toHaveValue('h264 (avc1)'));
+ expect(selectInput('Audio codec')).toHaveValue('aac');
+ expect(selectInput('Container')).toHaveValue('mp4');
+
+ fireEvent.click(screen.getByRole('button', {name: /Transcode/}));
+
+ await waitFor(() => expect(transcodeVideo).toHaveBeenCalledWith(7, {
+ video_codec: 'h264', audio_codec: 'aac', container: 'mp4', fragmented: false,
+ }));
+ await waitFor(() => expect(onQueued).toHaveBeenCalledWith('transcode_video-abc'));
+ });
+
+ test('shows the Job instead of the form once one is queued', () => {
+ renderWithProviders(
+ ,
+ );
+ expect(screen.queryAllByLabelText('Video codec')).toHaveLength(0);
+ expect(screen.getByText('running')).toBeInTheDocument();
+ expect(screen.getByRole('button', {name: 'Close'})).toBeInTheDocument();
+ expect(screen.queryByRole('button', {name: /Transcode/})).not.toBeInTheDocument();
+ });
+
+ test('keeping every stream is a remux into the chosen container', async () => {
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: [], audio_codecs: []});
+ // A webm cannot be kept (not a container we write), so the select starts at mp4: a change.
+ renderWithProviders(
+ ,
+ );
+
+ await waitFor(() => expect(selectInput('Container')).toHaveValue('mp4'));
+ expect(await screen.findByText(/only the container changes to mp4/)).toBeInTheDocument();
+
+ const button = screen.getByRole('button', {name: 'Remux'});
+ expect(button).toBeEnabled();
+ fireEvent.click(button);
+ await waitFor(() => expect(transcodeVideo).toHaveBeenCalledWith(7, {
+ video_codec: null, audio_codec: null, container: 'mp4', fragmented: false,
+ }));
+ });
+
+ test('Fragmented is offered for mp4 and sent with the request', async () => {
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: [], audio_codecs: []});
+ const mp4 = {...video, video_path: 'videos/movie.mp4'};
+ renderWithProviders(
+ ,
+ );
+ await waitFor(() => expect(selectInput('Container')).toHaveValue('mp4'));
+ const toggle = screen.getByRole('switch', {name: /Fragmented/});
+ expect(toggle).not.toBeChecked();
+
+ fireEvent.click(toggle);
+ expect(await screen.findByText(/rewritten as-is in fragments/)).toBeInTheDocument();
+ expect(screen.queryByText(/fast start/)).not.toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole('button', {name: 'Remux'}));
+ await waitFor(() => expect(transcodeVideo).toHaveBeenCalledWith(7, {
+ video_codec: null, audio_codec: null, container: 'mp4', fragmented: true,
+ }));
+ });
+
+ test('Fragmented is not offered for mkv', async () => {
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: [], audio_codecs: []});
+ const mkv = {...video, video_path: 'videos/movie.mkv'};
+ renderWithProviders();
+ await waitFor(() => expect(selectInput('Container')).toHaveValue('mkv'));
+ expect(screen.queryByRole('switch', {name: /Fragmented/})).not.toBeInTheDocument();
+ });
+
+ test('a remux into the same container is described as a fast-start rewrite', async () => {
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: [], audio_codecs: []});
+ const mkv = {...video, video_path: 'videos/movie.mkv'};
+ renderWithProviders();
+
+ await waitFor(() => expect(selectInput('Container')).toHaveValue('mkv'));
+ // mkv has no fast start; the hint must not promise one.
+ expect(await screen.findByText(/Both streams are kept: the file is rewritten as-is\./)).toBeInTheDocument();
+ expect(screen.queryByText(/fast start/)).not.toBeInTheDocument();
+ expect(screen.getByRole('button', {name: 'Remux'})).toBeEnabled();
+ });
+
+ test('an audio file offers audio codecs and containers only', async () => {
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: ['h264'], audio_codecs: []});
+ renderWithProviders(
+ ,
+ );
+
+ expect(screen.getByText('Transcode Audio')).toBeInTheDocument();
+ expect(screen.getByText(/Current codec: audio aac/)).toBeInTheDocument();
+ // The aac source, copied, belongs in m4a; the file already is one.
+ await waitFor(() => expect(selectInput('Container')).toHaveValue('m4a (aac)'));
+ expect(screen.queryAllByLabelText('Video codec')).toHaveLength(0);
+ expect(await screen.findByText(/The audio is kept: the file is rewritten as-is with fast start/))
+ .toBeInTheDocument();
+ expect(screen.queryByText(/Both streams/)).not.toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole('button', {name: 'Remux'}));
+ // No video codec is sent for an audio file, whatever the video preferences say.
+ await waitFor(() => expect(transcodeVideo).toHaveBeenCalledWith(8, {
+ video_codec: null, audio_codec: null, container: 'm4a', fragmented: false,
+ }));
+ });
+
+ test('the Remove video option is offered for videos', async () => {
+ renderWithProviders(
+ ,
+ );
+ await waitFor(() => expect(selectInput('Video codec')).toHaveValue('h264 (avc1)'));
+ expect(transcodeVideoCodecOptions.some(i => i.value === TRANSCODE_REMOVE_VIDEO)).toBe(true);
+ // Selecting it is exercised through containerFor (Mantine's dropdown is not driveable in jsdom).
+ });
+
+ test('WROL Mode disables starting a transcode', async () => {
+ renderWithProviders(
+ ,
+ wrolMode,
+ );
+ await waitFor(() => expect(selectInput('Video codec')).toHaveValue('h264 (avc1)'));
+ expect(screen.getByRole('button', {name: /Transcode/})).toBeDisabled();
+ });
+});
+
+describe('EditVideoModal', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ updateVideo.mockResolvedValue({...videoFile, title: 'New Title'});
+ });
+
+ test('starts from the current title and saves a changed one', async () => {
+ const onSaved = jest.fn().mockResolvedValue(undefined);
+ const onClose = jest.fn();
+ renderWithProviders(
+ ,
+ );
+
+ const input = screen.getByLabelText('Title');
+ expect(input).toHaveValue('Old Title');
+ expect(screen.getByLabelText('Description')).toHaveValue('Old words');
+ // Nothing changed yet: Save is disabled.
+ expect(screen.getByRole('button', {name: /Save/})).toBeDisabled();
+
+ fireEvent.change(input, {target: {value: ' New Title '}});
+ fireEvent.click(screen.getByRole('button', {name: /Save/}));
+
+ // Only the changed field is sent.
+ await waitFor(() => expect(updateVideo).toHaveBeenCalledWith(7, {title: 'New Title'}));
+ await waitFor(() => expect(onSaved).toHaveBeenCalledTimes(1));
+ expect(onClose).toHaveBeenCalledTimes(1);
+ });
+
+ test('saves a changed description on its own, including clearing it', async () => {
+ renderWithProviders(
+ ,
+ );
+ fireEvent.change(screen.getByLabelText('Description'), {target: {value: ''}});
+ fireEvent.click(screen.getByRole('button', {name: /Save/}));
+ await waitFor(() => expect(updateVideo).toHaveBeenCalledWith(7, {description: ''}));
+ });
+
+ test('WROL Mode disables saving', async () => {
+ renderWithProviders(
+ ,
+ wrolMode,
+ );
+ fireEvent.change(screen.getByLabelText('Title'), {target: {value: 'Changed'}});
+ expect(screen.getByRole('button', {name: /Save/})).toBeDisabled();
+ expect(screen.getByText(/WROL Mode/)).toBeInTheDocument();
+ });
+
+ test('an empty title cannot be saved', () => {
+ renderWithProviders();
+ fireEvent.change(screen.getByLabelText('Title'), {target: {value: ' '}});
+ expect(screen.getByText('A title is required')).toBeInTheDocument();
+ expect(screen.getByRole('button', {name: /Save/})).toBeDisabled();
+ expect(updateVideo).not.toHaveBeenCalled();
+ });
+
+ test('a failed save keeps the modal open', async () => {
+ updateVideo.mockRejectedValue(new Error('boom'));
+ const onClose = jest.fn();
+ renderWithProviders();
+ fireEvent.change(screen.getByLabelText('Title'), {target: {value: 'Other'}});
+ fireEvent.click(screen.getByRole('button', {name: /Save/}));
+ await waitFor(() => expect(updateVideo).toHaveBeenCalled());
+ expect(onClose).not.toHaveBeenCalled();
+ expect(screen.getByLabelText('Title')).toHaveValue('Other');
+ });
+});
+
+// A tiny harness so the hook can be driven like the menu drives it.
+function JobHarness({jobId, onFinished, onLost}) {
+ const {job, cancel} = useJob(jobId, onFinished, onLost);
+ return ;
+}
+
+// Let pending promise callbacks run (a settled request schedules its next poll), then advance
+// the fake clock past the poll interval.
+const nextPoll = async () => {
+ await act(async () => {
+ await Promise.resolve();
+ await Promise.resolve();
+ jest.advanceTimersByTime(2000);
+ });
+};
+
+describe('useJob + JobProgress', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ jest.useFakeTimers();
+ });
+
+ afterEach(() => {
+ jest.useRealTimers();
+ });
+
+ test('polls until the Job finishes, then reports once', async () => {
+ const onFinished = jest.fn();
+ getJob.mockResolvedValueOnce(runningJob).mockResolvedValueOnce(completeJob);
+
+ renderWithProviders();
+
+ await waitFor(() => expect(screen.getByText('running')).toBeInTheDocument());
+ expect(screen.getByRole('progressbar')).toHaveAttribute('aria-valuenow', '40');
+ expect(screen.getByTestId('job-log')).toHaveTextContent('out_time_us=1');
+ expect(screen.getByRole('button', {name: 'Cancel'})).toBeInTheDocument();
+
+ await act(async () => {
+ jest.advanceTimersByTime(2000);
+ });
+
+ await waitFor(() => expect(screen.getByText('complete')).toBeInTheDocument());
+ expect(onFinished).toHaveBeenCalledTimes(1);
+ expect(onFinished.mock.calls[0][0].status).toBe('complete');
+ // A finished Job is not polled again, and cannot be cancelled.
+ expect(getJob).toHaveBeenCalledTimes(2);
+ expect(screen.queryByRole('button', {name: 'Cancel'})).not.toBeInTheDocument();
+ });
+
+ test('a failed request is retried instead of ending the polling', async () => {
+ getJob
+ .mockRejectedValueOnce(new Error('network'))
+ .mockResolvedValueOnce(runningJob)
+ .mockRejectedValueOnce(new Error('network'))
+ .mockResolvedValueOnce(completeJob);
+ const onFinished = jest.fn();
+
+ renderWithProviders();
+
+ // First poll failed: still "Starting...", but another poll is scheduled.
+ await waitFor(() => expect(getJob).toHaveBeenCalledTimes(1));
+ expect(screen.getByText('Starting...')).toBeInTheDocument();
+ await nextPoll();
+ await waitFor(() => expect(screen.getByText('running')).toBeInTheDocument());
+
+ // A failure mid-way keeps the last good state and keeps polling.
+ await nextPoll();
+ await waitFor(() => expect(getJob).toHaveBeenCalledTimes(3));
+ expect(screen.getByText('running')).toBeInTheDocument();
+ await nextPoll();
+ await waitFor(() => expect(screen.getByText('complete')).toBeInTheDocument());
+ expect(onFinished).toHaveBeenCalledTimes(1);
+ });
+
+ test('a 404 means the API forgot the Job: give up at once', async () => {
+ getJob.mockRejectedValue(notFound);
+ const onLost = jest.fn();
+ const onFinished = jest.fn();
+ renderWithProviders();
+
+ await waitFor(() => expect(onLost).toHaveBeenCalledTimes(1));
+ expect(onLost.mock.calls[0][0].status).toBe(404);
+ await nextPoll();
+ expect(getJob).toHaveBeenCalledTimes(1);
+ expect(onFinished).not.toHaveBeenCalled();
+ });
+
+ test('repeated failures give up after a few polls', async () => {
+ getJob.mockRejectedValue(new Error('network'));
+ const onLost = jest.fn();
+ renderWithProviders();
+
+ for (let i = 0; i < 6; i++) {
+ await nextPoll();
+ }
+ await waitFor(() => expect(onLost).toHaveBeenCalledTimes(1));
+ expect(getJob).toHaveBeenCalledTimes(5);
+ });
+
+ test('Cancel asks the API to cancel the Job', async () => {
+ getJob.mockResolvedValue({...pendingJob, status: 'running', progress: 10});
+ cancelJob.mockResolvedValue({...pendingJob, status: 'running', progress: 10, cancel_requested: true});
+
+ renderWithProviders();
+
+ await waitFor(() => expect(screen.getByRole('button', {name: 'Cancel'})).toBeInTheDocument());
+ fireEvent.click(screen.getByRole('button', {name: 'Cancel'}));
+
+ await waitFor(() => expect(cancelJob).toHaveBeenCalledWith('transcode_video-abc'));
+ await waitFor(() => expect(screen.getByRole('button', {name: 'Cancelling...'})).toBeDisabled());
+ });
+
+ test('shows the error of a failed Job', async () => {
+ getJob.mockResolvedValue({...pendingJob, status: 'failed', error: 'ffmpeg exited with 1', log: ['boom']});
+
+ renderWithProviders();
+
+ await waitFor(() => expect(screen.getByText('failed')).toBeInTheDocument());
+ expect(screen.getByText('ffmpeg exited with 1')).toBeInTheDocument();
+ });
+});
+
+describe('VideoEditMenu', () => {
+ beforeEach(() => {
+ jest.clearAllMocks();
+ fetchVideoDownloadDefaults.mockResolvedValue({video_codecs: ['h264'], audio_codecs: ['aac']});
+ transcodeVideo.mockResolvedValue('transcode_video-abc');
+ });
+
+ // Mantine's Menu in jsdom: after a modal has closed, the first click on the target can land
+ // while the dropdown still counts as open (and so toggles it shut). Click until items show.
+ const openMenu = async () => {
+ for (let attempt = 0; attempt < 3; attempt++) {
+ fireEvent.click(screen.getByRole('button', {name: /Edit/}));
+ try {
+ return await screen.findByRole('menuitem', {name: /Refresh/}, {timeout: 300});
+ } catch (e) {
+ // Toggled shut; try again.
+ }
+ }
+ throw new Error('The Edit menu did not open');
+ };
+ const openMenuItem = async (name) => {
+ await openMenu();
+ fireEvent.click(screen.getByRole('menuitem', {name}));
+ };
+
+ test('Refresh calls onRefresh; Transcode opens the modal', async () => {
+ const onRefresh = jest.fn().mockResolvedValue(undefined);
+ renderWithProviders();
+
+ await openMenuItem(/Refresh/);
+ await waitFor(() => expect(onRefresh).toHaveBeenCalledTimes(1));
+
+ await openMenuItem(/Transcode/);
+ await waitFor(() => expect(screen.getByText('Transcode Video')).toBeInTheDocument());
+ });
+
+ test('Edit opens the details modal', async () => {
+ renderWithProviders();
+ await openMenuItem(/^Edit/);
+ expect(await screen.findByText('Edit Video')).toBeInTheDocument();
+ expect(screen.getByLabelText('Title')).toHaveValue('Old Title');
+ });
+
+ test('Refresh is unavailable without a URL; audio files can be transcoded', async () => {
+ renderWithProviders();
+
+ fireEvent.click(screen.getByRole('button', {name: /Edit/}));
+ const refresh = await screen.findByRole('menuitem', {name: /Refresh/});
+ expect(refresh).toBeDisabled();
+ expect(screen.getByRole('menuitem', {name: /Transcode/})).toBeEnabled();
+ fireEvent.click(screen.getByRole('menuitem', {name: /Transcode/}));
+ expect(await screen.findByText('Transcode Audio')).toBeInTheDocument();
+ });
+
+ test('Transcode is unavailable for files that are neither video nor audio', async () => {
+ const other = {...videoFile, mimetype: 'application/pdf'};
+ renderWithProviders();
+ fireEvent.click(screen.getByRole('button', {name: /Edit/}));
+ expect(await screen.findByRole('menuitem', {name: /Transcode/})).toBeDisabled();
+ });
+
+ test('WROL Mode disables every item', async () => {
+ renderWithProviders(
+ ,
+ wrolMode,
+ );
+ fireEvent.click(screen.getByRole('button', {name: /Edit/}));
+ expect(await screen.findByRole('menuitem', {name: /Refresh/})).toBeDisabled();
+ expect(screen.getByRole('menuitem', {name: /Transcode/})).toBeDisabled();
+ expect(screen.getByRole('menuitem', {name: /Delete/})).toBeDisabled();
+ expect(screen.getByRole('menuitem', {name: /^Edit/})).toBeDisabled();
+ });
+
+ test('Delete asks for confirmation before calling onDelete', async () => {
+ const onDelete = jest.fn().mockResolvedValue(undefined);
+ renderWithProviders(
+ ,
+ );
+
+ await openMenuItem(/Delete/);
+ expect(await screen.findByText('Delete video?')).toBeInTheDocument();
+ expect(onDelete).not.toHaveBeenCalled();
+
+ fireEvent.click(await screen.findByRole('button', {name: 'Delete'}));
+ await waitFor(() => expect(onDelete).toHaveBeenCalledTimes(1));
+ });
+
+ test('backing out of the Delete confirmation deletes nothing', async () => {
+ const onDelete = jest.fn();
+ renderWithProviders(
+ ,
+ );
+
+ await openMenuItem(/Delete/);
+ expect(await screen.findByText('Delete video?')).toBeInTheDocument();
+
+ fireEvent.click(screen.getByRole('button', {name: 'Cancel'}));
+ await waitFor(() => expect(screen.queryByText('Delete video?')).not.toBeInTheDocument());
+ expect(onDelete).not.toHaveBeenCalled();
+ });
+
+ test('closing the modal keeps watching the Job and reports completion to the page', async () => {
+ jest.useFakeTimers();
+ try {
+ getJob.mockResolvedValueOnce(runningJob).mockResolvedValueOnce(completeJob);
+ const onTranscodeComplete = jest.fn();
+ renderWithProviders(
+ ,
+ );
+
+ await openMenuItem(/Transcode/);
+ await waitFor(() => expect(selectInput('Video codec')).toHaveValue('h264 (avc1)'));
+ fireEvent.click(screen.getByRole('button', {name: 'Transcode'}));
+ await waitFor(() => expect(screen.getByText('running')).toBeInTheDocument());
+
+ // Close while it runs. The modal is gone but the Job is still watched.
+ fireEvent.click(screen.getByRole('button', {name: 'Close'}));
+ await waitFor(() => expect(screen.queryByText('Transcode Video')).not.toBeInTheDocument());
+
+ await nextPoll();
+ await waitFor(() => expect(onTranscodeComplete).toHaveBeenCalledTimes(1));
+ expect(onTranscodeComplete.mock.calls[0][0].status).toBe('complete');
+ expect(getJob).toHaveBeenCalledTimes(2);
+ } finally {
+ jest.useRealTimers();
+ }
+ });
+
+ test('while a Job runs, Transcode shows its progress and Delete waits', async () => {
+ getJob.mockResolvedValue(runningJob);
+ renderWithProviders(
+ ,
+ );
+ await waitFor(() => expect(getJob).toHaveBeenCalled());
+
+ await openMenu();
+ expect(screen.getByRole('menuitem', {name: 'Transcoding...'})).toBeEnabled();
+ expect(screen.getByRole('menuitem', {name: /Delete/})).toBeDisabled();
+
+ fireEvent.click(screen.getByRole('menuitem', {name: 'Transcoding...'}));
+ expect(await screen.findByText('running')).toBeInTheDocument();
+ expect(screen.queryAllByLabelText('Video codec')).toHaveLength(0);
+ expect(transcodeVideo).not.toHaveBeenCalled();
+ });
+
+ test('a Job that fails after the modal was closed still tells the user', async () => {
+ getJob.mockResolvedValue(failedJob);
+ renderWithProviders(
+ ,
+ );
+ await waitFor(() => expect(toast).toHaveBeenCalledTimes(1));
+ expect(toast.mock.calls[0][0]).toMatchObject({type: 'error', title: 'Transcode failed'});
+ expect(toast.mock.calls[0][0].description).toMatch(/ffmpeg exited with 1/);
+ expect(toast.mock.calls[0][0].description).toMatch(/original file was kept/);
+
+ // Forgotten: the menu is unlocked again.
+ await openMenu();
+ expect(screen.getByRole('menuitem', {name: 'Transcode...'})).toBeEnabled();
+ expect(screen.getByRole('menuitem', {name: /Delete/})).toBeEnabled();
+ });
+
+ test('a Job the API no longer knows unlocks the menu with a warning', async () => {
+ getJob.mockRejectedValue(notFound);
+ renderWithProviders(
+ ,
+ );
+ await waitFor(() => expect(toast).toHaveBeenCalledTimes(1));
+ expect(toast.mock.calls[0][0]).toMatchObject({type: 'warning', title: 'Lost track of the transcode'});
+
+ await openMenu();
+ expect(screen.getByRole('menuitem', {name: 'Transcode...'})).toBeEnabled();
+ expect(screen.getByRole('menuitem', {name: /Delete/})).toBeEnabled();
+ });
+
+ test('a Job that finished unwatched is forgotten, so the next open offers the form', async () => {
+ getJob.mockResolvedValue(completeJob);
+ renderWithProviders(
+ ,
+ );
+ await waitFor(() => expect(getJob).toHaveBeenCalled());
+
+ await openMenu();
+ expect(await screen.findByRole('menuitem', {name: 'Transcode...'})).toBeEnabled();
+ expect(screen.getByRole('menuitem', {name: /Delete/})).toBeEnabled();
+ fireEvent.click(screen.getByRole('menuitem', {name: 'Transcode...'}));
+ await waitFor(() => expect(selectInput('Video codec')).toBeInTheDocument());
+ });
+});
diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js
index 5b643cb1..d957f773 100644
--- a/app/src/components/VideoPlayer.js
+++ b/app/src/components/VideoPlayer.js
@@ -24,6 +24,7 @@ import {VideoCard} from "./Videos";
import {TagsSelector} from "../Tags";
import {AddToPlaylistButton} from "./AddToPlaylist";
import {TaggedDeleteConfirmModal} from "./TaggedDeleteConfirmModal";
+import {VideoEditMenu} from "./VideoEditMenu";
import {IconPlayerPause, IconPlayerPlay} from "@tabler/icons-react";
const MEDIA_PATH = '/media';
@@ -230,7 +231,7 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
// Get the Video's channel, fallback to the URL's channel id.
const {channelId} = useParams();
const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId);
- const {comments, description: videoDescription} = useVideoExtras(videoFile?.id);
+ const {comments, description: videoDescription, fetchDescription} = useVideoExtras(videoFile?.id);
const {captions, captionsLoading, fetchCaptions} = useVideoCaptions(videoFile?.id);
const mediaTitle = videoFile ? (videoFile.title || videoFile.stem || videoFile.name) : null;
@@ -488,19 +489,18 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
- await handleDeleteVideo(videoFile.id)}
- obeyWROLMode={true}
- >Delete
- Refresh
+ await handleDeleteVideo(videoFile.id)}
+ description={videoDescription}
+ onSaved={async () => {
+ await fetchVideo();
+ await fetchDescription();
+ }}
+ />
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index f6ab0065..5fa8a8fb 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -771,7 +771,7 @@ export const useVideoExtras = (fileGroupId) => {
}
}, [fileGroupId]);
- return {comments, description}
+ return {comments, description, fetchDescription}
}
export const useVideoCaptions = (fileGroupId) => {
diff --git a/modules/videos/common.py b/modules/videos/common.py
index 84e86b75..1d8cadc7 100644
--- a/modules/videos/common.py
+++ b/modules/videos/common.py
@@ -149,31 +149,70 @@ def ffmpeg_poster(video_path: Path, poster_path: Path, seconds: int):
return stderr
-def ffmpeg_video_complete(video_path: Path, seconds: int = None) -> bool:
- """Checks if video file is complete by taking screenshot from the end of the video.
+# How much of a video's tail is decoded to prove the file is complete. Corruption from an
+# interrupted download or write lands at the end of the file; decoding all of it would take as
+# long as playing it.
+TAIL_DECODE_SECONDS = 30
+TAIL_DECODE_TIMEOUT = 10 * 60
- @raise FileNotFoundError: raised if the video file does not exist.
- """
+
+def tail_decode_command(video_path: Path, seconds: int = None) -> Tuple[str | Path, ...]:
+ """ffmpeg command which decodes the last `seconds` of `video_path` to nothing, printing only
+ errors. `-sseof` seeks from the end; a file shorter than `seconds` is decoded whole."""
+ seconds = seconds or TAIL_DECODE_SECONDS
+ return FFMPEG_BIN, '-v', 'error', '-sseof', f'-{int(seconds)}', '-i', video_path, '-f', 'null', '-'
+
+
+def _tail_decode_verdict(return_code: int, stderr: bytes | str) -> Tuple[bool, str]:
+ errors = (stderr.decode(errors='replace') if isinstance(stderr, bytes) else (stderr or '')).strip()
+ ok = return_code == 0 and not errors
+ return ok, errors
+
+
+def _check_video_file(video_path: Path):
if not video_path.is_file():
raise FileNotFoundError(f'Video file not found: {video_path}')
if not video_path.stat().st_size:
raise RuntimeError(f'Video file is empty: {video_path}')
+ if not FFMPEG_BIN:
+ raise RuntimeError('ffmpeg was not found')
- with tempfile.NamedTemporaryFile(suffix='.jpg', delete=False) as fh:
- path = Path(fh.name)
- path.unlink()
- try:
- seconds = seconds or extract_video_duration(video_path) - 5
- ffmpeg_poster(video_path, path, seconds)
- return True
- except subprocess.CalledProcessError:
- return False
- except TypeError:
- # "N/A" duration.
- return False
- finally:
- if path.is_file():
- path.unlink()
+
+def ffmpeg_video_complete(video_path: Path, seconds: int = None) -> bool:
+ """True if the last `seconds` of the video decode without error.
+
+ Used after a download, and after a transcode before the original is replaced. A truncated or
+ corrupt file fails here because its damage is at the end; the index (moov) must also be
+ readable for ffmpeg to seek from the end at all.
+
+ @raise FileNotFoundError: raised if the video file does not exist.
+ @raise RuntimeError: raised if the video file is empty, or ffmpeg is missing.
+ """
+ _check_video_file(video_path)
+ cmd = tuple(str(i) for i in tail_decode_command(video_path, seconds))
+ try:
+ proc = subprocess.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, timeout=TAIL_DECODE_TIMEOUT)
+ except subprocess.TimeoutExpired:
+ logger.error(f'Timed out checking video is complete: {video_path}')
+ return False
+ ok, errors = _tail_decode_verdict(proc.returncode, proc.stderr)
+ if not ok:
+ logger.warning(f'Video is incomplete or corrupt: {video_path} (exit {proc.returncode}) {errors[-500:]}')
+ return ok
+
+
+async def ffmpeg_video_complete_async(video_path: Path, seconds: int = None, runner=None) -> Tuple[bool, str]:
+ """`ffmpeg_video_complete` for async callers; returns (complete, ffmpeg's error output).
+
+ `runner` defaults to `run_command`; a Job passes its own so ffmpeg's output lands in the Job log.
+ """
+ _check_video_file(video_path)
+ runner = runner or run_command
+ result = await runner(tail_decode_command(video_path, seconds), cwd=video_path.parent, timeout=TAIL_DECODE_TIMEOUT)
+ ok, errors = _tail_decode_verdict(result.return_code, result.stderr)
+ if not ok:
+ logger.warning(f'Video is incomplete or corrupt: {video_path} (exit {result.return_code}) {errors[-500:]}')
+ return ok, errors
def generate_video_poster(video_path: Path, seconds: int = 5) -> Tuple[Path, Optional[int]]:
diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 39636ab5..789febb7 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -887,6 +887,7 @@ class PreparedVideo:
destination: Optional[pathlib.Path] = None
info_json: Optional[dict] = None
location: Optional[str] = None # existing video, for error reports
+ wrolpi: dict = field(default_factory=dict) # existing video's wrolpi section, to carry over
@dataclass
@@ -907,6 +908,7 @@ class ExecutedVideo:
tag_names: List[str]
settings: dict
location: Optional[str] # for error reporting
+ wrolpi: dict = field(default_factory=dict) # existing video's wrolpi section, to carry over
class VideoDownloader(Downloader, ABC):
@@ -954,6 +956,8 @@ class VideoDownloader(Downloader, ABC):
# can still tell the user where the previous attempt landed.
existing = Video.get_by_url(session, url)
location = existing.location if existing else None
+ # yt-dlp rewrites the whole info json; the user's edits (its wrolpi section) must survive.
+ wrolpi = existing.file_group.get_wrolpi_json() if existing else dict()
return PreparedVideo(
url=url,
@@ -962,6 +966,7 @@ class VideoDownloader(Downloader, ABC):
destination=download.destination,
info_json=download.info_json,
location=location,
+ wrolpi=wrolpi,
)
async def execute_download(self, prepared: PreparedVideo, ctx: DownloadContext,
@@ -1201,6 +1206,7 @@ class VideoDownloader(Downloader, ABC):
tag_names=prepared.tag_names,
settings=settings,
location=location,
+ wrolpi=prepared.wrolpi,
)
except BotBlockedDownloadError:
@@ -1238,6 +1244,11 @@ class VideoDownloader(Downloader, ABC):
video.channel_id = executed.channel_id
if video.info_json_path and (new_info_json := video.clean_info_json()):
video.replace_info_json(new_info_json, clean=False)
+ if executed.wrolpi and video.info_json_path:
+ # The re-downloaded info json lost the user's edits; put them back, then derive the
+ # title/description again so the custom ones win (from_paths validated without them).
+ video.file_group.update_wrolpi_json(executed.wrolpi)
+ video.validate(session)
if parent_download_url := executed.settings.get('parent_download_url'):
try:
@@ -1368,9 +1379,12 @@ class VideoDownloader(Downloader, ABC):
settings = download.settings or dict()
with get_db_session(commit=True) as session:
- # Find the existing video, replace its info json.
+ # Find the existing video, replace its info json; keep the user's edits (wrolpi section).
video = Video.get_by_url(session, url)
+ wrolpi = video.file_group.get_wrolpi_json()
video.replace_info_json(info_json)
+ if wrolpi:
+ video.file_group.update_wrolpi_json(wrolpi)
if parent_download_url := settings.get('parent_download_url'):
try:
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 0f16648e..072d1e5e 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -58,7 +58,9 @@ def extract_video_info_json(video: Video) -> VideoInfoJSON:
"""
video_info_json = VideoInfoJSON()
if info_json := video.get_info_json():
- title = info_json.get('fulltitle') or info_json.get('title')
+ # A title the user set (Edit > Edit) is kept apart from yt-dlp's keys, and wins.
+ wrolpi_section = info_json.get('wrolpi') or dict()
+ title = wrolpi_section.get('custom_title') or info_json.get('fulltitle') or info_json.get('title')
video_info_json.title = html.unescape(title) if title else None
upload_date = dates.strpdate(i) if (i := info_json.get('upload_date')) else None
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 22cff5a8..750a4d5b 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -141,10 +141,16 @@ class Video(ModelHelper, Base):
def get_video_description(self) -> Optional[str]:
"""
- Get the Video description from the file system.
+ Get the Video description from the file system. A description the user set (Edit > Edit)
+ lives in the info json's `wrolpi` section and wins; an empty one means "none".
"""
- if (info_json := self.get_info_json()) and (description := info_json.get('description')):
- return description
+ info_json = self.get_info_json()
+ if not info_json:
+ return None
+ wrolpi_section = info_json.get('wrolpi') or dict()
+ if 'custom_description' in wrolpi_section:
+ return wrolpi_section['custom_description'] or None
+ return info_json.get('description') or None
def get_description(self) -> Optional[str]:
"""The indexed description (`FileGroup.c_text`), falling back to the info json on disk.
diff --git a/modules/videos/schema.py b/modules/videos/schema.py
index 579a6337..7296d8db 100644
--- a/modules/videos/schema.py
+++ b/modules/videos/schema.py
@@ -186,6 +186,26 @@ class VideoSearchRequest:
deep: bool = False # When True, search all text including captions (d_text)
+@dataclass
+class VideoUpdateRequest:
+ # Only the fields given are changed.
+ title: Optional[str] = None
+ description: Optional[str] = None
+
+
+@dataclass
+class VideoTranscodeRequest:
+ video_codec: Optional[str] = None # A key of TRANSCODE_VIDEO_TARGETS, or None to copy the stream.
+ audio_codec: Optional[str] = None # A key of TRANSCODE_AUDIO_TARGETS, or None to copy the stream.
+ container: str = 'mp4'
+ fragmented: bool = False # Fragmented mp4: quick start/seek for very long videos (mp4/m4a only).
+
+
+@dataclass
+class VideoTranscodeResponse:
+ job_id: str
+
+
@dataclass
class VideoSearchResponse:
videos: List[VideoWithChannel]
diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py
index 0e66fcf1..4045f731 100644
--- a/modules/videos/test/test_common.py
+++ b/modules/videos/test/test_common.py
@@ -644,3 +644,38 @@ async def test_video_modeler_does_not_hold_write_lock_during_ffprobe(async_clien
assert lock_held_during_ffprobe, 'ffprobe never ran, so the test proves nothing'
assert not any(lock_held_during_ffprobe), \
'the write lock was held while ffprobe ran; every other writer waits out busy_timeout'
+
+
+def test_ffmpeg_video_complete_real_files(test_directory, video_file):
+ """A whole video passes; a truncated copy fails. Uses the real ffmpeg on the example video."""
+ from modules.videos.common import ffmpeg_video_complete, tail_decode_command
+ assert ffmpeg_video_complete(video_file) is True
+
+ truncated = test_directory / 'truncated.mp4'
+ data = video_file.read_bytes()
+ truncated.write_bytes(data[:int(len(data) * 0.6)])
+ assert ffmpeg_video_complete(truncated) is False
+
+ empty = test_directory / 'empty.mp4'
+ empty.write_bytes(b'')
+ with pytest.raises(RuntimeError, match='empty'):
+ ffmpeg_video_complete(empty)
+ with pytest.raises(FileNotFoundError):
+ ffmpeg_video_complete(test_directory / 'missing.mp4')
+
+ cmd = tail_decode_command(video_file)
+ assert cmd[cmd.index('-sseof') + 1] == '-30' and cmd[cmd.index('-f') + 1] == 'null' and cmd[-1] == '-'
+
+
+@pytest.mark.asyncio
+async def test_ffmpeg_video_complete_async_real_files(test_directory, video_file):
+ """The async variant reaches the same verdicts and returns ffmpeg's error text."""
+ from modules.videos.common import ffmpeg_video_complete_async
+ ok, errors = await ffmpeg_video_complete_async(video_file)
+ assert ok is True and errors == ''
+
+ truncated = test_directory / 'truncated.mp4'
+ data = video_file.read_bytes()
+ truncated.write_bytes(data[:int(len(data) * 0.6)])
+ ok, errors = await ffmpeg_video_complete_async(truncated)
+ assert ok is False and errors
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 31fa25aa..0af65528 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -1789,3 +1789,76 @@ async def test_enforce_codecs():
with pytest.raises(UnrecoverableDownloadError):
await enforce_codecs(video_path, video_paths, effective)
mock_transcode.assert_not_called()
+
+
+@pytest.mark.asyncio
+async def test_metadata_refresh_keeps_wrolpi_section(test_session, test_directory, video_factory):
+ """Refreshing a Video's metadata replaces yt-dlp's info json; the wrolpi section (the user's
+ custom title and description) is carried over and still wins."""
+ source_id = 'video-source-id'
+ url = 'https://example.com/the-video'
+ video = video_factory(with_info_json={'id': source_id, 'title': 'Orig',
+ 'wrolpi': {'custom_title': 'Mine', 'custom_description': 'my words'}},
+ source_id=source_id)
+ video.file_group.url = url
+ test_session.commit()
+ assert video.file_group.title == 'Mine'
+
+ download = Download(url=url, info_json={'id': source_id}, settings={})
+ with mock.patch('modules.videos.downloader.download_video_info_json') as mock_download:
+ mock_download.return_value = {'id': source_id, 'title': 'Orig v2', 'description': 'theirs', 'duration': 9}
+ result = await VideoDownloader.download_info_json(download, video.video_path)
+ assert result.success is True, result
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, video.file_group_id)
+ info_json = video.get_info_json()
+ assert info_json['title'] == 'Orig v2', 'yt-dlp data was refreshed'
+ assert info_json['wrolpi'] == {'custom_title': 'Mine', 'custom_description': 'my words'}
+ assert video.file_group.title == 'Mine'
+ assert video.get_description() == 'my words'
+
+
+@pytest.mark.asyncio
+async def test_redownload_keeps_wrolpi_section(test_session, test_directory, mock_video_extract_info, simple_channel,
+ await_switches, video_download_manager, mock_video_process_runner,
+ image_file):
+ """A full re-download has yt-dlp overwrite the info json; the wrolpi section is carried over."""
+ simple_channel.source_id = example_video_json['channel_id']
+ simple_channel.directory = test_directory / 'videos/channel name'
+ simple_channel.directory.mkdir(parents=True)
+ url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
+
+ video_path = simple_channel.directory / 'a video.mp4'
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
+ info_json_path = video_path.with_suffix('.info.json')
+ info_json_path.write_text(json.dumps({'title': 'Orig', 'webpage_url': url, 'wrolpi': {'custom_title': 'Mine'}}))
+ from modules.videos.normalize_video_url import normalize_video_url
+ existing = Video.from_paths(test_session, video_path, info_json_path)
+ existing.validate(test_session)
+ test_session.commit()
+ assert existing.file_group.title == 'Mine'
+ assert Video.get_by_url(test_session, normalize_video_url(url)) is not None, 'lookup by normalized url'
+ assert existing.file_group.get_wrolpi_json() == {'custom_title': 'Mine'}
+
+ from wrolpi.cmd import CommandResult
+
+ async def yt_dlp_overwrites_info_json(*args, **kwargs):
+ # yt-dlp rewrites the whole info json; the wrolpi section is gone from disk.
+ info_json_path.write_text(json.dumps({'title': 'Orig v2', 'webpage_url': url, 'duration': 5}))
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
+
+ mock_video_process_runner.side_effect = yt_dlp_overwrites_info_json
+ with mock.patch('modules.videos.downloader.prepare_video_filename') as mock_prepare_filename:
+ mock_video_extract_info.return_value = example_video_json
+ mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
+ video_download_manager.create_download(test_session, url, video_downloader.name)
+ await video_download_manager.wait_for_all_downloads()
+ mock_video_process_runner.assert_called_once()
+
+ test_session.expire_all()
+ video: Video = test_session.query(Video).one()
+ info_json = video.get_info_json()
+ assert info_json['title'] == 'Orig v2'
+ assert info_json['wrolpi'] == {'custom_title': 'Mine'}
+ assert video.file_group.title == 'Mine'
diff --git a/modules/videos/test/test_transcode.py b/modules/videos/test/test_transcode.py
index e7967e53..24601e0b 100644
--- a/modules/videos/test/test_transcode.py
+++ b/modules/videos/test/test_transcode.py
@@ -1,3 +1,4 @@
+import asyncio
import json
import pathlib
from unittest import mock
@@ -5,10 +6,51 @@ from unittest import mock
import pytest
from modules.videos.transcode import codecs_match, get_stream_codec_names, get_transcode_target, \
- transcode_can_satisfy_codecs, transcode_video_file, TRANSCODE_VIDEO_TARGETS, TRANSCODE_AUDIO_TARGETS
+ transcode_can_satisfy_codecs, transcode_video_file, TRANSCODE_VIDEO_TARGETS, TRANSCODE_AUDIO_TARGETS, \
+ verify_transcode_output, validate_transcode_request, REMOVE_VIDEO, FRAGMENT_DURATION_US
from wrolpi.cmd import CommandResult
+def make_probe(vcodec='vp9', acodec='opus', duration=10.0, format_name='matroska,webm'):
+ """An ffprobe result with one video and one audio stream."""
+ streams = []
+ if vcodec:
+ streams.append({'codec_type': 'video', 'codec_name': vcodec})
+ if acodec:
+ streams.append({'codec_type': 'audio', 'codec_name': acodec})
+ return {'streams': streams, 'format': {'format_name': format_name, 'duration': str(duration)}}
+
+
+MP4 = 'mov,mp4,m4a,3gp,3g2,mj2'
+
+
+class FakeProbes:
+ """What ffprobe reports for the source (a vp9/opus webm) and for the transcode output.
+
+ The default output is a full h264/aac transcode; a test that copies a stream sets `output`
+ to what ffmpeg would really produce (a copied stream keeps the source codec)."""
+
+ def __init__(self):
+ self.source = make_probe('vp9', 'opus', 10.0)
+ self.output = make_probe('h264', 'aac', 10.0, MP4)
+
+ def __call__(self, path: pathlib.Path):
+ # Sync: mock.patch wraps the async target in an AsyncMock, which awaits for us.
+ if '.transcode.' in path.name:
+ output = dict(self.output)
+ if path.suffix != '.mp4' and output.get('format'):
+ output['format'] = {**output['format'], 'format_name': 'matroska,webm'}
+ return output
+ return self.source
+
+
+@pytest.fixture(autouse=True)
+def mock_probe() -> FakeProbes:
+ probes = FakeProbes()
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes):
+ yield probes
+
+
def test_get_stream_codec_names():
"""Embedded thumbnail streams (mjpeg/png) are not video."""
data = {'streams': [
@@ -79,6 +121,9 @@ async def test_transcode_video_file(test_directory, async_client):
async def fake_run_command(cmd, **kwargs):
# ffmpeg writes the temporary output file.
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
@@ -92,7 +137,7 @@ async def test_transcode_video_file(test_directory, async_client):
assert not sidecar.exists(), 'The stale ffprobe sidecar should be deleted'
assert not (test_directory / 'video.transcode.mp4').exists(), 'The temporary file should be renamed'
- cmd = mock_run.call_args[0][0]
+ cmd = mock_run.call_args_list[0][0][0]
assert '-c:v' in cmd and cmd[cmd.index('-c:v') + 1] == 'libx264'
assert '-c:a' in cmd and cmd[cmd.index('-c:a') + 1] == 'aac'
# Attached-picture streams are excluded; a missing audio stream is tolerated.
@@ -100,12 +145,16 @@ async def test_transcode_video_file(test_directory, async_client):
@pytest.mark.asyncio
-async def test_transcode_video_file_copies_matching_stream(test_directory, async_client):
+async def test_transcode_video_file_copies_matching_stream(test_directory, async_client, mock_probe):
"""Only the mismatching stream is re-encoded; the other stream is copied."""
video_path = test_directory / 'video.mp4'
video_path.write_bytes(b'fake video data')
+ mock_probe.output = make_probe('h264', 'opus', 10.0, MP4) # Audio was copied.
async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
@@ -113,18 +162,22 @@ async def test_transcode_video_file_copies_matching_stream(test_directory, async
result = await transcode_video_file(video_path, target_vcodec='h264', container='mp4')
assert result == video_path, 'Same container: the file is replaced in place'
- cmd = mock_run.call_args[0][0]
+ cmd = mock_run.call_args_list[0][0][0]
assert cmd[cmd.index('-c:v') + 1] == 'libx264'
assert cmd[cmd.index('-c:a') + 1] == 'copy'
@pytest.mark.asyncio
-async def test_transcode_video_file_forces_supported_container(test_directory, async_client):
+async def test_transcode_video_file_forces_supported_container(test_directory, async_client, mock_probe):
"""webm cannot contain h264/aac; the container is forced to mp4."""
video_path = test_directory / 'video.webm'
video_path.write_bytes(b'fake video data')
+ mock_probe.output = make_probe('h264', 'opus', 10.0, MP4) # Audio was copied.
async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
@@ -141,6 +194,9 @@ async def test_transcode_video_file_failure_removes_tmp(test_directory, async_cl
video_path.write_bytes(b'fake video data')
async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
pathlib.Path(cmd[-1]).write_bytes(b'partial data')
return CommandResult(return_code=1, cancelled=False, stdout=b'', stderr=b'boom', elapsed=1)
@@ -168,7 +224,7 @@ async def test_transcode_video_file_disk_space_guard(test_directory, async_clien
@pytest.mark.asyncio
-async def test_transcode_lock_serializes(test_directory, async_client):
+async def test_transcode_lock_serializes(test_directory, async_client, mock_probe):
"""A transcode waits while `shared_ctx.transcode_lock` is held and releases it when done.
This exercises the wait/release logic in-process; cross-process sharing of the lock is
@@ -178,8 +234,12 @@ async def test_transcode_lock_serializes(test_directory, async_client):
video_path = test_directory / 'video.webm'
video_path.write_bytes(b'fake video data')
+ mock_probe.output = make_probe('h264', 'opus', 10.0, MP4) # Audio was copied.
async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
@@ -203,7 +263,9 @@ async def test_transcode_lock_serializes(test_directory, async_client):
task.cancel()
assert result == test_directory / 'video.mp4'
- mock_run.assert_called_once()
+ # One encode (plus the tail-decode verification of its output); the second waiter ran nothing yet.
+ encodes = [c for c in mock_run.call_args_list if '-progress' in c[0][0]]
+ assert len(encodes) == 1
# The lock was released after the transcode.
assert lock.acquire(block=False), 'transcode_video_file did not release the lock'
@@ -211,6 +273,204 @@ async def test_transcode_lock_serializes(test_directory, async_client):
@pytest.mark.asyncio
-async def test_transcode_video_file_requires_target():
- with pytest.raises(RuntimeError, match='target codec'):
- await transcode_video_file(pathlib.Path('/tmp/video.mp4'))
+async def test_transcode_video_file_remux(test_directory, async_client, mock_probe):
+ """No target codec is a remux: both streams are copied into the requested container."""
+ video_path = test_directory / 'video.mkv'
+ video_path.write_bytes(b'fake video data')
+ mock_probe.output = make_probe('vp9', 'opus', 10.0, MP4) # Both copied.
+
+ async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
+ pathlib.Path(cmd[-1]).write_bytes(b'remuxed data')
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command) as mock_run:
+ result = await transcode_video_file(video_path, container='mp4')
+
+ assert result == test_directory / 'video.mp4' and result.is_file()
+ assert not video_path.exists()
+ cmd = mock_run.call_args_list[0][0][0]
+ assert cmd[cmd.index('-c:v') + 1] == 'copy' and cmd[cmd.index('-c:a') + 1] == 'copy'
+ assert '+faststart' in cmd
+
+
+
+# ------------------------------------------------------------------ output verification
+
+def test_verify_transcode_output_ok():
+ source = make_probe('vp9', 'opus', 100.0)
+ output = make_probe('h264', 'opus', 99.6, MP4) # Audio copied; a frame trimmed.
+ verify_transcode_output(source, output, target_vcodec='h264', target_acodec=None, container='mp4')
+
+
+def test_verify_transcode_output_remux_ok():
+ source = make_probe('vp9', 'opus', 100.0)
+ output = make_probe('vp9', 'opus', 100.0, MP4)
+ verify_transcode_output(source, output, target_vcodec=None, target_acodec=None, container='mp4')
+
+
+def test_verify_transcode_output_no_audio_ok():
+ """A silent source has no audio to demand of the output."""
+ source = make_probe('vp9', None, 100.0)
+ output = make_probe('h264', None, 100.0, MP4)
+ verify_transcode_output(source, output, target_vcodec='h264', target_acodec=None, container='mp4')
+
+
+@pytest.mark.parametrize('output,message', [
+ (make_probe('h264', 'aac', 50.0, MP4), 'duration'), # Truncated.
+ (make_probe('h264', None, 100.0, MP4), 'audio'), # Lost the audio stream.
+ (make_probe(None, 'aac', 100.0, MP4), 'video'), # Lost the video stream.
+ (make_probe('vp9', 'aac', 100.0, MP4), 'video codec'), # Not re-encoded.
+ (make_probe('h264', 'opus', 100.0, MP4), 'audio codec'), # Not re-encoded.
+ (make_probe('h264', 'aac', 100.0, 'matroska,webm'), 'container'), # Wrong container.
+ ({}, 'probe'), # ffprobe could not read it.
+])
+def test_verify_transcode_output_rejects(output, message):
+ source = make_probe('vp9', 'opus', 100.0)
+ with pytest.raises(RuntimeError, match=message):
+ verify_transcode_output(source, output, target_vcodec='h264', target_acodec='aac', container='mp4')
+
+
+def test_verify_transcode_output_copied_stream_must_match_source():
+ """A copied stream must come out with the source's codec."""
+ source = make_probe('vp9', 'opus', 100.0)
+ output = make_probe('h264', 'aac', 100.0, MP4)
+ with pytest.raises(RuntimeError, match='audio codec'):
+ verify_transcode_output(source, output, target_vcodec='h264', target_acodec=None, container='mp4')
+
+
+def test_verify_transcode_output_duration_tolerance():
+ """One second, or one percent, whichever is larger."""
+ source = make_probe('vp9', 'opus', 10.0)
+ verify_transcode_output(source, make_probe('h264', 'aac', 9.1, MP4), 'h264', 'aac', 'mp4')
+ with pytest.raises(RuntimeError, match='duration'):
+ verify_transcode_output(source, make_probe('h264', 'aac', 8.9, MP4), 'h264', 'aac', 'mp4')
+ long_source = make_probe('vp9', 'opus', 40000.0)
+ verify_transcode_output(long_source, make_probe('h264', 'aac', 39700.0, MP4), 'h264', 'aac', 'mp4')
+ with pytest.raises(RuntimeError, match='duration'):
+ verify_transcode_output(long_source, make_probe('h264', 'aac', 39500.0, MP4), 'h264', 'aac', 'mp4')
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_file_rejects_truncated_output(test_directory, async_client, mock_probe):
+ """ffmpeg exits 0 but the output is short: the original is kept, the output is removed."""
+ video_path = test_directory / 'video.webm'
+ video_path.write_bytes(b'fake video data')
+
+ mock_probe.output = make_probe('h264', 'aac', 4.0, MP4) # Source is 10s.
+
+ async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] != '-':
+ pathlib.Path(cmd[-1]).write_bytes(b'partial data')
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command):
+ with pytest.raises(RuntimeError, match='duration'):
+ await transcode_video_file(video_path, target_vcodec='h264', target_acodec='aac')
+
+ assert video_path.read_bytes() == b'fake video data', 'The original must survive'
+ assert not (test_directory / 'video.mp4').exists()
+ assert not (test_directory / 'video.transcode.mp4').exists(), 'The bad output is removed'
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_file_rejects_undecodable_tail(test_directory, async_client):
+ """The last seconds of the output must decode cleanly, or the original is kept."""
+ video_path = test_directory / 'video.webm'
+ video_path.write_bytes(b'fake video data')
+ tail_cmds = []
+
+ async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ tail_cmds.append(cmd)
+ return CommandResult(return_code=0, cancelled=False, stdout=b'',
+ stderr=b'[h264 @ 0x1] Invalid NAL unit size', elapsed=1)
+ pathlib.Path(cmd[-1]).write_bytes(b'transcoded data')
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command):
+ with pytest.raises(RuntimeError, match='decode'):
+ await transcode_video_file(video_path, target_vcodec='h264', target_acodec='aac')
+
+ assert video_path.read_bytes() == b'fake video data'
+ assert not (test_directory / 'video.transcode.mp4').exists()
+ # The check decodes only the tail of the temporary output, discarding the frames.
+ assert len(tail_cmds) == 1
+ cmd = tail_cmds[0]
+ assert '-sseof' in cmd and cmd[cmd.index('-sseof') + 1].startswith('-')
+ assert str(cmd[cmd.index('-i') + 1]).endswith('video.transcode.mp4')
+ assert cmd[cmd.index('-f') + 1] == 'null'
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_file_cancel_removes_tmp(test_directory, async_client):
+ """A cancelled encode (a Job's run_command raises CancelledError, a BaseException) must still
+ remove the temporary output and leave the original."""
+ video_path = test_directory / 'video.webm'
+ video_path.write_bytes(b'fake video data')
+
+ async def fake_run_command(cmd, **kwargs):
+ pathlib.Path(cmd[-1]).write_bytes(b'partial data')
+ raise asyncio.CancelledError('killed')
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command):
+ with pytest.raises(asyncio.CancelledError):
+ await transcode_video_file(video_path, target_vcodec='h264')
+
+ assert video_path.read_bytes() == b'fake video data'
+ assert not (test_directory / 'video.transcode.mp4').exists(), 'The partial output must be removed'
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_file_fragmented(test_directory, async_client, mock_probe):
+ """`fragmented=True` writes a fragmented mp4: a tiny moov, a global seek index, and fragments
+ at keyframes instead of fast start. Very long videos then start and seek quickly on phones."""
+ video_path = test_directory / 'video.mp4'
+ video_path.write_bytes(b'fake video data')
+ mock_probe.output = make_probe('vp9', 'opus', 10.0, MP4) # A remux: both streams copied.
+
+ async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] != '-':
+ pathlib.Path(cmd[-1]).write_bytes(b'fragmented data')
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command) as mock_run:
+ result = await transcode_video_file(video_path, container='mp4', fragmented=True)
+
+ assert result == video_path
+ cmd = mock_run.call_args_list[0][0][0]
+ flags = cmd[cmd.index('-movflags') + 1]
+ for flag in ('frag_keyframe', 'empty_moov', 'default_base_moof', 'global_sidx'):
+ assert flag in flags, flags
+ assert 'faststart' not in flags
+ assert cmd[cmd.index('-frag_duration') + 1] == str(FRAGMENT_DURATION_US)
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_file_fragmented_only_mp4(test_directory, async_client, mock_probe):
+ """Fragments are an mp4 concept; for another container the flag is ignored."""
+ video_path = test_directory / 'video.mp4'
+ video_path.write_bytes(b'fake video data')
+ mock_probe.output = make_probe('vp9', 'opus', 10.0, 'matroska,webm')
+
+ async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] != '-':
+ pathlib.Path(cmd[-1]).write_bytes(b'data')
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command) as mock_run:
+ await transcode_video_file(video_path, container='mkv', fragmented=True)
+
+ cmd = mock_run.call_args_list[0][0][0]
+ assert '-movflags' not in cmd and '-frag_duration' not in cmd
+
+
+def test_validate_fragmented_needs_mp4():
+ validate_transcode_request(None, None, 'mp4', fragmented=True)
+ validate_transcode_request(REMOVE_VIDEO, None, 'm4a', fragmented=True) # m4a is an mp4.
+ with pytest.raises(ValueError, match='mp4'):
+ validate_transcode_request(None, None, 'mkv', fragmented=True)
+ with pytest.raises(ValueError, match='mp4'):
+ validate_transcode_request(REMOVE_VIDEO, 'mp3', 'mp3', fragmented=True)
diff --git a/modules/videos/test/test_transcode_audio.py b/modules/videos/test/test_transcode_audio.py
new file mode 100644
index 00000000..0abdcd3e
--- /dev/null
+++ b/modules/videos/test/test_transcode_audio.py
@@ -0,0 +1,265 @@
+"""Audio-only transcodes: removing the video stream from a video, and transcoding audio files."""
+import json
+import pathlib
+import shutil
+from unittest import mock
+
+import pytest
+
+from modules.videos.transcode import transcode_video_file, validate_transcode_request, verify_transcode_output, \
+ REMOVE_VIDEO, TRANSCODE_AUDIO_CONTAINERS, default_audio_container, transcode_video_job
+from wrolpi import jobs
+from wrolpi.cmd import CommandResult
+from wrolpi.vars import PROJECT_DIR
+
+MP4 = 'mov,mp4,m4a,3gp,3g2,mj2'
+
+
+def make_probe(vcodec, acodec, duration=10.0, format_name=MP4):
+ streams = []
+ if vcodec:
+ streams.append({'codec_type': 'video', 'codec_name': vcodec})
+ if acodec:
+ streams.append({'codec_type': 'audio', 'codec_name': acodec})
+ return {'streams': streams, 'format': {'format_name': format_name, 'duration': str(duration)}}
+
+
+class FakeProbes:
+ def __init__(self, source, output):
+ self.source, self.output = source, output
+
+ def __call__(self, path: pathlib.Path):
+ return self.output if '.transcode.' in path.name else self.source
+
+
+def fake_ffmpeg(cmd, **kwargs):
+ """Writes the output file; the tail-decode check writes nothing."""
+ if cmd[-1] != '-':
+ pathlib.Path(cmd[-1]).write_bytes(b'output data')
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+
+async def async_fake_ffmpeg(cmd, **kwargs):
+ return fake_ffmpeg(cmd, **kwargs)
+
+
+# ------------------------------------------------------------------ validation
+
+def test_validate_remove_video():
+ """Removing the video needs an audio container; an audio container needs a compatible codec."""
+ validate_transcode_request(REMOVE_VIDEO, 'aac', 'm4a')
+ validate_transcode_request(REMOVE_VIDEO, None, 'ogg') # Copy: checked against the source later.
+ validate_transcode_request(REMOVE_VIDEO, 'mp3', 'mp3')
+ with pytest.raises(ValueError, match='audio container'):
+ validate_transcode_request(REMOVE_VIDEO, 'aac', 'mp4')
+ with pytest.raises(ValueError, match='ogg'):
+ validate_transcode_request(REMOVE_VIDEO, 'aac', 'ogg')
+ with pytest.raises(ValueError, match='mp3'):
+ validate_transcode_request(REMOVE_VIDEO, 'opus', 'mp3')
+ with pytest.raises(ValueError, match='video'):
+ validate_transcode_request('h264', 'aac', 'm4a') # A video stream cannot live in m4a.
+ assert set(TRANSCODE_AUDIO_CONTAINERS) == {'m4a', 'ogg', 'mp3'}
+
+
+def test_default_audio_container():
+ assert default_audio_container('aac') == 'm4a'
+ assert default_audio_container('opus') == 'ogg'
+ assert default_audio_container('vorbis') == 'ogg'
+ assert default_audio_container('mp3') == 'mp3'
+ assert default_audio_container('flac') == 'ogg'
+ assert default_audio_container(None) == 'm4a'
+
+
+# ------------------------------------------------------------------ verification
+
+def test_verify_audio_only_output():
+ source = make_probe('vp9', 'opus', 100.0, 'matroska,webm')
+ good = make_probe(None, 'aac', 100.0)
+ verify_transcode_output(source, good, REMOVE_VIDEO, 'aac', 'm4a')
+ # The video stream must be gone.
+ with pytest.raises(RuntimeError, match='video stream'):
+ verify_transcode_output(source, make_probe('vp9', 'aac', 100.0), REMOVE_VIDEO, 'aac', 'm4a')
+ # An embedded cover is not a video stream.
+ verify_transcode_output(source, make_probe('mjpeg', 'aac', 100.0), REMOVE_VIDEO, 'aac', 'm4a')
+ # Audio is required.
+ with pytest.raises(RuntimeError, match='audio'):
+ verify_transcode_output(source, make_probe(None, None, 100.0), REMOVE_VIDEO, 'aac', 'm4a')
+
+
+def test_verify_container_names():
+ """ffprobe names containers differently than the file suffix does."""
+ source = make_probe('h264', 'aac', 10.0)
+ verify_transcode_output(source, make_probe('h264', 'aac', 10.0, 'matroska,webm'), None, None, 'mkv')
+ verify_transcode_output(source, make_probe(None, 'opus', 10.0, 'ogg'), REMOVE_VIDEO, 'opus', 'ogg')
+ verify_transcode_output(source, make_probe(None, 'mp3', 10.0, 'mp3'), REMOVE_VIDEO, 'mp3', 'mp3')
+ verify_transcode_output(source, make_probe(None, 'aac', 10.0, MP4), REMOVE_VIDEO, None, 'm4a')
+ with pytest.raises(RuntimeError, match='container'):
+ verify_transcode_output(source, make_probe(None, 'mp3', 10.0, 'mp3'), REMOVE_VIDEO, 'mp3', 'ogg')
+
+
+def test_verify_audio_source():
+ """An audio-only source has no video to demand; the output is audio-only too."""
+ source = make_probe(None, 'mp3', 100.0, 'mp3')
+ verify_transcode_output(source, make_probe(None, 'aac', 100.0), None, 'aac', 'm4a')
+ verify_transcode_output(source, make_probe(None, 'mp3', 100.0, 'mp3'), None, None, 'mp3')
+
+
+# ------------------------------------------------------------------ transcode_video_file
+
+@pytest.mark.asyncio
+async def test_remove_video(test_directory, async_client):
+ """Removing the video: no video map, `-vn`, the audio is required, the output is an audio file
+ named for its container, and the original is replaced."""
+ video_path = test_directory / 'movie.mp4'
+ video_path.write_bytes(b'fake video data')
+ probes = FakeProbes(make_probe('h264', 'aac', 10.0), make_probe(None, 'aac', 10.0))
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes), \
+ mock.patch('modules.videos.transcode.run_command', side_effect=async_fake_ffmpeg) as mock_run:
+ result = await transcode_video_file(video_path, target_vcodec=REMOVE_VIDEO, target_acodec=None,
+ container='m4a')
+
+ assert result == test_directory / 'movie.m4a' and result.is_file()
+ assert not video_path.exists()
+ cmd = mock_run.call_args_list[0][0][0]
+ assert '-vn' in cmd
+ assert '0:V:0' not in cmd
+ assert cmd[cmd.index('-map') + 1] == '0:a:0', 'Audio is required, not optional'
+ assert '-c:v' not in cmd
+ assert cmd[cmd.index('-c:a') + 1] == 'copy'
+ assert '+faststart' in cmd, 'm4a is an mp4; fast start applies'
+ assert str(cmd[-1]).endswith('movie.transcode.m4a')
+
+
+@pytest.mark.asyncio
+async def test_remove_video_to_ogg_no_faststart(test_directory, async_client):
+ video_path = test_directory / 'movie.webm'
+ video_path.write_bytes(b'fake video data')
+ probes = FakeProbes(make_probe('vp9', 'opus', 10.0, 'matroska,webm'), make_probe(None, 'opus', 10.0, 'ogg'))
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes), \
+ mock.patch('modules.videos.transcode.run_command', side_effect=async_fake_ffmpeg) as mock_run:
+ result = await transcode_video_file(video_path, target_vcodec=REMOVE_VIDEO, container='ogg')
+
+ assert result == test_directory / 'movie.ogg'
+ cmd = mock_run.call_args_list[0][0][0]
+ assert '-movflags' not in cmd, 'movflags is an mp4 muxer option'
+
+
+@pytest.mark.asyncio
+async def test_audio_source_is_audio_output(test_directory, async_client):
+ """An audio file has no video to keep or remove: the output is audio-only whatever the video
+ target says, and an unsuitable container is replaced by the codec's natural one."""
+ audio_path = test_directory / 'song.webm'
+ audio_path.write_bytes(b'fake audio data')
+ probes = FakeProbes(make_probe(None, 'opus', 10.0, 'matroska,webm'), make_probe(None, 'opus', 10.0, 'ogg'))
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes), \
+ mock.patch('modules.videos.transcode.run_command', side_effect=async_fake_ffmpeg) as mock_run:
+ result = await transcode_video_file(audio_path, target_vcodec=None, target_acodec=None, container='mp4')
+
+ assert result == test_directory / 'song.ogg', 'opus copied: ogg, not the video container asked for'
+ cmd = mock_run.call_args_list[0][0][0]
+ assert '-vn' in cmd and '0:V:0' not in cmd
+
+
+@pytest.mark.asyncio
+async def test_copy_audio_into_incompatible_container_is_refused(test_directory, async_client):
+ """Copying opus into m4a cannot work; refuse before ffmpeg runs."""
+ video_path = test_directory / 'movie.webm'
+ video_path.write_bytes(b'fake video data')
+ probes = FakeProbes(make_probe('vp9', 'opus', 10.0, 'matroska,webm'), make_probe(None, 'opus', 10.0))
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes), \
+ mock.patch('modules.videos.transcode.run_command', side_effect=async_fake_ffmpeg) as mock_run:
+ with pytest.raises(RuntimeError, match='m4a'):
+ await transcode_video_file(video_path, target_vcodec=REMOVE_VIDEO, target_acodec=None, container='m4a')
+ mock_run.assert_not_called()
+ assert video_path.is_file()
+
+
+@pytest.mark.asyncio
+async def test_video_target_on_audio_source_is_refused(test_directory, async_client):
+ audio_path = test_directory / 'song.mp3'
+ audio_path.write_bytes(b'fake audio data')
+ probes = FakeProbes(make_probe(None, 'mp3', 10.0, 'mp3'), make_probe(None, 'mp3', 10.0, 'mp3'))
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes), \
+ mock.patch('modules.videos.transcode.run_command', side_effect=async_fake_ffmpeg) as mock_run:
+ with pytest.raises(RuntimeError, match='no video stream'):
+ await transcode_video_file(audio_path, target_vcodec='h264', container='mp4')
+ mock_run.assert_not_called()
+
+
+# ------------------------------------------------------------------ Job + API
+
+@pytest.mark.asyncio
+async def test_remove_video_job(test_session, test_directory, async_client, video_factory):
+ """The FileGroup follows the video into its audio-only life: same id, new suffix and mimetype."""
+ from modules.videos.models import Video
+ from wrolpi.files.models import FileGroup
+
+ video = video_factory()
+ video.file_group.length = 5
+ test_session.commit()
+ file_group_id = video.file_group_id
+ video_path = video.video_path
+ probes = FakeProbes(make_probe('h264', 'aac', 5.3), make_probe(None, 'mp3', 5.3, 'mp3'))
+
+ async def fake_run_command(cmd, stdout_callback=None, **kwargs):
+ if cmd[-1] != '-':
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny.mp3', cmd[-1])
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probes), \
+ mock.patch('wrolpi.cmd.run_command', side_effect=fake_run_command):
+ job_id = transcode_video_job.enqueue(file_group_id=file_group_id, video_codec=REMOVE_VIDEO,
+ audio_codec='mp3', container='mp3')
+ record = await jobs.wait_for_job(job_id)
+
+ assert record['status'] == jobs.COMPLETE, record
+ new_path = video_path.with_suffix('.mp3')
+ assert new_path.is_file() and not video_path.exists()
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, file_group_id)
+ assert video.video_path == new_path
+ assert video.file_group.mimetype.startswith('audio/')
+ assert test_session.query(FileGroup).count() == 1
+
+
+@pytest.mark.asyncio
+async def test_transcode_api_audio_file(async_client, test_session, audio_factory):
+ """An audio file can be transcoded (audio codec / audio container); a video target is a 400."""
+ audio = audio_factory()
+ test_session.commit()
+
+ request, response = await async_client.post(f'/api/videos/{audio.file_group_id}/transcode',
+ content=json.dumps({'video_codec': 'h264', 'container': 'mp4'}))
+ assert response.status == 400, response.json
+ assert 'audio' in response.json['message'].lower() or 'audio' in response.json['error'].lower()
+
+ request, response = await async_client.post(f'/api/videos/{audio.file_group_id}/transcode',
+ content=json.dumps({'audio_codec': 'aac', 'container': 'm4a'}))
+ assert response.status == 200, response.json
+ jobs.cancel_job(response.json['job_id'])
+
+ # Remove-video on an audio file is a no-op wording-wise but allowed (it is already audio).
+ request, response = await async_client.post(f'/api/videos/{audio.file_group_id}/transcode',
+ content=json.dumps({'video_codec': REMOVE_VIDEO, 'container': 'mp3'}))
+ assert response.status == 200, response.json
+ jobs.cancel_job(response.json['job_id'])
+
+
+@pytest.mark.asyncio
+async def test_transcode_api_remove_video_needs_audio_container(async_client, test_session, video_factory):
+ video = video_factory()
+ test_session.commit()
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'video_codec': REMOVE_VIDEO, 'container': 'mp4'}))
+ assert response.status == 400, response.json
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'video_codec': REMOVE_VIDEO, 'audio_codec': 'aac',
+ 'container': 'm4a'}))
+ assert response.status == 200, response.json
+ jobs.cancel_job(response.json['job_id'])
diff --git a/modules/videos/test/test_transcode_job.py b/modules/videos/test/test_transcode_job.py
new file mode 100644
index 00000000..d773f9a5
--- /dev/null
+++ b/modules/videos/test/test_transcode_job.py
@@ -0,0 +1,295 @@
+"""Tests for the transcode Job and its API (see `wrolpi.jobs`)."""
+import json
+import pathlib
+import shutil
+from unittest import mock
+
+import pytest
+
+from modules.videos.models import Video
+from modules.videos.transcode import make_ffmpeg_progress_parser, transcode_video_job
+from wrolpi import jobs
+from wrolpi.cmd import CommandResult
+from wrolpi.files.models import FileGroup
+from wrolpi.vars import PROJECT_DIR
+
+MP4 = 'mov,mp4,m4a,3gp,3g2,mj2'
+
+
+@pytest.fixture(autouse=True)
+def mock_probe():
+ """The output probe agrees with what the Job asked for (targets or copied source codecs)."""
+
+ async def probe(path: pathlib.Path):
+ # The test video is h264/aac; a copied stream keeps that, a target replaces it.
+ fmt = MP4 if path.suffix == '.mp4' else 'matroska,webm'
+ return {'streams': [{'codec_type': 'video', 'codec_name': 'h264'},
+ {'codec_type': 'audio', 'codec_name': 'aac'}],
+ 'format': {'format_name': fmt, 'duration': '5.3'}}
+
+ with mock.patch('modules.videos.transcode.probe_for_verify', side_effect=probe):
+ yield
+
+
+def test_make_ffmpeg_progress_parser():
+ """`-progress pipe:1` lines are converted to whole-percent progress, reported only on change."""
+ reported = []
+ parse = make_ffmpeg_progress_parser(100.0, reported.append)
+ parse('frame=12') # Ignored.
+ parse('out_time_us=25000000')
+ parse('out_time_us=25400000') # Still 25%, not reported again.
+ parse('out_time_ms=50000000') # Older ffmpeg key, also microseconds.
+ parse('out_time_us=garbage')
+ parse('out_time_us=250000000') # Clamped to 100.
+ assert reported == [25, 50, 100]
+
+ # No duration: nothing can be reported.
+ reported.clear()
+ make_ffmpeg_progress_parser(None, reported.append)('out_time_us=25000000')
+ assert reported == []
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_job(test_session, test_directory, async_client, video_factory):
+ """The transcode Job runs ffmpeg inside the Job (log + progress captured) and re-indexes the
+ Video so its FileGroup points at the new file."""
+ video_path = test_directory / 'videos/NO CHANNEL/movie.webm'
+ video_path.parent.mkdir(parents=True, exist_ok=True)
+ video = video_factory(with_video_file=video_path)
+ video.file_group.length = 10
+ test_session.commit()
+ file_group_id = video.file_group_id
+
+ async def fake_run_command(cmd, stdout_callback=None, **kwargs):
+ # ffmpeg reports progress on stdout, then writes the temporary output file. The output
+ # must be a real video: the refresh would otherwise drop the Video model (mimetype changed).
+ stdout_callback('out_time_us=5000000')
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', cmd[-1])
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'ffmpeg version fake', elapsed=1)
+
+ # In tests the file refresh is drained inline by the waiter, inside the Job's task, so its
+ # (many) log lines land in the Job log too and could push the early lines out of the tail.
+ with mock.patch('wrolpi.cmd.run_command', side_effect=fake_run_command), \
+ mock.patch.object(jobs, 'JOB_LOG_LINES', 10_000):
+ job_id = transcode_video_job.enqueue(description='Transcode movie.webm', file_group_id=file_group_id,
+ video_codec='h264', audio_codec='aac', container='mp4')
+ record = await jobs.wait_for_job(job_id)
+
+ assert record['status'] == jobs.COMPLETE, record
+ assert record['result'] == {'path': 'videos/NO CHANNEL/movie.mp4'}
+ assert 'out_time_us=5000000' in record['log'], 'ffmpeg stdout is in the Job log'
+ assert 'ffmpeg version fake' in record['log'], 'ffmpeg stderr is in the Job log'
+ assert any('Transcoding' in i for i in record['log']), 'logger output is in the Job log'
+
+ assert not video_path.exists()
+ assert (test_directory / 'videos/NO CHANNEL/movie.mp4').is_file()
+
+ test_session.expire_all()
+ # The same FileGroup (tags, history) now points at the mp4; no new FileGroup was created.
+ video = Video.find_by_file_group_id(test_session, file_group_id)
+ assert video.video_path == test_directory / 'videos/NO CHANNEL/movie.mp4'
+ assert video.file_group.primary_path == test_directory / 'videos/NO CHANNEL/movie.mp4'
+ assert video.file_group.mimetype == 'video/mp4'
+ # The modeler probed the new file and wrote a fresh sidecar; the webm is gone.
+ assert sorted(i['path'] for i in video.file_group.files) == ['movie.ffprobe.json', 'movie.mp4']
+ assert video.ffprobe_json, 'ffprobe ran again on the new file'
+ assert video.file_group.indexed is True, 'The refresh re-modeled the Video'
+ assert test_session.query(FileGroup).count() == 1
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_job_remux(test_session, test_directory, async_client, video_factory):
+ """No target codec is a remux: both streams are copied into the requested container."""
+ video_path = test_directory / 'videos/NO CHANNEL/movie.mkv'
+ video_path.parent.mkdir(parents=True, exist_ok=True)
+ video = video_factory(with_video_file=video_path)
+ test_session.commit()
+
+ async def fake_run_command(cmd, stdout_callback=None, **kwargs):
+ if cmd[-1] == '-':
+ # The tail-decode check: nothing to write.
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', cmd[-1])
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('wrolpi.cmd.run_command', side_effect=fake_run_command) as mock_run:
+ job_id = transcode_video_job.enqueue(file_group_id=video.file_group_id, container='mp4')
+ record = await jobs.wait_for_job(job_id)
+
+ assert record['status'] == jobs.COMPLETE, record
+ cmd = mock_run.call_args_list[0][0][0]
+ assert cmd[cmd.index('-c:v') + 1] == 'copy' and cmd[cmd.index('-c:a') + 1] == 'copy'
+ assert '+faststart' in cmd
+ assert not video_path.exists()
+ assert (test_directory / 'videos/NO CHANNEL/movie.mp4').is_file()
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_job_failure(test_session, test_directory, async_client, video_factory):
+ """A failed ffmpeg fails the Job and leaves the original file in place."""
+ video = video_factory()
+ video_path = video.video_path
+
+ async def fake_run_command(cmd, stdout_callback=None, **kwargs):
+ return CommandResult(return_code=1, cancelled=False, stdout=b'', stderr=b'Conversion failed!', elapsed=1)
+
+ with mock.patch('wrolpi.cmd.run_command', side_effect=fake_run_command):
+ job_id = transcode_video_job.enqueue(file_group_id=video.file_group_id, video_codec='h264')
+ record = await jobs.wait_for_job(job_id)
+
+ assert record['status'] == jobs.FAILED
+ assert 'Conversion failed!' in record['error']
+ assert video_path.is_file()
+
+
+@pytest.mark.asyncio
+async def test_video_transcode_api(async_client, test_session, video_factory, wrol_mode_fixture):
+ """The transcode endpoint validates the request, then queues a Job."""
+ video = video_factory()
+ test_session.commit()
+
+ # Unsupported target.
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'video_codec': 'av1'}))
+ assert response.status == 400
+ # Unknown video.
+ request, response = await async_client.post('/api/videos/999999/transcode',
+ content=json.dumps({'video_codec': 'h264'}))
+ assert response.status == 404
+ assert jobs.get_jobs() == [], 'Nothing was queued'
+
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'video_codec': 'h264', 'audio_codec': 'aac'}))
+ assert response.status == 200, response.json
+ job_id = response.json['job_id']
+ record = jobs.get_job(job_id)
+ assert record['status'] == jobs.PENDING
+ assert record['name'] == 'transcode_video'
+ assert record['description'] == f'Transcode {video.video_path.name}'
+ assert record['kwargs'] == {'file_group_id': video.file_group_id, 'video_codec': 'h264',
+ 'audio_codec': 'aac', 'container': 'mp4', 'fragmented': False}
+ # Do not run it (ffmpeg); cancel instead.
+ jobs.cancel_job(job_id)
+
+ # No codec at all is allowed: a container-only remux.
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'container': 'mkv'}))
+ assert response.status == 200, response.json
+ assert jobs.get_job(response.json['job_id'])['kwargs'] == {
+ 'file_group_id': video.file_group_id, 'video_codec': None, 'audio_codec': None, 'container': 'mkv', 'fragmented': False}
+ jobs.cancel_job(response.json['job_id'])
+
+ # WROL mode forbids modifying files.
+ await wrol_mode_fixture(True)
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'video_codec': 'h264'}))
+ assert response.status == 403
+ await wrol_mode_fixture(False)
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_job_wrol_mode(test_session, test_directory, async_client, video_factory,
+ wrol_mode_fixture):
+ """A transcode queued before WROL Mode was enabled must not run: the Job fails, ffmpeg is never
+ started, and the file is untouched."""
+ video = video_factory()
+ video_path = video.video_path
+ before = video_path.read_bytes()
+
+ job_id = transcode_video_job.enqueue(file_group_id=video.file_group_id, video_codec='h264')
+ await wrol_mode_fixture(True)
+ try:
+ with mock.patch('wrolpi.cmd.run_command') as mock_run:
+ record = await jobs.wait_for_job(job_id)
+ finally:
+ await wrol_mode_fixture(False)
+
+ assert record['status'] == jobs.FAILED, record
+ assert 'WROL Mode' in record['error']
+ mock_run.assert_not_called()
+ assert video_path.read_bytes() == before
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_job_retargets_before_deleting(test_session, test_directory, async_client,
+ video_factory):
+ """When the suffix changes, the FileGroup is pointed at the new file (and committed) while the
+ old file still exists; the old file is deleted only afterwards. A crash in between then leaves
+ both files with the database pointing at a real one."""
+ from modules.videos import transcode as transcode_module
+
+ video_path = test_directory / 'videos/NO CHANNEL/movie.webm'
+ video_path.parent.mkdir(parents=True, exist_ok=True)
+ video = video_factory(with_video_file=video_path)
+ test_session.commit()
+ seen = {}
+
+ real_retarget = transcode_module.retarget_file_group
+
+ def spy_retarget(file_group_id, old_path, new_path):
+ seen['old_exists'] = old_path.exists()
+ seen['new_exists'] = new_path.exists()
+ real_retarget(file_group_id, old_path, new_path)
+ test_session.expire_all()
+ seen['db_primary'] = Video.find_by_file_group_id(test_session, file_group_id).file_group.primary_path
+
+ async def fake_run_command(cmd, stdout_callback=None, **kwargs):
+ if cmd[-1] == '-':
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', cmd[-1])
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('wrolpi.cmd.run_command', side_effect=fake_run_command), \
+ mock.patch.object(transcode_module, 'retarget_file_group', side_effect=spy_retarget):
+ job_id = transcode_video_job.enqueue(file_group_id=video.file_group_id, video_codec='h264',
+ audio_codec='aac', container='mp4')
+ record = await jobs.wait_for_job(job_id)
+
+ assert record['status'] == jobs.COMPLETE, record
+ new_path = test_directory / 'videos/NO CHANNEL/movie.mp4'
+ assert seen == {'old_exists': True, 'new_exists': True, 'db_primary': new_path}, seen
+ assert not video_path.exists(), 'The old file is deleted once the database points at the new one'
+ assert new_path.is_file()
+
+
+@pytest.mark.asyncio
+async def test_transcode_video_file_keep_original(test_directory, async_client):
+ """`transcode_video_file(keep_original=True)` renames the output into place but leaves the
+ original for the caller to delete once its records point at the new file."""
+ from modules.videos.transcode import transcode_video_file
+ video_path = test_directory / 'video.webm'
+ video_path.write_bytes(b'fake video data')
+
+ async def fake_run_command(cmd, **kwargs):
+ if cmd[-1] == '-':
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=0)
+ shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', cmd[-1])
+ return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
+
+ with mock.patch('modules.videos.transcode.run_command', side_effect=fake_run_command):
+ result = await transcode_video_file(video_path, target_vcodec='h264', target_acodec='aac',
+ container='mp4', keep_original=True)
+
+ assert result == test_directory / 'video.mp4' and result.is_file()
+ assert video_path.read_bytes() == b'fake video data', 'The original is kept'
+
+
+@pytest.mark.asyncio
+async def test_video_transcode_api_fragmented(async_client, test_session, video_factory):
+ """`fragmented` reaches the Job; it needs an mp4 container."""
+ video = video_factory()
+ test_session.commit()
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'container': 'mp4', 'fragmented': True}))
+ assert response.status == 200, response.json
+ record = jobs.get_job(response.json['job_id'])
+ assert record['kwargs']['fragmented'] is True
+ assert record['description'].startswith('Remux')
+ jobs.cancel_job(response.json['job_id'])
+
+ request, response = await async_client.post(f'/api/videos/{video.file_group_id}/transcode',
+ content=json.dumps({'container': 'mkv', 'fragmented': True}))
+ assert response.status == 400
diff --git a/modules/videos/test/test_video_edit.py b/modules/videos/test/test_video_edit.py
new file mode 100644
index 00000000..095b7db7
--- /dev/null
+++ b/modules/videos/test/test_video_edit.py
@@ -0,0 +1,212 @@
+"""Editing a Video's details from the video page.
+
+Edits never touch yt-dlp's own keys: they are written to the `wrolpi` section of the info json
+(`custom_title`, `custom_description`), which the modeler prefers when it derives the Video."""
+import json
+
+import pytest
+
+from modules.videos.lib import extract_video_info_json
+from modules.videos.models import Video
+from modules.videos.video.lib import update_video
+from wrolpi.errors import ValidationError
+
+
+def test_update_video_title(test_session, test_directory, video_factory):
+ """The new title lands in the wrolpi section; yt-dlp's title/fulltitle stay as they were. The
+ Video is re-validated, so its title and search text change."""
+ video = video_factory(with_info_json={'title': 'Old Title', 'fulltitle': 'Old Title', 'duration': 7,
+ 'formats': ['kept, not cleaned']})
+ test_session.commit()
+ assert video.file_group.title == 'Old Title'
+
+ update_video(video.file_group_id, ' New Title ')
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, video.file_group_id)
+ assert video.file_group.title == 'New Title'
+ assert video.file_group.a_text == 'New Title'
+ assert video.file_group.length == 7, 'Other details from the info json are unchanged'
+ info_json = json.loads(video.info_json_path.read_text())
+ assert info_json['title'] == 'Old Title' and info_json['fulltitle'] == 'Old Title', 'The original is kept'
+ assert info_json['wrolpi']['custom_title'] == 'New Title'
+ assert info_json['formats'] == ['kept, not cleaned']
+
+
+def test_update_video_title_creates_info_json(test_session, test_directory, video_factory):
+ """A Video without an info json gets one holding only the wrolpi section."""
+ video = video_factory()
+ test_session.commit()
+ assert video.info_json_path is None
+
+ update_video(video.file_group_id, 'Brand New')
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, video.file_group_id)
+ assert video.file_group.title == 'Brand New'
+ assert video.info_json_path and video.info_json_path.is_file()
+ info_json = json.loads(video.info_json_path.read_text())
+ assert info_json == {'wrolpi': {'custom_title': 'Brand New'}}
+ assert video.info_json_path.name in [i['path'] for i in video.file_group.files], 'The new file is tracked'
+
+
+def test_update_video_description(test_session, test_directory, video_factory):
+ """The description lands in the wrolpi section and is indexed; yt-dlp's description stays.
+ An empty custom description clears the shown description."""
+ video = video_factory(with_info_json={'title': 'Kept', 'description': 'old words'})
+ test_session.commit()
+ assert video.file_group.c_text == 'old words'
+
+ update_video(video.file_group_id, description='new words\nsecond line')
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, video.file_group_id)
+ assert video.file_group.title == 'Kept'
+ assert video.file_group.c_text == 'new words\nsecond line'
+ assert video.get_description() == 'new words\nsecond line'
+ info_json = json.loads(video.info_json_path.read_text())
+ assert info_json['description'] == 'old words', 'The original is kept'
+ assert info_json['wrolpi']['custom_description'] == 'new words\nsecond line'
+
+ update_video(video.file_group_id, description='')
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, video.file_group_id)
+ assert video.get_description() is None
+ assert json.loads(video.info_json_path.read_text())['description'] == 'old words'
+ assert video.file_group.title == 'Kept'
+
+
+def test_update_video_nothing(test_session, video_factory):
+ video = video_factory(with_info_json={'title': 'Old'})
+ test_session.commit()
+ with pytest.raises(ValidationError):
+ update_video(video.file_group_id)
+
+
+def test_update_video_title_empty(test_session, video_factory):
+ video = video_factory(with_info_json={'title': 'Old'})
+ test_session.commit()
+ with pytest.raises(ValidationError):
+ update_video(video.file_group_id, ' ')
+ test_session.expire_all()
+ assert Video.find_by_file_group_id(test_session, video.file_group_id).file_group.title == 'Old'
+
+
+def test_extract_video_info_json_prefers_custom_title(test_session, video_factory):
+ """The modeler's reader prefers the wrolpi custom title over fulltitle/title."""
+ video = video_factory(with_info_json={'title': 'Orig', 'fulltitle': 'Orig Full',
+ 'wrolpi': {'custom_title': 'Mine & yours'}})
+ assert extract_video_info_json(video).title == 'Mine & yours', 'HTML entities are unescaped as for any title'
+ video2 = video_factory(with_info_json={'title': 'Orig', 'fulltitle': 'Orig Full', 'wrolpi': {}})
+ assert extract_video_info_json(video2).title == 'Orig Full'
+
+
+@pytest.mark.asyncio
+async def test_custom_title_survives_refresh(test_session, test_directory, async_client, video_factory):
+ """A file refresh re-models the Video from its files; the custom title must win again."""
+ from wrolpi.files.worker import file_worker
+
+ video = video_factory(with_info_json={'title': 'Original'})
+ test_session.commit()
+ update_video(video.file_group_id, title='Custom', description='custom words')
+ video_path = video.video_path
+ file_group_id = video.file_group_id
+
+ # Force a full re-model, as a refresh of a modified file does.
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, file_group_id)
+ video.file_group.indexed = False
+ video.file_group.title = None
+ video.file_group.c_text = None
+ test_session.commit()
+
+ job_id = file_worker.queue_refresh([video_path], send_events=False)
+ await file_worker.wait_for_job(job_id)
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, file_group_id)
+ assert video.file_group.title == 'Custom'
+ assert video.get_description() == 'custom words'
+ assert json.loads(video.info_json_path.read_text())['title'] == 'Original'
+
+
+@pytest.mark.asyncio
+async def test_video_update_api(async_client, test_session, video_factory, wrol_mode_fixture):
+ video = video_factory(with_info_json={'title': 'Old'})
+ test_session.commit()
+
+ request, response = await async_client.put(f'/api/videos/{video.file_group_id}',
+ content=json.dumps({'title': 'Edited'}))
+ assert response.status == 200, response.json
+ assert response.json['file_group']['title'] == 'Edited'
+ assert response.json['file_group']['id'] == video.file_group_id
+
+ request, response = await async_client.put(f'/api/videos/{video.file_group_id}', content=json.dumps({'title': ''}))
+ assert response.status == 400
+
+ request, response = await async_client.put(f'/api/videos/{video.file_group_id}', content=json.dumps({}))
+ assert response.status == 400
+
+ request, response = await async_client.put(f'/api/videos/{video.file_group_id}',
+ content=json.dumps({'description': 'about it'}))
+ assert response.status == 200, response.json
+ assert response.json['file_group']['title'] == 'Edited', 'Title untouched'
+ request, response = await async_client.get(f'/api/videos/{video.file_group_id}/description')
+ assert response.json['description'] == 'about it'
+
+ request, response = await async_client.put('/api/videos/999999', content=json.dumps({'title': 'x'}))
+ assert response.status == 404
+
+ await wrol_mode_fixture(True)
+ request, response = await async_client.put(f'/api/videos/{video.file_group_id}',
+ content=json.dumps({'title': 'Nope'}))
+ assert response.status == 403
+ await wrol_mode_fixture(False)
+
+
+@pytest.mark.asyncio
+async def test_update_video_wrol_mode(test_session, video_factory, wrol_mode_fixture):
+ """The library function refuses in WROL Mode, not only the API route, so no caller can edit."""
+ from wrolpi.errors import WROLModeEnabled
+ video = video_factory(with_info_json={'title': 'Old'})
+ test_session.commit()
+
+ await wrol_mode_fixture(True)
+ try:
+ with pytest.raises(WROLModeEnabled):
+ update_video(video.file_group_id, title='Nope')
+ finally:
+ await wrol_mode_fixture(False)
+
+ test_session.expire_all()
+ assert Video.find_by_file_group_id(test_session, video.file_group_id).file_group.title == 'Old'
+
+
+@pytest.mark.asyncio
+async def test_edit_creates_info_json_that_survives_refresh(test_session, test_directory, async_client,
+ video_factory):
+ """A Video with no info json is edited (the edit creates one), then its file is refreshed: the
+ new sidecar must stay in the FileGroup and the custom title/description must still win."""
+ from wrolpi.files.worker import file_worker
+
+ video = video_factory()
+ test_session.commit()
+ video_path = video.video_path
+ file_group_id = video.file_group_id
+
+ update_video(file_group_id, title='Custom', description='my words')
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, file_group_id)
+ assert video.info_json_path and video.info_json_path.is_file()
+ assert video.get_description() == 'my words'
+
+ job_id = file_worker.queue_refresh([video_path], send_events=False)
+ await file_worker.wait_for_job(job_id)
+
+ test_session.expire_all()
+ video = Video.find_by_file_group_id(test_session, file_group_id)
+ assert video.info_json_path is not None, 'The refresh forgot the info json'
+ names = {i['path'] for i in video.file_group.files}
+ assert video_path.name in names and video.info_json_path.name in names, names
+ assert video.file_group.title == 'Custom'
+ assert video.get_description() == 'my words'
diff --git a/modules/videos/transcode.py b/modules/videos/transcode.py
index 2a97e8e9..701c2440 100644
--- a/modules/videos/transcode.py
+++ b/modules/videos/transcode.py
@@ -2,11 +2,14 @@ import asyncio
import os
import pathlib
import shutil
-from typing import List, Optional, Tuple
+from typing import List, Optional, Tuple, Callable
+from modules.videos.common import ffprobe_json, ffmpeg_video_complete_async
from wrolpi.captions import FFMPEG_BIN
from wrolpi.cmd import run_command
-from wrolpi.common import logger
+from wrolpi.common import logger, get_relative_to_media_directory, wrol_mode_enabled
+from wrolpi.db import get_db_session
+from wrolpi.jobs import register_job, get_current_job
from wrolpi.vars import DEFAULT_FILE_PERMISSIONS, PYTEST
logger = logger.getChild(__name__)
@@ -41,10 +44,45 @@ TRANSCODE_AUDIO_TARGETS = {
}
# Containers that can hold an h264/aac transcode result. webm cannot.
-TRANSCODE_CONTAINERS = ('mp4', 'mkv')
+TRANSCODE_CONTAINERS = ('mp4', 'mkv') # For output with a video stream.
+# For audio-only output: the video stream removed (`REMOVE_VIDEO`), or an audio-only source.
+TRANSCODE_AUDIO_CONTAINERS = ('m4a', 'ogg', 'mp3')
+# ffprobe codec names each audio container can hold without re-encoding.
+# Ordered: `default_audio_container` picks the first container that holds a codec, so the most
+# specific container comes first (mp3 fits in m4a too, but an .mp3 is what a user expects).
+AUDIO_CONTAINER_CODECS = {
+ 'mp3': {'mp3'},
+ 'ogg': {'opus', 'vorbis', 'flac'},
+ 'm4a': {'aac', 'mp3', 'alac'},
+}
+# `target_vcodec` value meaning "drop the video stream"; the output is an audio file.
+REMOVE_VIDEO = 'none'
+# ffprobe's `format_name` token for each container (mkv reports "matroska,webm").
+CONTAINER_FORMAT_NAMES = {'mp4': 'mp4', 'mkv': 'matroska', 'm4a': 'm4a', 'ogg': 'ogg', 'mp3': 'mp3'}
+# `-movflags +faststart` is an mp4-muxer option; other muxers warn about it.
+FASTSTART_CONTAINERS = ('mp4', 'm4a')
+# Fragmented mp4 (`fragmented=True`): a tiny moov plus a global seek index, samples described per
+# fragment. A very long video's whole-file index can reach tens of MB, which a phone must fetch
+# and parse before the first frame; fragments make it start and seek quickly. mp4/m4a only.
+FRAGMENTED_MOVFLAGS = '+frag_keyframe+empty_moov+default_base_moof+global_sidx'
+FRAGMENT_DURATION_US = 10_000_000
+
+
+def default_audio_container(codec: Optional[str]) -> str:
+ """The natural audio container for a codec: what a player expects from the suffix."""
+ for container, codecs in AUDIO_CONTAINER_CODECS.items():
+ if codec in codecs:
+ return container
+ return 'm4a'
TRANSCODE_TIMEOUT = 4 * 60 * 60 # A long video on a Raspberry Pi can take hours.
+# Output verification (see `verify_transcode_output`): the output's duration may differ from the
+# source by this much before it is considered truncated. Muxers trim a frame or two; a truncated
+# file is off by minutes.
+DURATION_TOLERANCE_SECONDS = 1.0
+DURATION_TOLERANCE_RATIO = 0.01
+
# Transcoding writes a whole new copy of the video next to the original.
MINIMUM_FREE_SPACE_RATIO = 1.5
@@ -114,34 +152,178 @@ async def _acquire_transcode_lock():
async def transcode_video_file(video_path: pathlib.Path,
target_vcodec: Optional[str] = None,
target_acodec: Optional[str] = None,
- container: str = 'mp4') -> pathlib.Path:
+ container: str = 'mp4',
+ duration: Optional[float] = None,
+ keep_original: bool = False,
+ fragmented: bool = False) -> pathlib.Path:
"""Transcode a video file in place (same stem, possibly a new container extension).
Only one transcode runs at a time machine-wide (across all Sanic workers); this call waits
for its turn. Only the stream(s) with a target are re-encoded; the other stream is copied.
The output is written to a temporary file in the same directory, then atomically renamed over
- the final path. The original file is deleted if the extension changed. Returns the final
- path.
+ the final path. The original file is deleted if the extension changed, unless
+ `keep_original`: a caller with database records pointing at the original must retarget them
+ first, then delete it (see `transcode_video_job`), so a crash never leaves the records pointing
+ at a deleted file. Returns the final path.
+
+ When run inside a Job (see `wrolpi.jobs`), ffmpeg's output is captured into the Job log and, if
+ `duration` (seconds) is known, progress is reported to the Job.
@raise RuntimeError: when ffmpeg fails, or there is not enough free disk space.
"""
if not FFMPEG_BIN:
raise RuntimeError('ffmpeg was not found')
- if not target_vcodec and not target_acodec:
- raise RuntimeError('Refusing to transcode without a target codec')
- if container not in TRANSCODE_CONTAINERS:
- logger.info(f'Forcing mp4 container for transcode of {video_path} ({container} is not supported)')
- container = 'mp4'
+ # No target for either stream is a remux: both streams are copied into the (new) container.
+ # `target_vcodec=REMOVE_VIDEO` (or an audio-only source) makes the output an audio file; see
+ # `resolve_output`, which also settles an unsuitable container.
lock = await _acquire_transcode_lock()
try:
- return await _transcode_video_file(video_path, target_vcodec, target_acodec, container)
+ return await _transcode_video_file(video_path, target_vcodec, target_acodec, container, duration,
+ keep_original, fragmented)
finally:
lock.release()
+async def probe_for_verify(path: pathlib.Path) -> dict:
+ """ffprobe a file for `verify_transcode_output`. Separate so tests can replace it."""
+ return await ffprobe_json(path)
+
+
+def _format_names(probe: dict) -> List[str]:
+ return str((probe.get('format') or {}).get('format_name') or '').split(',')
+
+
+def _duration(probe: dict) -> Optional[float]:
+ try:
+ return float((probe.get('format') or {}).get('duration'))
+ except (TypeError, ValueError):
+ return None
+
+
+def verify_transcode_output(source: dict, output: dict, target_vcodec: Optional[str],
+ target_acodec: Optional[str], container: str):
+ """Compare ffprobe results of the source and the transcoded output; the output must be a
+ complete rendition of the source before the source is replaced by it.
+
+ Checks: the output parses (its index was written), it is in the requested container, it has a
+ video stream (and an audio stream when the source has one), each stream's codec is the target
+ or, when copied, the source's, and the duration matches within a tolerance.
+
+ @raise RuntimeError: describing the first failed check.
+ """
+ if not output or output.get('streams') is None:
+ raise RuntimeError('Transcode output could not be probed (no streams); its index may be missing')
+
+ expected_format = CONTAINER_FORMAT_NAMES.get(container, container)
+ if expected_format not in _format_names(output):
+ raise RuntimeError(f'Transcode output container is {_format_names(output)}, expected {container}')
+
+ source_video = get_stream_codec_names(source, 'video')
+ source_audio = get_stream_codec_names(source, 'audio')
+ output_video = get_stream_codec_names(output, 'video')
+ output_audio = get_stream_codec_names(output, 'audio')
+
+ audio_only = target_vcodec == REMOVE_VIDEO or not source_video
+ if audio_only:
+ if output_video:
+ raise RuntimeError(f'Transcode output still has a video stream ({output_video[0]})')
+ if not output_audio:
+ raise RuntimeError('Transcode output has no audio stream')
+ else:
+ if not output_video:
+ raise RuntimeError('Transcode output has no video stream')
+ if source_audio and not output_audio:
+ raise RuntimeError('Transcode output has no audio stream, but the source has one')
+ expected_video = target_vcodec or (source_video[0] if source_video else None)
+ if expected_video and output_video[0] != expected_video:
+ raise RuntimeError(f'Transcode output video codec is {output_video[0]}, expected {expected_video}')
+ expected_audio = target_acodec or (source_audio[0] if source_audio else None)
+ if output_audio and expected_audio and output_audio[0] != expected_audio:
+ raise RuntimeError(f'Transcode output audio codec is {output_audio[0]}, expected {expected_audio}')
+
+ source_duration, output_duration = _duration(source), _duration(output)
+ if source_duration is not None:
+ if output_duration is None:
+ raise RuntimeError('Transcode output has no duration')
+ tolerance = max(DURATION_TOLERANCE_SECONDS, DURATION_TOLERANCE_RATIO * source_duration)
+ if abs(source_duration - output_duration) > tolerance:
+ raise RuntimeError(
+ f'Transcode output duration is {output_duration:.1f}s, source is {source_duration:.1f}s')
+
+
+async def _verify_tail_decodes(path: pathlib.Path, runner: Callable):
+ """The same completeness check a download gets (`ffmpeg_video_complete`), as a hard failure.
+
+ @raise RuntimeError: when the tail of `path` does not decode cleanly."""
+ ok, errors = await ffmpeg_video_complete_async(path, runner=runner)
+ if not ok:
+ raise RuntimeError(f'Transcode output failed to decode: {errors[-2000:] or "ffmpeg exited non-zero"}')
+
+
+def make_ffmpeg_progress_parser(duration: Optional[float], set_progress: Callable[[float], None]) \
+ -> Callable[[str], None]:
+ """Return a callback for `ffmpeg -progress pipe:1` stdout lines which reports percent complete.
+
+ ffmpeg emits `out_time_us=` (older builds: `out_time_ms`, also microseconds)."""
+ last = [-1]
+
+ def parse(line: str):
+ if not duration or duration <= 0:
+ return
+ key, sep, value = line.partition('=')
+ if not sep or key not in ('out_time_us', 'out_time_ms'):
+ return
+ try:
+ seconds = int(value) / 1_000_000
+ except ValueError:
+ return
+ percent = int(min(100.0, max(0.0, 100.0 * seconds / duration)))
+ if percent != last[0]:
+ last[0] = percent
+ set_progress(percent)
+
+ return parse
+
+
+def resolve_output(source_probe: dict, target_vcodec: Optional[str], target_acodec: Optional[str],
+ container: str) -> Tuple[bool, str]:
+ """Decide whether the output is audio-only and which container it gets.
+
+ Audio-only when the video is to be removed, or the source has no video stream. An audio
+ container that cannot hold the (copied or target) audio codec is refused; a video container
+ asked for an audio-only output is replaced by the codec's natural one.
+
+ @raise RuntimeError: when the request cannot be satisfied by this source."""
+ source_video = get_stream_codec_names(source_probe, 'video')
+ source_audio = get_stream_codec_names(source_probe, 'audio')
+
+ if target_vcodec and target_vcodec != REMOVE_VIDEO and not source_video:
+ raise RuntimeError(f'Cannot transcode video to {target_vcodec}: the file has no video stream')
+
+ audio_only = target_vcodec == REMOVE_VIDEO or not source_video
+ if not audio_only:
+ if container not in TRANSCODE_CONTAINERS:
+ logger.info(f'Forcing mp4 container ({container} cannot hold a video stream)')
+ container = 'mp4'
+ return False, container
+
+ if not source_audio:
+ raise RuntimeError('Cannot produce an audio file: the source has no audio stream')
+ effective_acodec = target_acodec or source_audio[0]
+ if container not in TRANSCODE_AUDIO_CONTAINERS:
+ container = default_audio_container(effective_acodec)
+ logger.info(f'Using {container} container for audio-only output ({effective_acodec})')
+ elif effective_acodec not in AUDIO_CONTAINER_CODECS[container]:
+ raise RuntimeError(
+ f'{effective_acodec} audio cannot be stored in {container}; choose another container or codec')
+ return True, container
+
+
async def _transcode_video_file(video_path: pathlib.Path, target_vcodec: Optional[str],
- target_acodec: Optional[str], container: str) -> pathlib.Path:
+ target_acodec: Optional[str], container: str,
+ duration: Optional[float] = None, keep_original: bool = False,
+ fragmented: bool = False) -> pathlib.Path:
"""The ffmpeg work of `transcode_video_file`; the caller holds the transcode lock."""
# Notify here, after the lock: the wait for another transcode can last hours, and the user
# should hear "Transcoding" only when this file's work actually begins.
@@ -158,28 +340,63 @@ async def _transcode_video_file(video_path: pathlib.Path, target_vcodec: Optiona
raise RuntimeError(
f'Not enough free space to transcode {video_path} ({free} bytes free, source is {source_size} bytes)')
- video_args = TRANSCODE_VIDEO_TARGETS[target_vcodec] if target_vcodec else ('-c:v', 'copy')
+ # Probed before ffmpeg: the command depends on the source's streams, and verification compares
+ # against the file as it was.
+ source_probe = await probe_for_verify(video_path)
+ audio_only, container = resolve_output(source_probe, target_vcodec, target_acodec, container)
+
audio_args = TRANSCODE_AUDIO_TARGETS[target_acodec] if target_acodec else ('-c:a', 'copy')
+ if audio_only:
+ # `-vn` also drops embedded cover art; the audio stream is required.
+ map_args = ('-map', '0:a:0', '-vn')
+ video_args = ()
+ else:
+ # 0:V:0 excludes attached-picture streams (embedded thumbnails); 0:a:0? tolerates a video
+ # with no audio stream.
+ map_args = ('-map', '0:V:0', '-map', '0:a:0?')
+ video_args = TRANSCODE_VIDEO_TARGETS[target_vcodec] if target_vcodec else ('-c:v', 'copy')
+ if container in FASTSTART_CONTAINERS and fragmented:
+ mux_args = ('-movflags', FRAGMENTED_MOVFLAGS, '-frag_duration', str(FRAGMENT_DURATION_US))
+ elif container in FASTSTART_CONTAINERS:
+ mux_args = ('-movflags', '+faststart')
+ else:
+ mux_args = ()
final_path = video_path.with_suffix(f'.{container}')
tmp_path = video_path.with_suffix(f'.transcode.{container}')
- # 0:V:0 excludes attached-picture streams (embedded thumbnails); 0:a:0? tolerates a video
- # with no audio stream.
cmd = (FFMPEG_BIN, '-y',
'-i', video_path,
- '-map', '0:V:0', '-map', '0:a:0?',
+ *map_args,
*video_args,
*audio_args,
- '-movflags', '+faststart',
+ *mux_args,
+ # Machine-readable progress on stdout (the human progress line is suppressed).
+ '-nostats', '-progress', 'pipe:1',
str(tmp_path))
logger.warning(f'Transcoding {video_path} to {final_path} (video={target_vcodec}, audio={target_acodec})')
try:
- result = await run_command(cmd, cwd=video_path.parent, timeout=TRANSCODE_TIMEOUT)
+ job = get_current_job()
+ runner = job.run_command if job else run_command
+ if job:
+ # Inside a Job: ffmpeg's output lands in the Job log and progress is reported.
+ result = await job.run_command(cmd, cwd=video_path.parent, timeout=TRANSCODE_TIMEOUT,
+ stdout_callback=make_ffmpeg_progress_parser(duration, job.set_progress))
+ else:
+ result = await run_command(cmd, cwd=video_path.parent, timeout=TRANSCODE_TIMEOUT)
if result.return_code != 0:
raise RuntimeError(
f'ffmpeg exited with {result.return_code} while transcoding {video_path}:'
f'\n{result.stderr.decode()[-2000:]}')
- except Exception:
+
+ # The original is only replaced by a complete output: compare probes, then prove the tail
+ # decodes. A failure here leaves the original untouched.
+ logger.info(f'Verifying transcode output {tmp_path}')
+ output_probe = await probe_for_verify(tmp_path)
+ verify_transcode_output(source_probe, output_probe, target_vcodec, target_acodec, container)
+ await _verify_tail_decodes(tmp_path, runner)
+ except BaseException:
+ # BaseException: a cancelled Job's run_command raises CancelledError, which must not leave
+ # the partial output behind for a refresh to index.
tmp_path.unlink(missing_ok=True)
raise
@@ -194,7 +411,7 @@ async def _transcode_video_file(video_path: pathlib.Path, target_vcodec: Optiona
# The old ffprobe sidecar describes the old streams.
video_path.with_suffix('.ffprobe.json').unlink(missing_ok=True)
tmp_path.rename(final_path)
- if final_path != video_path:
+ if final_path != video_path and not keep_original:
video_path.unlink()
final_path.chmod(DEFAULT_FILE_PERMISSIONS)
@@ -207,3 +424,89 @@ async def _transcode_video_file(video_path: pathlib.Path, target_vcodec: Optiona
finally:
os.close(fd)
return final_path
+
+
+def validate_transcode_request(video_codec: Optional[str], audio_codec: Optional[str], container: str,
+ fragmented: bool = False):
+ """@raise ValueError: when the request cannot be transcoded. No codec at all is a remux;
+ `video_codec=REMOVE_VIDEO` drops the video stream and needs an audio container; `fragmented`
+ needs an mp4 container."""
+ if video_codec and video_codec != REMOVE_VIDEO and video_codec not in TRANSCODE_VIDEO_TARGETS:
+ raise ValueError(f'Cannot transcode video to {video_codec!r}; supported: {sorted(TRANSCODE_VIDEO_TARGETS)}')
+ if audio_codec and audio_codec not in TRANSCODE_AUDIO_TARGETS:
+ raise ValueError(f'Cannot transcode audio to {audio_codec!r}; supported: {sorted(TRANSCODE_AUDIO_TARGETS)}')
+ containers = TRANSCODE_CONTAINERS + TRANSCODE_AUDIO_CONTAINERS
+ if container not in containers:
+ raise ValueError(f'Unsupported container {container!r}; supported: {containers}')
+ if fragmented and container not in FASTSTART_CONTAINERS:
+ raise ValueError(f'A fragmented file must be mp4 (or m4a), not {container}')
+ if video_codec == REMOVE_VIDEO and container not in TRANSCODE_AUDIO_CONTAINERS:
+ raise ValueError(f'Removing the video needs an audio container ({TRANSCODE_AUDIO_CONTAINERS}), not {container}')
+ if container in TRANSCODE_AUDIO_CONTAINERS:
+ if video_codec and video_codec != REMOVE_VIDEO:
+ raise ValueError(f'A video stream cannot be stored in {container}; remove the video or choose a video container')
+ if audio_codec and audio_codec not in AUDIO_CONTAINER_CODECS[container]:
+ raise ValueError(f'{audio_codec} audio cannot be stored in {container}; '
+ f'it holds {sorted(AUDIO_CONTAINER_CODECS[container])}')
+
+
+def retarget_file_group(file_group_id: int, old_path: pathlib.Path, new_path: pathlib.Path):
+ """Point a Video's FileGroup at the transcoded file and mark it for re-modeling. Committed."""
+ from modules.videos.models import Video
+ from wrolpi.files.lib import get_mimetype
+
+ with get_db_session(commit=True) as session:
+ video = Video.find_by_file_group_id(session, file_group_id)
+ file_group = video.file_group
+ if new_path != old_path:
+ mimetype = get_mimetype(new_path)
+ file_group.files = [
+ {**i, 'path': new_path.name, 'mimetype': mimetype} if i['path'] == old_path.name else i
+ for i in file_group.files
+ ]
+ file_group.primary_path = new_path
+ file_group.mimetype = mimetype
+ # The streams changed: the modeler must ffprobe the file again.
+ video.ffprobe_json = None
+ file_group.indexed = False
+
+
+@register_job('transcode_video')
+async def transcode_video_job(file_group_id: int, video_codec: Optional[str] = None,
+ audio_codec: Optional[str] = None, container: str = 'mp4',
+ fragmented: bool = False) -> dict:
+ """Transcode a Video's file, then re-index it so the FileGroup reflects the new file.
+
+ Run as a Job (`transcode_video_job.enqueue(...)`) so the user can watch and cancel it."""
+ from modules.videos.models import Video
+ from wrolpi.files.worker import file_worker
+
+ # The API refuses to queue in WROL Mode; this catches a Job queued before it was enabled.
+ if wrol_mode_enabled():
+ raise RuntimeError('Cannot transcode while WROL Mode is enabled')
+
+ validate_transcode_request(video_codec, audio_codec, container, fragmented)
+
+ with get_db_session() as session:
+ video = Video.find_by_file_group_id(session, file_group_id)
+ video_path = video.video_path
+ duration = video.file_group.length
+ if not video_path or not video_path.is_file():
+ raise RuntimeError(f'Video file does not exist: {video_path}')
+
+ # The original is kept until the database points at the new file (below).
+ final_path = await transcode_video_file(video_path, video_codec, audio_codec, container, duration=duration,
+ keep_original=True, fragmented=fragmented)
+
+ # Point the existing FileGroup at the new file *before* deleting the original and *before*
+ # refreshing. A crash after the delete would leave the database pointing at a missing file;
+ # a refresh which finds a new filename on disk would delete this FileGroup and create another,
+ # losing tags and history.
+ retarget_file_group(file_group_id, video_path, final_path)
+ if final_path != video_path:
+ video_path.unlink(missing_ok=True)
+
+ job_id = file_worker.queue_refresh([final_path], send_events=False)
+ await file_worker.wait_for_job(job_id)
+
+ return {'path': str(get_relative_to_media_directory(final_path))}
diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py
index ac1bc60f..c1ca769c 100644
--- a/modules/videos/video/api.py
+++ b/modules/videos/video/api.py
@@ -7,11 +7,12 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
-from wrolpi.common import logger
-from wrolpi.errors import InvalidOrderBy
+from wrolpi.common import logger, wrol_mode_check
+from wrolpi.errors import InvalidOrderBy, InvalidJob
from wrolpi.schema import JSONErrorResponse
from . import lib
from .. import schema
+from ..transcode import transcode_video_job, validate_transcode_request, REMOVE_VIDEO
video_bp = Blueprint('Video', '/api/videos')
@@ -28,6 +29,20 @@ def video_get(request: Request, file_group_id: int):
return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
+@video_bp.put('/')
+@openapi.description('Edit a Video\'s details. Written to its info json; the Video is validated again.')
+@openapi.body({'application/json': schema.VideoUpdateRequest})
+@openapi.response(HTTPStatus.OK, schema.VideoResponse)
+@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
+@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
+@validate(schema.VideoUpdateRequest)
+@wrol_mode_check
+async def video_update(_: Request, file_group_id: int, body: schema.VideoUpdateRequest):
+ lib.update_video(file_group_id, title=body.title, description=body.description)
+ video, previous_video, next_video = lib.get_video_for_app(file_group_id)
+ return json_response({'file_group': video, 'prev': previous_video, 'next': next_video})
+
+
@video_bp.get('//comments')
@openapi.description('Get Video comments')
@openapi.response(HTTPStatus.OK, schema.VideoCommentsResponse)
@@ -55,6 +70,41 @@ def video_get_captions(_: Request, file_group_id: int):
return json_response({'captions': video.get_caption_chunks()})
+@video_bp.post('//transcode')
+@openapi.description('Queue a Job which transcodes the Video to the requested codecs/container.')
+@openapi.body({'application/json': schema.VideoTranscodeRequest})
+@openapi.response(HTTPStatus.OK, schema.VideoTranscodeResponse)
+@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
+@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
+@validate(schema.VideoTranscodeRequest)
+@wrol_mode_check
+async def video_transcode(_: Request, file_group_id: int, body: schema.VideoTranscodeRequest):
+ try:
+ validate_transcode_request(body.video_codec, body.audio_codec, body.container, body.fragmented)
+ except ValueError as e:
+ raise InvalidJob(str(e))
+ # Raises UnknownVideo (404) before anything is queued.
+ video = lib.get_video(file_group_id)
+ is_audio = (video.file_group.mimetype or '').startswith('audio/')
+ if is_audio and body.video_codec and body.video_codec != REMOVE_VIDEO:
+ raise InvalidJob(f'This is an audio file; it has no video stream to transcode to {body.video_codec}')
+ if body.video_codec == REMOVE_VIDEO and not is_audio:
+ action = 'Extract audio from'
+ elif not body.video_codec and not body.audio_codec:
+ action = 'Remux'
+ else:
+ action = 'Transcode'
+ job_id = transcode_video_job.enqueue(
+ description=f'{action} {video.video_path.name}',
+ file_group_id=file_group_id,
+ video_codec=body.video_codec,
+ audio_codec=body.audio_codec,
+ container=body.container,
+ fragmented=body.fragmented,
+ )
+ return json_response({'job_id': job_id})
+
+
@video_bp.post('/search')
@openapi.definition(
summary='Search Video titles and captions',
diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py
index b7424835..dfad5d54 100644
--- a/modules/videos/video/lib.py
+++ b/modules/videos/video/lib.py
@@ -50,6 +50,46 @@ def get_video(file_group_id: int) -> Video:
return video
+@wrol_mode_check
+def update_video(file_group_id: int, title: str = None, description: str = None) -> Video:
+ """Edit a Video's details. The files are the source of truth, so the change is written to the
+ Video's .info.json (created if missing) and the Video is validated again from it. Only the
+ fields given (not None) are changed; an empty description clears it.
+
+ Edits go into the info json's `wrolpi` section (`custom_title`, `custom_description`), never
+ over yt-dlp's own keys: the original title and description stay in the file, and a metadata
+ re-download carries the section over.
+
+ @raise UnknownVideo: if the Video can not be found
+ @raise ValidationError: when nothing is given, or the title is empty
+ """
+ from wrolpi.errors import ValidationError
+ if title is None and description is None:
+ raise ValidationError('Nothing to change')
+ if title is not None:
+ title = title.strip()
+ if not title:
+ raise ValidationError('Video title cannot be empty')
+
+ changes = dict()
+ if title is not None:
+ changes['custom_title'] = title
+ if description is not None:
+ changes['custom_description'] = description
+
+ with get_db_session(commit=True) as session:
+ video = Video.find_by_file_group_id(session, file_group_id)
+ if video.info_json_path is None or not video.info_json_path.is_file():
+ # No info json yet: create one holding only the wrolpi section.
+ video.replace_info_json({'wrolpi': changes}, clean=False)
+ else:
+ video.file_group.update_wrolpi_json(changes)
+ # Re-derive the title (and search text) from the file just written.
+ video.validate(session)
+ video.flush()
+ return video
+
+
VIDEO_ORDERS = {
# fg.id (the rowid) is the pagination tiebreaker: it is present in every index, so these
# orders stay index-only. LOWER(fg.primary_path) forced SQLite to read every (large) row.
diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py
index e1c0b73b..9e94a7af 100644
--- a/wrolpi/contexts.py
+++ b/wrolpi/contexts.py
@@ -106,6 +106,12 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.file_worker_status = manager.dict()
app.shared_ctx.file_worker_jobs = manager.dict()
+ # Jobs (wrolpi/jobs.py): FIFO queue of registered functions run one at a time by the
+ # perpetual-tasks owner. Never persisted; an API restart loses them.
+ app.shared_ctx.jobs = manager.dict()
+ app.shared_ctx.jobs_queue = manager.Queue()
+ app.shared_ctx.jobs_lock = multiprocessing.Lock()
+
reset_shared_contexts(app)
@@ -205,6 +211,14 @@ def reset_shared_contexts(app: Sanic):
break
app.shared_ctx.file_worker_status.clear()
app.shared_ctx.file_worker_jobs.clear()
+
+ # Jobs
+ while True:
+ try:
+ app.shared_ctx.jobs_queue.get_nowait()
+ except queue.Empty:
+ break
+ app.shared_ctx.jobs.clear()
app.shared_ctx.file_worker_status.update(dict(
status='idle',
task_type=None,
diff --git a/wrolpi/errors.py b/wrolpi/errors.py
index 76a822bf..da7a6939 100644
--- a/wrolpi/errors.py
+++ b/wrolpi/errors.py
@@ -211,3 +211,15 @@ class InvalidArchiveMember(APIError):
code = 'INVALID_ARCHIVE_MEMBER'
summary = 'The requested member does not exist in the archive.'
status_code = HTTPStatus.BAD_REQUEST
+
+
+class UnknownJob(APIError):
+ code = 'UNKNOWN_JOB'
+ summary = 'The Job cannot be found.'
+ status_code = HTTPStatus.NOT_FOUND
+
+
+class InvalidJob(APIError):
+ code = 'INVALID_JOB'
+ summary = 'The Job request is invalid.'
+ status_code = HTTPStatus.BAD_REQUEST
diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py
index b5656f88..81b54984 100644
--- a/wrolpi/files/models.py
+++ b/wrolpi/files/models.py
@@ -432,6 +432,19 @@ class FileGroup(ModelHelper, Base):
if json_path.name not in tracked_paths:
self.append_files(json_path)
+ def get_wrolpi_json(self) -> dict:
+ """The `wrolpi` section of this FileGroup's metadata JSON: WROLPi-owned keys (a user's
+ custom title, the parent download) that yt-dlp/readability never write. Empty if none."""
+ json_path = self.metadata_json_path
+ if not json_path or not json_path.is_file():
+ return dict()
+ try:
+ contents = json.loads(json_path.read_text())
+ except (OSError, ValueError):
+ return dict()
+ section = contents.get('wrolpi') if isinstance(contents, dict) else None
+ return dict(section) if isinstance(section, dict) else dict()
+
def update_wrolpi_json(self, data: dict):
"""Read this FileGroup's metadata JSON, merge `data` into its `wrolpi` section, and write back.
diff --git a/wrolpi/jobs.py b/wrolpi/jobs.py
new file mode 100644
index 00000000..44acb5c0
--- /dev/null
+++ b/wrolpi/jobs.py
@@ -0,0 +1,438 @@
+"""In-memory FIFO queue of background Jobs.
+
+A Job is a call of a *registered* function with JSON-able kwargs. Any Sanic worker may enqueue a
+Job; the perpetual-tasks owner process runs them one at a time, in order. Python logging emitted
+while a Job runs, and the output of subprocesses started through `JobContext.run_command`, are
+captured into the Job's log. A caller may wait for a Job, poll it, or cancel it.
+
+Jobs are never written to disk: when the API restarts they are lost.
+
+>>> @register_job('example')
+>>> async def example(path: str):
+>>> job = get_current_job()
+>>> job.set_progress(50)
+>>> await job.run_command(('ls', path))
+
+>>> job_id = enqueue_job('example', description='List a directory', path='/tmp')
+>>> await wait_for_job(job_id)
+
+Functions are registered by name (rather than enqueued as objects) because kwargs and the call
+must cross the process boundary between Sanic workers; a name and JSON kwargs always can.
+"""
+import asyncio
+import contextvars
+import inspect
+import json
+import logging
+import multiprocessing
+import os
+import pathlib
+import queue
+import time
+import uuid
+from collections import deque
+from functools import partial
+from http import HTTPStatus
+from typing import Callable, Dict, List, Optional
+
+from sanic import Blueprint
+from sanic.request import Request
+from sanic_ext.extensions.openapi import openapi
+
+from wrolpi.api_utils import api_app, json_response, perpetual_signal
+from wrolpi.common import logger
+from wrolpi.dates import now
+from wrolpi.errors import UnknownJob, InvalidJob
+from wrolpi.events import Events
+from wrolpi.schema import JSONErrorResponse
+from wrolpi.vars import PYTEST
+
+logger = logger.getChild(__name__)
+
+JOB_HANDLERS: Dict[str, Callable] = dict()
+
+PENDING = 'pending'
+RUNNING = 'running'
+COMPLETE = 'complete'
+FAILED = 'failed'
+CANCELLED = 'cancelled'
+FINISHED_STATUSES = (COMPLETE, FAILED, CANCELLED)
+
+# Only the tail of a Job's log is kept; ffmpeg alone can emit thousands of lines.
+JOB_LOG_LINES = 200
+# Finished Jobs kept for the UI/API before the oldest are forgotten.
+JOB_HISTORY_LIMIT = 100
+# Pending Jobs allowed at once; `enqueue_job` refuses past this so a runaway client cannot grow
+# shared memory without bound.
+JOB_PENDING_LIMIT = 100
+# How often the running Job checks whether it has been cancelled from another process.
+JOB_CANCEL_POLL_SECONDS = 0.05 if PYTEST else 1
+# The Job log lives in this process; it is copied to the shared dict at most this often.
+JOB_LOG_FLUSH_SECONDS = 0 if PYTEST else 0.5
+
+
+def _jobs() -> dict:
+ try:
+ return api_app.shared_ctx.jobs
+ except AttributeError:
+ raise RuntimeError('Sanic shared_ctx has not been initialized. If testing, use `async_client` fixture.')
+
+
+def _queue():
+ return api_app.shared_ctx.jobs_queue
+
+
+def _lock() -> multiprocessing.Lock:
+ return api_app.shared_ctx.jobs_lock
+
+
+def register_job(name: str):
+ """Register a function that can be run as a Job by `name`.
+
+ The function may be sync or async, and is called with the kwargs given to `enqueue_job`. Use
+ `get_current_job()` inside the function for logging, progress, subprocesses, and cancellation.
+ The wrapped function gains an `enqueue(description=None, **kwargs)` method.
+ """
+
+ def wrapper(func: Callable):
+ if name in JOB_HANDLERS and not PYTEST:
+ raise RuntimeError(f'register_job: job name already taken {name}')
+ JOB_HANDLERS[name] = func
+ setattr(func, 'enqueue', partial(enqueue_job, name))
+ return func
+
+ return wrapper
+
+
+def enqueue_job(name: str, description: str = None, **kwargs) -> str:
+ """Add a Job to the end of the queue. Returns its id.
+
+ @raise InvalidJob: when `name` is not registered, or kwargs are not JSON serializable.
+ """
+ if name not in JOB_HANDLERS:
+ raise InvalidJob(f'No job is registered with the name {name!r}')
+ try:
+ json.dumps(kwargs)
+ except (TypeError, ValueError) as e:
+ raise InvalidJob(f'Job kwargs must be JSON serializable: {e}')
+
+ job_id = f'{name}-{uuid.uuid4().hex[:8]}'
+ record = dict(
+ id=job_id,
+ name=name,
+ description=description or name,
+ kwargs=kwargs,
+ status=PENDING,
+ created_at=now().isoformat(),
+ started_at=None,
+ finished_at=None,
+ error=None,
+ progress=None,
+ result=None,
+ log=[],
+ pid=None,
+ cancel_requested=False,
+ )
+ with _lock():
+ pending = sum(1 for i in _jobs().values() if i['status'] == PENDING)
+ if pending >= JOB_PENDING_LIMIT:
+ raise InvalidJob(f'Too many pending jobs ({pending}); try again once some have finished')
+ _jobs()[job_id] = record
+ _prune_finished_jobs()
+ _queue().put_nowait(job_id)
+ logger.info(f'enqueue_job: {job_id} ({description or name})')
+ return job_id
+
+
+def get_job(job_id: str) -> dict:
+ """@raise UnknownJob: when no Job has this id."""
+ record = _jobs().get(job_id)
+ if not record:
+ raise UnknownJob(f'Job {job_id} does not exist')
+ return dict(record)
+
+
+def get_jobs() -> List[dict]:
+ """All Jobs, oldest first."""
+ return sorted((dict(i) for i in _jobs().values()), key=lambda i: i['created_at'])
+
+
+def _update_job(job_id: str, **changes):
+ """Manager dict values are copies; a change must be written back as a whole record."""
+ with _lock():
+ record = _jobs().get(job_id)
+ if record is None:
+ return
+ record = dict(record)
+ record.update(changes)
+ _jobs()[job_id] = record
+
+
+def _prune_finished_jobs():
+ """Forget the oldest finished Jobs beyond `JOB_HISTORY_LIMIT`. The caller holds the lock."""
+ jobs = _jobs()
+ finished = sorted((i for i in jobs.values() if i['status'] in FINISHED_STATUSES),
+ key=lambda i: i['finished_at'] or '')
+ for record in finished[:max(0, len(finished) - JOB_HISTORY_LIMIT)]:
+ del jobs[record['id']]
+
+
+def cancel_job(job_id: str) -> dict:
+ """Cancel a pending Job immediately, or ask the running Job to stop.
+
+ A running Job's handler is cancelled (its `run_command` subprocess is killed) the next time the
+ worker polls. Finished Jobs are unchanged.
+ """
+ with _lock():
+ record = _jobs().get(job_id)
+ if not record:
+ raise UnknownJob(f'Job {job_id} does not exist')
+ record = dict(record)
+ if record['status'] == PENDING:
+ record.update(status=CANCELLED, finished_at=now().isoformat())
+ logger.info(f'cancel_job: cancelled pending {job_id}')
+ elif record['status'] == RUNNING:
+ record['cancel_requested'] = True
+ logger.info(f'cancel_job: requested cancel of running {job_id}')
+ _jobs()[job_id] = record
+ return record
+
+
+def _claim_job(job_id: str) -> Optional[dict]:
+ """Move a Job from PENDING to RUNNING in one step under the lock; returns the running record,
+ or None when the Job is no longer pending (cancelled while queued, or already claimed)."""
+ with _lock():
+ record = _jobs().get(job_id)
+ if not record or record['status'] != PENDING:
+ return None
+ record = dict(record)
+ record.update(status=RUNNING, started_at=now().isoformat(), pid=os.getpid())
+ _jobs()[job_id] = record
+ return record
+
+
+def _cancel_requested(job_id: str) -> bool:
+ record = _jobs().get(job_id)
+ return bool(record and record.get('cancel_requested'))
+
+
+class JobContext:
+ """The running Job, available to its handler via `get_current_job()`."""
+
+ def __init__(self, job_id: str, name: str):
+ self.id = job_id
+ self.name = name
+ self._log: deque = deque(maxlen=JOB_LOG_LINES)
+ self._last_flush = 0.0
+ self._dirty = False
+
+ def log(self, line: str):
+ """Append a line to the Job's log. Must never call `logger` (the log handler would recurse)."""
+ self._log.append(str(line).rstrip())
+ self._dirty = True
+ if time.monotonic() - self._last_flush >= JOB_LOG_FLUSH_SECONDS:
+ self.flush()
+
+ def flush(self):
+ if not self._dirty:
+ return
+ self._last_flush = time.monotonic()
+ self._dirty = False
+ _update_job(self.id, log=list(self._log))
+
+ def set_progress(self, percent: Optional[float]):
+ """Report progress 0-100 to the UI."""
+ if percent is not None:
+ percent = max(0.0, min(100.0, float(percent)))
+ _update_job(self.id, progress=percent)
+
+ def cancel_requested(self) -> bool:
+ return _cancel_requested(self.id)
+
+ async def run_command(self, cmd: tuple[str | pathlib.Path, ...], stdout_callback: Callable = None, **kwargs):
+ """`wrolpi.cmd.run_command` whose stdout lines (live) and stderr tail (on exit) are
+ captured into this Job's log. `stdout_callback` still receives each stdout line."""
+ from wrolpi.cmd import run_command
+
+ def _callback(line: str):
+ self.log(line)
+ if stdout_callback:
+ stdout_callback(line)
+
+ # Its own session: killing the command kills anything it spawned, as elsewhere in WROLPi.
+ kwargs.setdefault('start_new_session', True)
+ result = await run_command(cmd, stdout_callback=_callback, **kwargs)
+ if result.stderr:
+ for line in result.stderr.decode(errors='replace').splitlines()[-JOB_LOG_LINES:]:
+ if line.strip():
+ self.log(line)
+ self.flush()
+ if result.cancelled:
+ # `run_command` swallows the cancellation and returns; a handler checking return codes
+ # must not mistake a killed command for a normal exit.
+ raise asyncio.CancelledError(f'Command was cancelled: {cmd[0]}')
+ return result
+
+
+_current_job: contextvars.ContextVar[Optional[JobContext]] = contextvars.ContextVar('current_job', default=None)
+
+
+def get_current_job() -> Optional[JobContext]:
+ """The Job whose handler is running in this task, or None outside of a Job."""
+ return _current_job.get()
+
+
+class JobLogHandler(logging.Handler):
+ """Copies log records emitted inside a running Job into that Job's log.
+
+ The context variable is set on the task that runs the handler, so only records from the Job
+ (and tasks it spawns) are captured, not those of the other coroutines in this process."""
+
+ def emit(self, record: logging.LogRecord):
+ job = _current_job.get()
+ if job is None:
+ return
+ try:
+ job.log(self.format(record))
+ except Exception:
+ self.handleError(record)
+
+
+_log_handler = JobLogHandler()
+_log_handler.setFormatter(logging.Formatter('%(levelname)s:%(name)s:%(message)s'))
+if not any(isinstance(i, JobLogHandler) for i in logging.getLogger().handlers):
+ logging.getLogger().addHandler(_log_handler)
+
+
+def _json_safe(value):
+ try:
+ json.dumps(value)
+ return value
+ except (TypeError, ValueError):
+ return str(value)
+
+
+async def _run_job(job_id: str):
+ """Run one Job to completion (or failure/cancellation) in this process."""
+ record = _claim_job(job_id)
+ if not record:
+ # Cancelled while it was waiting in the queue.
+ return
+
+ name = record['name']
+ context = JobContext(job_id, name)
+ handler = JOB_HANDLERS.get(name)
+ if handler is None:
+ _update_job(job_id, status=FAILED, error=f'No handler is registered for {name!r}',
+ finished_at=now().isoformat())
+ return
+
+ token = _current_job.set(context)
+ try:
+ async def call():
+ result = handler(**record['kwargs'])
+ if inspect.isawaitable(result):
+ result = await result
+ return result
+
+ # The task copies the current context, so the handler (and any tasks it creates) see this Job.
+ task = asyncio.create_task(call())
+ while not task.done():
+ await asyncio.wait({task}, timeout=JOB_CANCEL_POLL_SECONDS)
+ if not task.done() and _cancel_requested(job_id):
+ task.cancel()
+ try:
+ await task
+ except BaseException: # noqa Includes CancelledError; the Job is what was cancelled.
+ pass
+ context.log('Cancelled')
+ context.flush()
+ _update_job(job_id, status=CANCELLED, finished_at=now().isoformat())
+ logger.warning(f'Job {job_id} was cancelled')
+ Events.send_user_notify(f'Cancelled: {record["description"]}')
+ return
+
+ try:
+ result = task.result()
+ except asyncio.CancelledError:
+ # The worker itself is shutting down; leave the record honest.
+ _update_job(job_id, status=CANCELLED, finished_at=now().isoformat())
+ raise
+ except Exception as e:
+ logger.error(f'Job {job_id} failed', exc_info=e)
+ context.flush()
+ _update_job(job_id, status=FAILED, error=str(e) or repr(e), finished_at=now().isoformat())
+ Events.send_user_notify(f'Failed: {record["description"]}')
+ return
+
+ context.flush()
+ _update_job(job_id, status=COMPLETE, result=_json_safe(result), progress=100.0,
+ finished_at=now().isoformat())
+ logger.info(f'Job {job_id} completed')
+ Events.send_user_notify(f'Completed: {record["description"]}')
+ finally:
+ context.flush()
+ _current_job.reset(token)
+
+
+async def process_job_queue() -> int:
+ """Run every queued Job, in order. Returns how many were run."""
+ count = 0
+ while True:
+ try:
+ job_id = _queue().get_nowait()
+ except queue.Empty:
+ return count
+ count += 1
+ await _run_job(job_id)
+
+
+@perpetual_signal(sleep=0.5)
+async def job_worker():
+ """The single consumer of the Job queue; runs in the perpetual-tasks owner process only."""
+ try:
+ job_id = _queue().get_nowait()
+ except queue.Empty:
+ return
+ await _run_job(job_id)
+
+
+async def wait_for_job(job_id: str, timeout: float = 300) -> dict:
+ """Wait for a Job to finish; returns its record.
+
+ In tests (no perpetual loop) the queue is drained by this waiter so the Job actually runs.
+
+ @raise TimeoutError: when the Job has not finished in time.
+ """
+ start = time.time()
+ while time.time() - start < timeout:
+ record = get_job(job_id)
+ if record['status'] in FINISHED_STATUSES:
+ return record
+ if PYTEST:
+ await process_job_queue()
+ await asyncio.sleep(0.01 if PYTEST else 0.5)
+ raise TimeoutError(f'Job {job_id} did not finish in time')
+
+
+jobs_bp = Blueprint('Jobs', url_prefix='/api/jobs')
+
+
+@jobs_bp.get('/')
+@openapi.description('List all Jobs, oldest first. Jobs are not saved; an API restart forgets them.')
+async def get_jobs_request(_: Request):
+ return json_response({'jobs': get_jobs()})
+
+
+@jobs_bp.get('/')
+@openapi.description('Get one Job, including its log tail.')
+@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
+async def get_job_request(_: Request, job_id: str):
+ return json_response({'job': get_job(job_id)})
+
+
+@jobs_bp.delete('/')
+@openapi.description('Cancel a pending or running Job. Allowed in WROL Mode: it only stops work.')
+@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
+async def cancel_job_request(_: Request, job_id: str):
+ return json_response({'job': cancel_job(job_id)})
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index 95d36512..5bcab2b9 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -35,6 +35,7 @@ from wrolpi.errors import WROLModeEnabled, InvalidConfig, ValidationError
from wrolpi.events import get_events, Events
from wrolpi.files import files_bp
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
+from wrolpi.jobs import jobs_bp
from wrolpi.log_levels import int_to_name
from wrolpi.speedtest import speedtest_bp
from wrolpi.tags import Tag
@@ -60,6 +61,7 @@ api_app.blueprint(docs_bp)
api_app.blueprint(files_bp)
api_app.blueprint(flasher_bp)
api_app.blueprint(inventory_bp)
+api_app.blueprint(jobs_bp)
api_app.blueprint(map_bp)
api_app.blueprint(speedtest_bp)
api_app.blueprint(videos_bp)
diff --git a/wrolpi/test/test_jobs.py b/wrolpi/test/test_jobs.py
new file mode 100644
index 00000000..a2e1a628
--- /dev/null
+++ b/wrolpi/test/test_jobs.py
@@ -0,0 +1,266 @@
+import asyncio
+import logging
+from unittest import mock
+from http import HTTPStatus
+
+import pytest
+
+from wrolpi import jobs
+from wrolpi.errors import InvalidJob, UnknownJob
+from wrolpi.jobs import register_job, enqueue_job, wait_for_job, get_job, get_current_job, cancel_job, \
+ process_job_queue, get_jobs
+
+test_logger = logging.getLogger('wrolpi.test.jobs')
+
+order = []
+
+
+@register_job('test_add')
+async def job_add(a: int, b: int):
+ order.append(('add', a, b))
+ test_logger.warning(f'adding {a} and {b}')
+ return a + b
+
+
+@register_job('test_sync')
+def job_sync(value: str):
+ return value.upper()
+
+
+@register_job('test_fail')
+async def job_fail(message: str):
+ raise RuntimeError(message)
+
+
+@register_job('test_progress')
+async def job_progress():
+ job = get_current_job()
+ job.log('first line')
+ job.set_progress(42)
+ return {'job_id': job.id}
+
+
+@register_job('test_slow')
+async def job_slow(seconds: float):
+ await asyncio.sleep(seconds)
+ return 'done'
+
+
+@register_job('test_command')
+async def job_command():
+ job = get_current_job()
+ result = await job.run_command(('sh', '-c', 'echo out1; echo out2; echo err1 >&2'))
+ return result.return_code
+
+
+@register_job('test_slow_command')
+async def job_slow_command():
+ job = get_current_job()
+ result = await job.run_command(('sleep', '30'))
+ return result.cancelled
+
+
+@pytest.mark.asyncio
+async def test_jobs_fifo(async_client):
+ """Jobs run in the order they were queued; sync and async handlers both work; results are stored."""
+ order.clear()
+ first = enqueue_job('test_add', a=1, b=2)
+ second = enqueue_job('test_add', a=3, b=4)
+ third = enqueue_job('test_sync', value='abc')
+
+ for job_id in (first, second, third):
+ assert get_job(job_id)['status'] == jobs.PENDING
+
+ assert (await wait_for_job(third))['status'] == jobs.COMPLETE
+ assert order == [('add', 1, 2), ('add', 3, 4)]
+ assert get_job(first)['result'] == 3
+ assert get_job(second)['result'] == 7
+ assert get_job(third)['result'] == 'ABC'
+ assert get_job(first)['progress'] == 100
+ assert get_job(first)['started_at'] and get_job(first)['finished_at']
+ assert [i['id'] for i in get_jobs()] == [first, second, third]
+
+
+@pytest.mark.asyncio
+async def test_jobs_capture_logging(async_client):
+ """`logging` emitted while a Job runs is captured into that Job's log, and only that Job's."""
+ a = enqueue_job('test_add', a=5, b=6)
+ b = enqueue_job('test_add', a=7, b=8)
+ await wait_for_job(b)
+ a_log = get_job(a)['log']
+ assert any('adding 5 and 6' in i for i in a_log), a_log
+ assert not any('adding 7 and 8' in i for i in a_log), a_log
+ assert any('adding 7 and 8' in i for i in get_job(b)['log'])
+
+
+@pytest.mark.asyncio
+async def test_jobs_failure(async_client):
+ """A handler exception fails the Job and records the error; the queue keeps going."""
+ failed = enqueue_job('test_fail', message='boom')
+ after = enqueue_job('test_sync', value='after')
+ record = await wait_for_job(after)
+ assert record['status'] == jobs.COMPLETE
+ failed = get_job(failed)
+ assert failed['status'] == jobs.FAILED
+ assert failed['error'] == 'boom'
+ assert any('boom' in i for i in failed['log'])
+
+
+@pytest.mark.asyncio
+async def test_jobs_context(async_client):
+ """The handler can log, report progress, and learn its own id through `get_current_job`."""
+ job_id = enqueue_job('test_progress', description='Progress test')
+ record = await wait_for_job(job_id)
+ assert record['result'] == {'job_id': job_id}
+ assert record['description'] == 'Progress test'
+ assert 'first line' in record['log']
+ assert record['progress'] == 100 # Completion sets 100.
+ assert get_current_job() is None, 'No Job is running outside of the worker'
+
+
+@pytest.mark.asyncio
+async def test_jobs_run_command_captured(async_client):
+ """stdout and stderr of a Job subprocess land in the Job log."""
+ job_id = enqueue_job('test_command')
+ record = await wait_for_job(job_id)
+ assert record['status'] == jobs.COMPLETE, record
+ assert record['result'] == 0
+ assert 'out1' in record['log'] and 'out2' in record['log'] and 'err1' in record['log']
+
+
+@pytest.mark.asyncio
+async def test_jobs_cancel_pending(async_client):
+ """A pending Job is cancelled immediately and never runs."""
+ order.clear()
+ job_id = enqueue_job('test_add', a=1, b=1)
+ record = cancel_job(job_id)
+ assert record['status'] == jobs.CANCELLED
+ assert record['finished_at']
+ assert await process_job_queue() == 1, 'The queued id is consumed'
+ assert order == [], 'The cancelled Job did not run'
+ assert get_job(job_id)['status'] == jobs.CANCELLED
+
+
+@pytest.mark.asyncio
+async def test_jobs_cancel_running(async_client):
+ """A running Job is cancelled the next time the worker polls; its subprocess is killed."""
+ job_id = enqueue_job('test_slow_command')
+ task = asyncio.create_task(process_job_queue())
+ # Wait for the Job to actually be running.
+ for _ in range(200):
+ if get_job(job_id)['status'] == jobs.RUNNING:
+ break
+ await asyncio.sleep(0.01)
+ else:
+ raise AssertionError('Job never started')
+
+ record = cancel_job(job_id)
+ assert record['cancel_requested'] is True
+ await asyncio.wait_for(task, timeout=5)
+ record = get_job(job_id)
+ assert record['status'] == jobs.CANCELLED
+ assert 'Cancelled' in record['log']
+
+
+@pytest.mark.asyncio
+async def test_jobs_validation(async_client):
+ with pytest.raises(InvalidJob):
+ enqueue_job('does_not_exist')
+ with pytest.raises(InvalidJob):
+ enqueue_job('test_add', a=object(), b=1)
+ with pytest.raises(UnknownJob):
+ get_job('nope')
+ with pytest.raises(UnknownJob):
+ cancel_job('nope')
+
+
+@pytest.mark.asyncio
+async def test_jobs_prune(async_client):
+ """Only the newest finished Jobs are kept."""
+ old_limit = jobs.JOB_HISTORY_LIMIT
+ jobs.JOB_HISTORY_LIMIT = 2
+ try:
+ ids = [enqueue_job('test_sync', value=str(i)) for i in range(3)]
+ await wait_for_job(ids[-1])
+ # A new enqueue triggers the prune.
+ newest = enqueue_job('test_sync', value='x')
+ remaining = {i['id'] for i in get_jobs()}
+ assert ids[0] not in remaining
+ assert ids[1] in remaining and ids[2] in remaining and newest in remaining
+ finally:
+ jobs.JOB_HISTORY_LIMIT = old_limit
+
+
+@pytest.mark.asyncio
+async def test_jobs_api(async_client):
+ """Jobs can be listed, fetched, and cancelled over the API."""
+ job_id = enqueue_job('test_slow', description='Sleepy', seconds=30)
+
+ request, response = await async_client.get('/api/jobs')
+ assert response.status == HTTPStatus.OK
+ assert [i['id'] for i in response.json['jobs']] == [job_id]
+
+ request, response = await async_client.get(f'/api/jobs/{job_id}')
+ assert response.status == HTTPStatus.OK
+ assert response.json['job']['description'] == 'Sleepy'
+ assert response.json['job']['status'] == jobs.PENDING
+
+ request, response = await async_client.delete(f'/api/jobs/{job_id}')
+ assert response.status == HTTPStatus.OK
+ assert response.json['job']['status'] == jobs.CANCELLED
+
+ request, response = await async_client.get('/api/jobs/nope')
+ assert response.status == HTTPStatus.NOT_FOUND
+
+
+@pytest.mark.asyncio
+async def test_jobs_claim_is_atomic(async_client):
+ """The worker claims a Job (PENDING -> RUNNING) in one step under the lock, so a cancel that
+ lands first wins and the Job never runs; a cancel that lands after only requests a stop."""
+ from wrolpi.jobs import _claim_job
+
+ cancelled_first = enqueue_job('test_sync', value='a')
+ cancel_job(cancelled_first)
+ assert _claim_job(cancelled_first) is None, 'A cancelled Job cannot be claimed'
+ assert get_job(cancelled_first)['status'] == jobs.CANCELLED, 'The claim must not promote it to RUNNING'
+
+ claimed = enqueue_job('test_sync', value='b')
+ record = _claim_job(claimed)
+ assert record['status'] == jobs.RUNNING and record['pid'] and record['started_at']
+ assert get_job(claimed)['status'] == jobs.RUNNING
+ assert _claim_job(claimed) is None, 'A Job is claimed once'
+ # Cancelling a claimed Job only asks it to stop.
+ assert cancel_job(claimed)['status'] == jobs.RUNNING
+ assert get_job(claimed)['cancel_requested'] is True
+
+
+@pytest.mark.asyncio
+async def test_jobs_pending_limit(async_client):
+ """The queue refuses new Jobs past a pending cap, so a runaway client cannot grow shared memory."""
+ with mock.patch.object(jobs, 'JOB_PENDING_LIMIT', 2):
+ first = enqueue_job('test_slow', seconds=30)
+ second = enqueue_job('test_slow', seconds=30)
+ with pytest.raises(InvalidJob, match='pending'):
+ enqueue_job('test_slow', seconds=30)
+ # A cancelled Job no longer counts.
+ cancel_job(first)
+ third = enqueue_job('test_slow', seconds=30)
+ for job_id in (second, third):
+ cancel_job(job_id)
+ assert await process_job_queue() == 3, 'Cancelled ids are drained as no-ops'
+ assert [i['status'] for i in get_jobs()] == [jobs.CANCELLED] * 3
+
+
+@pytest.mark.asyncio
+async def test_jobs_run_command_cancel_semantics(async_client):
+ """A Job's subprocess runs in its own session (the whole group dies with it), and a killed
+ command raises rather than returning a result a handler could mistake for a normal exit."""
+ from wrolpi.cmd import CommandResult
+ from wrolpi.jobs import JobContext
+
+ context = JobContext('test-1', 'test')
+ killed = CommandResult(return_code=-9, cancelled=True, stdout=b'', stderr=b'', elapsed=1)
+ with mock.patch('wrolpi.cmd.run_command', return_value=killed) as mock_run:
+ with pytest.raises(asyncio.CancelledError):
+ await context.run_command(('sleep', '30'))
+ assert mock_run.call_args.kwargs['start_new_session'] is True