Adding links next to video captions to allow scrubbing of video by the captions, adding filter to make finding quick.

This commit is contained in:
Roland 2026-03-21 16:50:27 -06:00
parent 2bae99c6ef
commit 282b0f63e8
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
10 changed files with 224 additions and 46 deletions

View file

@ -177,7 +177,7 @@ describe('Domain Editing Workflow', () => {
}
}).as('invalidUpdate');
cy.contains('label', 'Directory').parent().find('input').clear().type('/invalid/absolute/path');
cy.contains('label', 'Directory').parent().find('input').should('not.be.disabled').clear().type('/invalid/absolute/path');
cy.contains('button', 'Save').click();
cy.wait('@invalidUpdate');

View file

@ -1,4 +1,4 @@
import React, {useContext} from 'react';
import React, {useContext, useState} from 'react';
import {useHotkeys} from 'react-hotkeys-hook';
import {deleteVideos, downloadVideoMetadata, tagFileGroup, untagFileGroup} from "../api";
import {Link, useNavigate, useParams} from "react-router";
@ -19,12 +19,12 @@ import {
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
import {VideoPlaceholder} from "./Placeholder";
import {useChannel, useVideoExtras} from "../hooks/customHooks";
import {useChannel, useVideoCaptions, useVideoExtras} from "../hooks/customHooks";
import {ThemeContext} from "../contexts/contexts";
import {Button, darkTheme, Header, Icon, Segment, Tab} from "./Theme";
import {Button, darkTheme, Header, Icon, Loader, Segment, Tab} from "./Theme";
import {VideoCard} from "./Videos";
import {TagsSelector} from "../Tags";
import {Comment, CommentGroup, Label} from "semantic-ui-react";
import {Comment, CommentGroup, Input, Label} from "semantic-ui-react";
const MEDIA_PATH = '/media';
@ -198,7 +198,8 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
// Get the Video's channel, fallback to the URL's channel id.
const {channelId} = useParams();
const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId);
const {comments, captions} = useVideoExtras(videoFile?.video?.id);
const {comments} = useVideoExtras(videoFile?.id);
const {captions, captionsLoading, fetchCaptions} = useVideoCaptions(videoFile?.id);
const mediaTitle = videoFile ? (videoFile.title || videoFile.stem || videoFile.name) : null;
const mediaArtworkUrl = video && video.poster_path ? `${MEDIA_PATH}/${encodeMediaPath(video.poster_path)}` : null;
@ -290,9 +291,12 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
};
const captionsPane = {
menuItem: 'Captions', render: () => <Tab.Pane>
<pre>{captions || 'No captions available.'}</pre>
</Tab.Pane>
menuItem: 'Captions', render: () => <CaptionsPane
captions={captions}
captionsLoading={captionsLoading}
fetchCaptions={fetchCaptions}
setVideoTime={setVideoTime}
/>
};
const getFile = (suffix) => videoFile.files.find(i => i.path.toLowerCase().endsWith(suffix));
@ -479,6 +483,71 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
</>
}
function formatCaptionTimestamp(totalSeconds) {
totalSeconds = Math.floor(totalSeconds);
const hours = Math.floor(totalSeconds / 3600);
const minutes = Math.floor((totalSeconds % 3600) / 60);
const seconds = totalSeconds % 60;
const pad2 = (n) => String(n).padStart(2, '0');
if (hours > 0) {
return `${hours}:${pad2(minutes)}:${pad2(seconds)}`;
}
return `${minutes}:${pad2(seconds)}`;
}
function CaptionsPane({captions, captionsLoading, fetchCaptions, setVideoTime}) {
const [filter, setFilter] = useState('');
// Trigger fetch on first render of this pane.
React.useEffect(() => {
fetchCaptions();
}, []);
if (captionsLoading) {
return <Tab.Pane><Loader active/></Tab.Pane>;
}
if (!captions || captions.length === 0) {
return <Tab.Pane>No captions available.</Tab.Pane>;
}
let filtered = captions;
if (filter.trim()) {
const words = filter.toLowerCase().split(/\s+/).filter(Boolean);
filtered = captions.filter(chunk => {
const text = chunk.text.toLowerCase();
return words.every(word => text.includes(word));
});
}
return <Tab.Pane>
<Input
fluid
icon='filter'
placeholder='Filter captions...'
value={filter}
onChange={(e, {value}) => setFilter(value)}
style={{marginBottom: '1em'}}
/>
<div style={{maxHeight: '400px', overflowY: 'auto'}}>
{filtered.map((chunk, idx) => (
<div key={idx} style={{display: 'flex', gap: '1em', marginBottom: '0.4em'}}>
<a href='#'
onClick={(e) => {
e.preventDefault();
setVideoTime(chunk.start_seconds);
}}
style={{whiteSpace: 'nowrap', fontFamily: 'monospace'}}
>
{formatCaptionTimestamp(chunk.start_seconds)}
</a>
<span>{chunk.text}</span>
</div>
))}
</div>
</Tab.Pane>;
}
export default VideoPage;
const chapterRegex = new RegExp('^(\\(?(?:((\\d?\\d):)?(?:(\\d?\\d):(\\d\\d)))\\)?)\.?\\s+(.*)$', 'i');

View file

@ -692,7 +692,6 @@ export const useVideo = (fileGroupId) => {
export const useVideoExtras = (fileGroupId) => {
// Fetches extra data to display on a Video's page.
const [comments, setComments] = useState(null);
const [captions, setCaptions] = useState(null);
const fetchComments = async () => {
try {
@ -703,26 +702,43 @@ export const useVideoExtras = (fileGroupId) => {
}
}
useEffect(() => {
if (fileGroupId) {
fetchComments();
} else {
setComments(null);
}
}, [fileGroupId]);
return {comments}
}
export const useVideoCaptions = (fileGroupId) => {
const [captions, setCaptions] = useState(null);
const [captionsLoading, setCaptionsLoading] = useState(false);
const captionsFetched = useRef(false);
const fetchCaptions = async () => {
if (captionsFetched.current || !fileGroupId) return;
setCaptionsLoading(true);
try {
const result = await getVideoCaptions(fileGroupId);
setCaptions(result.captions);
} catch (e) {
console.error(e);
} finally {
setCaptionsLoading(false);
captionsFetched.current = true;
}
}
useEffect(() => {
if (fileGroupId) {
fetchComments();
fetchCaptions();
} else {
setComments(null);
setCaptions(null);
}
setCaptions(null);
setCaptionsLoading(false);
captionsFetched.current = false;
}, [fileGroupId]);
return {comments, captions}
return {captions, captionsLoading, fetchCaptions}
}
export const useChannel = (channel_id) => {

View file

@ -8,7 +8,7 @@ from sqlalchemy.orm import relationship, Session, deferred
from sqlalchemy.orm.collections import InstrumentedList
from modules.videos.errors import UnknownVideo, UnknownChannel
from wrolpi.captions import read_captions
from wrolpi.captions import read_captions, read_captions_with_timestamps
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
background_task
from wrolpi.db import get_db_curs, get_db_session
@ -336,6 +336,20 @@ class Video(ModelHelper, Base):
return caption_text
def get_caption_chunks(self) -> Optional[List[dict]]:
"""Search the FileGroup's files for a caption file. Return captions with timestamps."""
caption_paths = self.caption_paths
caption_chunks = None
if english_vtt := [i for i in caption_paths if i.name.endswith('.en.vtt')]:
caption_chunks = read_captions_with_timestamps(english_vtt[0])
elif vtt := [i for i in caption_paths if i.name.endswith('.vtt')]:
caption_chunks = read_captions_with_timestamps(vtt[0])
elif english_srt := [i for i in caption_paths if i.name.endswith('.en.srt')]:
caption_chunks = read_captions_with_timestamps(english_srt[0])
elif srt := [i for i in caption_paths if i.name.endswith('.srt')]:
caption_chunks = read_captions_with_timestamps(srt[0])
return caption_chunks
@staticmethod
def get_by_path(session: Session, path) -> Optional['Video']:
video = session.query(Video) \

View file

@ -164,7 +164,7 @@ class VideoCommentsResponse:
@dataclass
class VideoCaptionsResponse:
captions: str
captions: list
@dataclass

View file

@ -44,7 +44,7 @@ def video_get_comments(_: Request, file_group_id: int):
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
def video_get_captions(_: Request, file_group_id: int):
video = lib.get_video(file_group_id)
return json_response({'captions': video.file_group.d_text})
return json_response({'captions': video.get_caption_chunks()})
@video_bp.post('/search')

View file

@ -197,7 +197,11 @@ async def test_api_video_extras(async_client, simple_channel, video_factory):
request, response = await async_client.get(f'/api/videos/{video.file_group_id}/captions')
assert response.status_code == HTTPStatus.OK
assert response.json.get('captions')
captions = response.json.get('captions')
assert isinstance(captions, list)
assert len(captions) > 0
assert 'start_seconds' in captions[0]
assert 'text' in captions[0]
request, response = await async_client.get(f'/api/videos/{video.file_group_id}/comments')
assert response.status_code == HTTPStatus.OK

View file

@ -3,7 +3,7 @@ import pathlib
import subprocess
import tempfile
from pathlib import Path
from typing import Generator, Union
from typing import Generator, List, Optional, Union
import srt
import webvtt
@ -11,23 +11,66 @@ import webvtt
from wrolpi.cmd import FFMPEG_BIN
from wrolpi.common import logger
__all__ = ['read_captions', 'extract_captions']
__all__ = ['read_captions', 'read_captions_with_timestamps', 'extract_captions']
def _parse_vtt_timestamp(timestamp: str) -> float:
"""Parse a VTT timestamp like '00:01:05.269' to seconds as a float."""
parts = timestamp.strip().split(':')
if len(parts) == 3:
hours, minutes, seconds = parts
elif len(parts) == 2:
hours = '0'
minutes, seconds = parts
else:
return 0.0
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
def _parse_caption_file(caption_path: Union[str, Path]) -> List[dict]:
"""Parse a VTT or SRT file into raw caption chunks with timestamps.
Returns a list of dicts with 'start_seconds' and 'text' keys."""
raw_chunks = []
caption_path = str(caption_path)
if caption_path.endswith('vtt'):
for caption in webvtt.read(caption_path):
text = str(caption.text).strip()
if text:
start = _parse_vtt_timestamp(caption.start)
raw_chunks.append({'start_seconds': start, 'text': text})
else:
with open(caption_path, 'rt') as fh:
contents = fh.read()
for subtitle in srt.parse(contents):
text = subtitle.content.strip()
if text:
start = subtitle.start.total_seconds()
raw_chunks.append({'start_seconds': start, 'text': text})
return raw_chunks
def _deduplicate_caption_chunks(raw_chunks: List[dict]) -> List[dict]:
"""Deduplicate caption chunks by extracting only new lines from each chunk.
YouTube auto-captions produce overlapping chunks where each chunk repeats lines from the previous one
plus new content. This extracts only the new lines and assigns the chunk's start timestamp."""
chunks = []
last_lines = []
for chunk in raw_chunks:
lines = [line for line in chunk['text'].split('\n') if line.strip()]
new_lines = [line for line in lines if line not in last_lines]
if new_lines:
text = '\n'.join(new_lines)
if not chunks or text != chunks[-1]['text']:
chunks.append({'start_seconds': chunk['start_seconds'], 'text': text})
last_lines = lines
return chunks
def get_caption_text(caption_path: Union[str, Path]) -> Generator:
"""Return all text from each caption of a caption file."""
if str(caption_path).endswith('vtt'):
# VTT
for caption in webvtt.read(caption_path):
text = str(caption.text).strip()
yield text
else:
# Finally, try SRT
with open(caption_path, 'rt') as fh:
contents = fh.read()
for subtitle in srt.parse(contents):
if any(subtitle.content):
yield subtitle.content
for chunk in _parse_caption_file(caption_path):
yield chunk['text']
def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator:
@ -41,25 +84,37 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator:
def read_captions(caption_path: Path):
"""Parse video captions from the video's captions file."""
"""Parse video captions from the video's captions file. Returns deduplicated caption text as a string."""
try:
lines = get_unique_caption_lines(str(caption_path))
block = '\n'.join(lines)
return block
except UnicodeDecodeError as e:
# Failed to decode the caption file
# TODO handle this error
except UnicodeDecodeError:
pass
except webvtt.errors.MalformedFileError as e:
# File format is broken
except webvtt.errors.MalformedFileError:
pass
except webvtt.errors.MalformedCaptionError as e:
# Captions form is broken somehow
except webvtt.errors.MalformedCaptionError:
pass
logger.debug(f'Failed to parse caption file {caption_path}')
def read_captions_with_timestamps(caption_path: Path) -> Optional[List[dict]]:
"""Parse video captions preserving timestamps. Returns a list of dicts with 'start_seconds' and 'text' keys.
Overlapping and duplicate lines are deduplicated."""
try:
raw_chunks = _parse_caption_file(caption_path)
chunks = _deduplicate_caption_chunks(raw_chunks)
return chunks if chunks else None
except UnicodeDecodeError:
pass
except webvtt.errors.MalformedFileError:
pass
except webvtt.errors.MalformedCaptionError:
pass
logger.debug(f'Failed to parse caption file {caption_path}')
return None
def extract_captions(path: pathlib.Path) -> str | None:
"""Extract captions that are embedded in a video file."""
with tempfile.TemporaryDirectory() as directory:

View file

@ -1,3 +1,5 @@
import pytest
from wrolpi import captions
@ -47,3 +49,21 @@ def test_get_unique_caption_lines(vtt_file1, srt_file3):
'of background information we come from a',
'company called le code from finland and']
assert list(captions.get_unique_caption_lines(srt_file3)) == expected
EXPECTED_CAPTION_CHUNKS = [
{'start_seconds': 5.269, 'text': 'okay welcome to this session this is'},
{'start_seconds': 5.279, 'text': 'called the kinetic bunny need to meet'},
{'start_seconds': 7.77, 'text': 'the virtual bonnie emilio stop this is'},
{'start_seconds': 11.46, 'text': 'jana car who just to give a little bit'},
{'start_seconds': 15.42, 'text': 'of background information we come from a'},
{'start_seconds': 18.869, 'text': 'company called le code from finland and'},
]
@pytest.mark.parametrize('fixture_name', ['vtt_file1', 'srt_file3'])
def test_read_captions_with_timestamps(fixture_name, request):
"""read_captions_with_timestamps returns timestamped caption chunks with overlapping lines deduplicated."""
caption_file = request.getfixturevalue(fixture_name)
chunks = captions.read_captions_with_timestamps(caption_file)
assert chunks == EXPECTED_CAPTION_CHUNKS

View file

@ -1 +1 @@
0.18.1-beta
0.19.0-beta