mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Adding links next to video captions to allow scrubbing of video by the captions, adding filter to make finding quick.
This commit is contained in:
parent
2bae99c6ef
commit
282b0f63e8
10 changed files with 224 additions and 46 deletions
|
|
@ -177,7 +177,7 @@ describe('Domain Editing Workflow', () => {
|
|||
}
|
||||
}).as('invalidUpdate');
|
||||
|
||||
cy.contains('label', 'Directory').parent().find('input').clear().type('/invalid/absolute/path');
|
||||
cy.contains('label', 'Directory').parent().find('input').should('not.be.disabled').clear().type('/invalid/absolute/path');
|
||||
cy.contains('button', 'Save').click();
|
||||
|
||||
cy.wait('@invalidUpdate');
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import React, {useContext} from 'react';
|
||||
import React, {useContext, useState} from 'react';
|
||||
import {useHotkeys} from 'react-hotkeys-hook';
|
||||
import {deleteVideos, downloadVideoMetadata, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {Link, useNavigate, useParams} from "react-router";
|
||||
|
|
@ -19,12 +19,12 @@ import {
|
|||
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
|
||||
import Container from "semantic-ui-react/dist/commonjs/elements/Container";
|
||||
import {VideoPlaceholder} from "./Placeholder";
|
||||
import {useChannel, useVideoExtras} from "../hooks/customHooks";
|
||||
import {useChannel, useVideoCaptions, useVideoExtras} from "../hooks/customHooks";
|
||||
import {ThemeContext} from "../contexts/contexts";
|
||||
import {Button, darkTheme, Header, Icon, Segment, Tab} from "./Theme";
|
||||
import {Button, darkTheme, Header, Icon, Loader, Segment, Tab} from "./Theme";
|
||||
import {VideoCard} from "./Videos";
|
||||
import {TagsSelector} from "../Tags";
|
||||
import {Comment, CommentGroup, Label} from "semantic-ui-react";
|
||||
import {Comment, CommentGroup, Input, Label} from "semantic-ui-react";
|
||||
|
||||
const MEDIA_PATH = '/media';
|
||||
|
||||
|
|
@ -198,7 +198,8 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
// Get the Video's channel, fallback to the URL's channel id.
|
||||
const {channelId} = useParams();
|
||||
const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId);
|
||||
const {comments, captions} = useVideoExtras(videoFile?.video?.id);
|
||||
const {comments} = useVideoExtras(videoFile?.id);
|
||||
const {captions, captionsLoading, fetchCaptions} = useVideoCaptions(videoFile?.id);
|
||||
|
||||
const mediaTitle = videoFile ? (videoFile.title || videoFile.stem || videoFile.name) : null;
|
||||
const mediaArtworkUrl = video && video.poster_path ? `${MEDIA_PATH}/${encodeMediaPath(video.poster_path)}` : null;
|
||||
|
|
@ -290,9 +291,12 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
};
|
||||
|
||||
const captionsPane = {
|
||||
menuItem: 'Captions', render: () => <Tab.Pane>
|
||||
<pre>{captions || 'No captions available.'}</pre>
|
||||
</Tab.Pane>
|
||||
menuItem: 'Captions', render: () => <CaptionsPane
|
||||
captions={captions}
|
||||
captionsLoading={captionsLoading}
|
||||
fetchCaptions={fetchCaptions}
|
||||
setVideoTime={setVideoTime}
|
||||
/>
|
||||
};
|
||||
|
||||
const getFile = (suffix) => videoFile.files.find(i => i.path.toLowerCase().endsWith(suffix));
|
||||
|
|
@ -479,6 +483,71 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) {
|
|||
</>
|
||||
}
|
||||
|
||||
function formatCaptionTimestamp(totalSeconds) {
|
||||
totalSeconds = Math.floor(totalSeconds);
|
||||
const hours = Math.floor(totalSeconds / 3600);
|
||||
const minutes = Math.floor((totalSeconds % 3600) / 60);
|
||||
const seconds = totalSeconds % 60;
|
||||
const pad2 = (n) => String(n).padStart(2, '0');
|
||||
if (hours > 0) {
|
||||
return `${hours}:${pad2(minutes)}:${pad2(seconds)}`;
|
||||
}
|
||||
return `${minutes}:${pad2(seconds)}`;
|
||||
}
|
||||
|
||||
function CaptionsPane({captions, captionsLoading, fetchCaptions, setVideoTime}) {
|
||||
const [filter, setFilter] = useState('');
|
||||
|
||||
// Trigger fetch on first render of this pane.
|
||||
React.useEffect(() => {
|
||||
fetchCaptions();
|
||||
}, []);
|
||||
|
||||
if (captionsLoading) {
|
||||
return <Tab.Pane><Loader active/></Tab.Pane>;
|
||||
}
|
||||
|
||||
if (!captions || captions.length === 0) {
|
||||
return <Tab.Pane>No captions available.</Tab.Pane>;
|
||||
}
|
||||
|
||||
let filtered = captions;
|
||||
if (filter.trim()) {
|
||||
const words = filter.toLowerCase().split(/\s+/).filter(Boolean);
|
||||
filtered = captions.filter(chunk => {
|
||||
const text = chunk.text.toLowerCase();
|
||||
return words.every(word => text.includes(word));
|
||||
});
|
||||
}
|
||||
|
||||
return <Tab.Pane>
|
||||
<Input
|
||||
fluid
|
||||
icon='filter'
|
||||
placeholder='Filter captions...'
|
||||
value={filter}
|
||||
onChange={(e, {value}) => setFilter(value)}
|
||||
style={{marginBottom: '1em'}}
|
||||
/>
|
||||
<div style={{maxHeight: '400px', overflowY: 'auto'}}>
|
||||
{filtered.map((chunk, idx) => (
|
||||
<div key={idx} style={{display: 'flex', gap: '1em', marginBottom: '0.4em'}}>
|
||||
<a href='#'
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
setVideoTime(chunk.start_seconds);
|
||||
}}
|
||||
style={{whiteSpace: 'nowrap', fontFamily: 'monospace'}}
|
||||
>
|
||||
{formatCaptionTimestamp(chunk.start_seconds)}
|
||||
</a>
|
||||
<span>{chunk.text}</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
</Tab.Pane>;
|
||||
}
|
||||
|
||||
export default VideoPage;
|
||||
|
||||
const chapterRegex = new RegExp('^(\\(?(?:((\\d?\\d):)?(?:(\\d?\\d):(\\d\\d)))\\)?)\.?\\s+(.*)$', 'i');
|
||||
|
|
|
|||
|
|
@ -692,7 +692,6 @@ export const useVideo = (fileGroupId) => {
|
|||
export const useVideoExtras = (fileGroupId) => {
|
||||
// Fetches extra data to display on a Video's page.
|
||||
const [comments, setComments] = useState(null);
|
||||
const [captions, setCaptions] = useState(null);
|
||||
|
||||
const fetchComments = async () => {
|
||||
try {
|
||||
|
|
@ -703,26 +702,43 @@ export const useVideoExtras = (fileGroupId) => {
|
|||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (fileGroupId) {
|
||||
fetchComments();
|
||||
} else {
|
||||
setComments(null);
|
||||
}
|
||||
}, [fileGroupId]);
|
||||
|
||||
return {comments}
|
||||
}
|
||||
|
||||
export const useVideoCaptions = (fileGroupId) => {
|
||||
const [captions, setCaptions] = useState(null);
|
||||
const [captionsLoading, setCaptionsLoading] = useState(false);
|
||||
const captionsFetched = useRef(false);
|
||||
|
||||
const fetchCaptions = async () => {
|
||||
if (captionsFetched.current || !fileGroupId) return;
|
||||
setCaptionsLoading(true);
|
||||
try {
|
||||
const result = await getVideoCaptions(fileGroupId);
|
||||
setCaptions(result.captions);
|
||||
} catch (e) {
|
||||
console.error(e);
|
||||
} finally {
|
||||
setCaptionsLoading(false);
|
||||
captionsFetched.current = true;
|
||||
}
|
||||
}
|
||||
|
||||
useEffect(() => {
|
||||
if (fileGroupId) {
|
||||
fetchComments();
|
||||
fetchCaptions();
|
||||
} else {
|
||||
setComments(null);
|
||||
setCaptions(null);
|
||||
}
|
||||
setCaptions(null);
|
||||
setCaptionsLoading(false);
|
||||
captionsFetched.current = false;
|
||||
}, [fileGroupId]);
|
||||
|
||||
return {comments, captions}
|
||||
return {captions, captionsLoading, fetchCaptions}
|
||||
}
|
||||
|
||||
export const useChannel = (channel_id) => {
|
||||
|
|
|
|||
|
|
@ -8,7 +8,7 @@ from sqlalchemy.orm import relationship, Session, deferred
|
|||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from modules.videos.errors import UnknownVideo, UnknownChannel
|
||||
from wrolpi.captions import read_captions
|
||||
from wrolpi.captions import read_captions, read_captions_with_timestamps
|
||||
from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \
|
||||
background_task
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
|
|
@ -336,6 +336,20 @@ class Video(ModelHelper, Base):
|
|||
|
||||
return caption_text
|
||||
|
||||
def get_caption_chunks(self) -> Optional[List[dict]]:
|
||||
"""Search the FileGroup's files for a caption file. Return captions with timestamps."""
|
||||
caption_paths = self.caption_paths
|
||||
caption_chunks = None
|
||||
if english_vtt := [i for i in caption_paths if i.name.endswith('.en.vtt')]:
|
||||
caption_chunks = read_captions_with_timestamps(english_vtt[0])
|
||||
elif vtt := [i for i in caption_paths if i.name.endswith('.vtt')]:
|
||||
caption_chunks = read_captions_with_timestamps(vtt[0])
|
||||
elif english_srt := [i for i in caption_paths if i.name.endswith('.en.srt')]:
|
||||
caption_chunks = read_captions_with_timestamps(english_srt[0])
|
||||
elif srt := [i for i in caption_paths if i.name.endswith('.srt')]:
|
||||
caption_chunks = read_captions_with_timestamps(srt[0])
|
||||
return caption_chunks
|
||||
|
||||
@staticmethod
|
||||
def get_by_path(session: Session, path) -> Optional['Video']:
|
||||
video = session.query(Video) \
|
||||
|
|
|
|||
|
|
@ -164,7 +164,7 @@ class VideoCommentsResponse:
|
|||
|
||||
@dataclass
|
||||
class VideoCaptionsResponse:
|
||||
captions: str
|
||||
captions: list
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -44,7 +44,7 @@ def video_get_comments(_: Request, file_group_id: int):
|
|||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
def video_get_captions(_: Request, file_group_id: int):
|
||||
video = lib.get_video(file_group_id)
|
||||
return json_response({'captions': video.file_group.d_text})
|
||||
return json_response({'captions': video.get_caption_chunks()})
|
||||
|
||||
|
||||
@video_bp.post('/search')
|
||||
|
|
|
|||
|
|
@ -197,7 +197,11 @@ async def test_api_video_extras(async_client, simple_channel, video_factory):
|
|||
|
||||
request, response = await async_client.get(f'/api/videos/{video.file_group_id}/captions')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json.get('captions')
|
||||
captions = response.json.get('captions')
|
||||
assert isinstance(captions, list)
|
||||
assert len(captions) > 0
|
||||
assert 'start_seconds' in captions[0]
|
||||
assert 'text' in captions[0]
|
||||
|
||||
request, response = await async_client.get(f'/api/videos/{video.file_group_id}/comments')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
|
|
|||
|
|
@ -3,7 +3,7 @@ import pathlib
|
|||
import subprocess
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
from typing import Generator, Union
|
||||
from typing import Generator, List, Optional, Union
|
||||
|
||||
import srt
|
||||
import webvtt
|
||||
|
|
@ -11,23 +11,66 @@ import webvtt
|
|||
from wrolpi.cmd import FFMPEG_BIN
|
||||
from wrolpi.common import logger
|
||||
|
||||
__all__ = ['read_captions', 'extract_captions']
|
||||
__all__ = ['read_captions', 'read_captions_with_timestamps', 'extract_captions']
|
||||
|
||||
|
||||
def _parse_vtt_timestamp(timestamp: str) -> float:
|
||||
"""Parse a VTT timestamp like '00:01:05.269' to seconds as a float."""
|
||||
parts = timestamp.strip().split(':')
|
||||
if len(parts) == 3:
|
||||
hours, minutes, seconds = parts
|
||||
elif len(parts) == 2:
|
||||
hours = '0'
|
||||
minutes, seconds = parts
|
||||
else:
|
||||
return 0.0
|
||||
return int(hours) * 3600 + int(minutes) * 60 + float(seconds)
|
||||
|
||||
|
||||
def _parse_caption_file(caption_path: Union[str, Path]) -> List[dict]:
|
||||
"""Parse a VTT or SRT file into raw caption chunks with timestamps.
|
||||
Returns a list of dicts with 'start_seconds' and 'text' keys."""
|
||||
raw_chunks = []
|
||||
caption_path = str(caption_path)
|
||||
if caption_path.endswith('vtt'):
|
||||
for caption in webvtt.read(caption_path):
|
||||
text = str(caption.text).strip()
|
||||
if text:
|
||||
start = _parse_vtt_timestamp(caption.start)
|
||||
raw_chunks.append({'start_seconds': start, 'text': text})
|
||||
else:
|
||||
with open(caption_path, 'rt') as fh:
|
||||
contents = fh.read()
|
||||
for subtitle in srt.parse(contents):
|
||||
text = subtitle.content.strip()
|
||||
if text:
|
||||
start = subtitle.start.total_seconds()
|
||||
raw_chunks.append({'start_seconds': start, 'text': text})
|
||||
return raw_chunks
|
||||
|
||||
|
||||
def _deduplicate_caption_chunks(raw_chunks: List[dict]) -> List[dict]:
|
||||
"""Deduplicate caption chunks by extracting only new lines from each chunk.
|
||||
|
||||
YouTube auto-captions produce overlapping chunks where each chunk repeats lines from the previous one
|
||||
plus new content. This extracts only the new lines and assigns the chunk's start timestamp."""
|
||||
chunks = []
|
||||
last_lines = []
|
||||
for chunk in raw_chunks:
|
||||
lines = [line for line in chunk['text'].split('\n') if line.strip()]
|
||||
new_lines = [line for line in lines if line not in last_lines]
|
||||
if new_lines:
|
||||
text = '\n'.join(new_lines)
|
||||
if not chunks or text != chunks[-1]['text']:
|
||||
chunks.append({'start_seconds': chunk['start_seconds'], 'text': text})
|
||||
last_lines = lines
|
||||
return chunks
|
||||
|
||||
|
||||
def get_caption_text(caption_path: Union[str, Path]) -> Generator:
|
||||
"""Return all text from each caption of a caption file."""
|
||||
if str(caption_path).endswith('vtt'):
|
||||
# VTT
|
||||
for caption in webvtt.read(caption_path):
|
||||
text = str(caption.text).strip()
|
||||
yield text
|
||||
else:
|
||||
# Finally, try SRT
|
||||
with open(caption_path, 'rt') as fh:
|
||||
contents = fh.read()
|
||||
for subtitle in srt.parse(contents):
|
||||
if any(subtitle.content):
|
||||
yield subtitle.content
|
||||
for chunk in _parse_caption_file(caption_path):
|
||||
yield chunk['text']
|
||||
|
||||
|
||||
def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator:
|
||||
|
|
@ -41,25 +84,37 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator:
|
|||
|
||||
|
||||
def read_captions(caption_path: Path):
|
||||
"""Parse video captions from the video's captions file."""
|
||||
"""Parse video captions from the video's captions file. Returns deduplicated caption text as a string."""
|
||||
try:
|
||||
lines = get_unique_caption_lines(str(caption_path))
|
||||
block = '\n'.join(lines)
|
||||
return block
|
||||
except UnicodeDecodeError as e:
|
||||
# Failed to decode the caption file
|
||||
# TODO handle this error
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
except webvtt.errors.MalformedFileError as e:
|
||||
# File format is broken
|
||||
except webvtt.errors.MalformedFileError:
|
||||
pass
|
||||
except webvtt.errors.MalformedCaptionError as e:
|
||||
# Captions form is broken somehow
|
||||
except webvtt.errors.MalformedCaptionError:
|
||||
pass
|
||||
|
||||
logger.debug(f'Failed to parse caption file {caption_path}')
|
||||
|
||||
|
||||
def read_captions_with_timestamps(caption_path: Path) -> Optional[List[dict]]:
|
||||
"""Parse video captions preserving timestamps. Returns a list of dicts with 'start_seconds' and 'text' keys.
|
||||
Overlapping and duplicate lines are deduplicated."""
|
||||
try:
|
||||
raw_chunks = _parse_caption_file(caption_path)
|
||||
chunks = _deduplicate_caption_chunks(raw_chunks)
|
||||
return chunks if chunks else None
|
||||
except UnicodeDecodeError:
|
||||
pass
|
||||
except webvtt.errors.MalformedFileError:
|
||||
pass
|
||||
except webvtt.errors.MalformedCaptionError:
|
||||
pass
|
||||
logger.debug(f'Failed to parse caption file {caption_path}')
|
||||
return None
|
||||
|
||||
|
||||
def extract_captions(path: pathlib.Path) -> str | None:
|
||||
"""Extract captions that are embedded in a video file."""
|
||||
with tempfile.TemporaryDirectory() as directory:
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
import pytest
|
||||
|
||||
from wrolpi import captions
|
||||
|
||||
|
||||
|
|
@ -47,3 +49,21 @@ def test_get_unique_caption_lines(vtt_file1, srt_file3):
|
|||
'of background information we come from a',
|
||||
'company called le code from finland and']
|
||||
assert list(captions.get_unique_caption_lines(srt_file3)) == expected
|
||||
|
||||
|
||||
EXPECTED_CAPTION_CHUNKS = [
|
||||
{'start_seconds': 5.269, 'text': 'okay welcome to this session this is'},
|
||||
{'start_seconds': 5.279, 'text': 'called the kinetic bunny need to meet'},
|
||||
{'start_seconds': 7.77, 'text': 'the virtual bonnie emilio stop this is'},
|
||||
{'start_seconds': 11.46, 'text': 'jana car who just to give a little bit'},
|
||||
{'start_seconds': 15.42, 'text': 'of background information we come from a'},
|
||||
{'start_seconds': 18.869, 'text': 'company called le code from finland and'},
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize('fixture_name', ['vtt_file1', 'srt_file3'])
|
||||
def test_read_captions_with_timestamps(fixture_name, request):
|
||||
"""read_captions_with_timestamps returns timestamped caption chunks with overlapping lines deduplicated."""
|
||||
caption_file = request.getfixturevalue(fixture_name)
|
||||
chunks = captions.read_captions_with_timestamps(caption_file)
|
||||
assert chunks == EXPECTED_CAPTION_CHUNKS
|
||||
|
|
|
|||
|
|
@ -1 +1 @@
|
|||
0.18.1-beta
|
||||
0.19.0-beta
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue