diff --git a/app/cypress/e2e/domains/domain-edit.cy.js b/app/cypress/e2e/domains/domain-edit.cy.js index e944e067..d5688503 100644 --- a/app/cypress/e2e/domains/domain-edit.cy.js +++ b/app/cypress/e2e/domains/domain-edit.cy.js @@ -177,7 +177,7 @@ describe('Domain Editing Workflow', () => { } }).as('invalidUpdate'); - cy.contains('label', 'Directory').parent().find('input').clear().type('/invalid/absolute/path'); + cy.contains('label', 'Directory').parent().find('input').should('not.be.disabled').clear().type('/invalid/absolute/path'); cy.contains('button', 'Save').click(); cy.wait('@invalidUpdate'); diff --git a/app/src/components/VideoPlayer.js b/app/src/components/VideoPlayer.js index f6a605f7..d8f4376e 100644 --- a/app/src/components/VideoPlayer.js +++ b/app/src/components/VideoPlayer.js @@ -1,4 +1,4 @@ -import React, {useContext} from 'react'; +import React, {useContext, useState} from 'react'; import {useHotkeys} from 'react-hotkeys-hook'; import {deleteVideos, downloadVideoMetadata, tagFileGroup, untagFileGroup} from "../api"; import {Link, useNavigate, useParams} from "react-router"; @@ -19,12 +19,12 @@ import { import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Container from "semantic-ui-react/dist/commonjs/elements/Container"; import {VideoPlaceholder} from "./Placeholder"; -import {useChannel, useVideoExtras} from "../hooks/customHooks"; +import {useChannel, useVideoCaptions, useVideoExtras} from "../hooks/customHooks"; import {ThemeContext} from "../contexts/contexts"; -import {Button, darkTheme, Header, Icon, Segment, Tab} from "./Theme"; +import {Button, darkTheme, Header, Icon, Loader, Segment, Tab} from "./Theme"; import {VideoCard} from "./Videos"; import {TagsSelector} from "../Tags"; -import {Comment, CommentGroup, Label} from "semantic-ui-react"; +import {Comment, CommentGroup, Input, Label} from "semantic-ui-react"; const MEDIA_PATH = '/media'; @@ -198,7 +198,8 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { // Get the Video's channel, fallback to the URL's channel id. const {channelId} = useParams(); const {channel} = useChannel(video && video.channel_id ? video.channel_id : channelId); - const {comments, captions} = useVideoExtras(videoFile?.video?.id); + const {comments} = useVideoExtras(videoFile?.id); + const {captions, captionsLoading, fetchCaptions} = useVideoCaptions(videoFile?.id); const mediaTitle = videoFile ? (videoFile.title || videoFile.stem || videoFile.name) : null; const mediaArtworkUrl = video && video.poster_path ? `${MEDIA_PATH}/${encodeMediaPath(video.poster_path)}` : null; @@ -290,9 +291,12 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { }; const captionsPane = { - menuItem: 'Captions', render: () => -
{captions || 'No captions available.'}
-
+ menuItem: 'Captions', render: () => }; const getFile = (suffix) => videoFile.files.find(i => i.path.toLowerCase().endsWith(suffix)); @@ -479,6 +483,71 @@ function VideoPage({videoFile, prevFile, nextFile, fetchVideo, ...props}) { } +function formatCaptionTimestamp(totalSeconds) { + totalSeconds = Math.floor(totalSeconds); + const hours = Math.floor(totalSeconds / 3600); + const minutes = Math.floor((totalSeconds % 3600) / 60); + const seconds = totalSeconds % 60; + const pad2 = (n) => String(n).padStart(2, '0'); + if (hours > 0) { + return `${hours}:${pad2(minutes)}:${pad2(seconds)}`; + } + return `${minutes}:${pad2(seconds)}`; +} + +function CaptionsPane({captions, captionsLoading, fetchCaptions, setVideoTime}) { + const [filter, setFilter] = useState(''); + + // Trigger fetch on first render of this pane. + React.useEffect(() => { + fetchCaptions(); + }, []); + + if (captionsLoading) { + return ; + } + + if (!captions || captions.length === 0) { + return No captions available.; + } + + let filtered = captions; + if (filter.trim()) { + const words = filter.toLowerCase().split(/\s+/).filter(Boolean); + filtered = captions.filter(chunk => { + const text = chunk.text.toLowerCase(); + return words.every(word => text.includes(word)); + }); + } + + return + setFilter(value)} + style={{marginBottom: '1em'}} + /> +
+ {filtered.map((chunk, idx) => ( +
+ { + e.preventDefault(); + setVideoTime(chunk.start_seconds); + }} + style={{whiteSpace: 'nowrap', fontFamily: 'monospace'}} + > + {formatCaptionTimestamp(chunk.start_seconds)} + + {chunk.text} +
+ ))} +
+
; +} + export default VideoPage; const chapterRegex = new RegExp('^(\\(?(?:((\\d?\\d):)?(?:(\\d?\\d):(\\d\\d)))\\)?)\.?\\s+(.*)$', 'i'); diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 52a9e563..8bc63e91 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -692,7 +692,6 @@ export const useVideo = (fileGroupId) => { export const useVideoExtras = (fileGroupId) => { // Fetches extra data to display on a Video's page. const [comments, setComments] = useState(null); - const [captions, setCaptions] = useState(null); const fetchComments = async () => { try { @@ -703,26 +702,43 @@ export const useVideoExtras = (fileGroupId) => { } } + useEffect(() => { + if (fileGroupId) { + fetchComments(); + } else { + setComments(null); + } + }, [fileGroupId]); + + return {comments} +} + +export const useVideoCaptions = (fileGroupId) => { + const [captions, setCaptions] = useState(null); + const [captionsLoading, setCaptionsLoading] = useState(false); + const captionsFetched = useRef(false); + const fetchCaptions = async () => { + if (captionsFetched.current || !fileGroupId) return; + setCaptionsLoading(true); try { const result = await getVideoCaptions(fileGroupId); setCaptions(result.captions); } catch (e) { console.error(e); + } finally { + setCaptionsLoading(false); + captionsFetched.current = true; } } useEffect(() => { - if (fileGroupId) { - fetchComments(); - fetchCaptions(); - } else { - setComments(null); - setCaptions(null); - } + setCaptions(null); + setCaptionsLoading(false); + captionsFetched.current = false; }, [fileGroupId]); - return {comments, captions} + return {captions, captionsLoading, fetchCaptions} } export const useChannel = (channel_id) => { diff --git a/modules/videos/models.py b/modules/videos/models.py index fe77f541..b9d80527 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -8,7 +8,7 @@ from sqlalchemy.orm import relationship, Session, deferred from sqlalchemy.orm.collections import InstrumentedList from modules.videos.errors import UnknownVideo, UnknownChannel -from wrolpi.captions import read_captions +from wrolpi.captions import read_captions, read_captions_with_timestamps from wrolpi.common import Base, ModelHelper, logger, get_media_directory, get_relative_to_media_directory, \ background_task from wrolpi.db import get_db_curs, get_db_session @@ -336,6 +336,20 @@ class Video(ModelHelper, Base): return caption_text + def get_caption_chunks(self) -> Optional[List[dict]]: + """Search the FileGroup's files for a caption file. Return captions with timestamps.""" + caption_paths = self.caption_paths + caption_chunks = None + if english_vtt := [i for i in caption_paths if i.name.endswith('.en.vtt')]: + caption_chunks = read_captions_with_timestamps(english_vtt[0]) + elif vtt := [i for i in caption_paths if i.name.endswith('.vtt')]: + caption_chunks = read_captions_with_timestamps(vtt[0]) + elif english_srt := [i for i in caption_paths if i.name.endswith('.en.srt')]: + caption_chunks = read_captions_with_timestamps(english_srt[0]) + elif srt := [i for i in caption_paths if i.name.endswith('.srt')]: + caption_chunks = read_captions_with_timestamps(srt[0]) + return caption_chunks + @staticmethod def get_by_path(session: Session, path) -> Optional['Video']: video = session.query(Video) \ diff --git a/modules/videos/schema.py b/modules/videos/schema.py index d46ffd77..85d47363 100644 --- a/modules/videos/schema.py +++ b/modules/videos/schema.py @@ -164,7 +164,7 @@ class VideoCommentsResponse: @dataclass class VideoCaptionsResponse: - captions: str + captions: list @dataclass diff --git a/modules/videos/video/api.py b/modules/videos/video/api.py index cd5db3bf..459c8a52 100644 --- a/modules/videos/video/api.py +++ b/modules/videos/video/api.py @@ -44,7 +44,7 @@ def video_get_comments(_: Request, file_group_id: int): @openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse) def video_get_captions(_: Request, file_group_id: int): video = lib.get_video(file_group_id) - return json_response({'captions': video.file_group.d_text}) + return json_response({'captions': video.get_caption_chunks()}) @video_bp.post('/search') diff --git a/modules/videos/video/test/test_api.py b/modules/videos/video/test/test_api.py index 27cd7f2e..5c0f6a62 100644 --- a/modules/videos/video/test/test_api.py +++ b/modules/videos/video/test/test_api.py @@ -197,7 +197,11 @@ async def test_api_video_extras(async_client, simple_channel, video_factory): request, response = await async_client.get(f'/api/videos/{video.file_group_id}/captions') assert response.status_code == HTTPStatus.OK - assert response.json.get('captions') + captions = response.json.get('captions') + assert isinstance(captions, list) + assert len(captions) > 0 + assert 'start_seconds' in captions[0] + assert 'text' in captions[0] request, response = await async_client.get(f'/api/videos/{video.file_group_id}/comments') assert response.status_code == HTTPStatus.OK diff --git a/wrolpi/captions.py b/wrolpi/captions.py index 092f12f4..c88ab549 100755 --- a/wrolpi/captions.py +++ b/wrolpi/captions.py @@ -3,7 +3,7 @@ import pathlib import subprocess import tempfile from pathlib import Path -from typing import Generator, Union +from typing import Generator, List, Optional, Union import srt import webvtt @@ -11,23 +11,66 @@ import webvtt from wrolpi.cmd import FFMPEG_BIN from wrolpi.common import logger -__all__ = ['read_captions', 'extract_captions'] +__all__ = ['read_captions', 'read_captions_with_timestamps', 'extract_captions'] + + +def _parse_vtt_timestamp(timestamp: str) -> float: + """Parse a VTT timestamp like '00:01:05.269' to seconds as a float.""" + parts = timestamp.strip().split(':') + if len(parts) == 3: + hours, minutes, seconds = parts + elif len(parts) == 2: + hours = '0' + minutes, seconds = parts + else: + return 0.0 + return int(hours) * 3600 + int(minutes) * 60 + float(seconds) + + +def _parse_caption_file(caption_path: Union[str, Path]) -> List[dict]: + """Parse a VTT or SRT file into raw caption chunks with timestamps. + Returns a list of dicts with 'start_seconds' and 'text' keys.""" + raw_chunks = [] + caption_path = str(caption_path) + if caption_path.endswith('vtt'): + for caption in webvtt.read(caption_path): + text = str(caption.text).strip() + if text: + start = _parse_vtt_timestamp(caption.start) + raw_chunks.append({'start_seconds': start, 'text': text}) + else: + with open(caption_path, 'rt') as fh: + contents = fh.read() + for subtitle in srt.parse(contents): + text = subtitle.content.strip() + if text: + start = subtitle.start.total_seconds() + raw_chunks.append({'start_seconds': start, 'text': text}) + return raw_chunks + + +def _deduplicate_caption_chunks(raw_chunks: List[dict]) -> List[dict]: + """Deduplicate caption chunks by extracting only new lines from each chunk. + + YouTube auto-captions produce overlapping chunks where each chunk repeats lines from the previous one + plus new content. This extracts only the new lines and assigns the chunk's start timestamp.""" + chunks = [] + last_lines = [] + for chunk in raw_chunks: + lines = [line for line in chunk['text'].split('\n') if line.strip()] + new_lines = [line for line in lines if line not in last_lines] + if new_lines: + text = '\n'.join(new_lines) + if not chunks or text != chunks[-1]['text']: + chunks.append({'start_seconds': chunk['start_seconds'], 'text': text}) + last_lines = lines + return chunks def get_caption_text(caption_path: Union[str, Path]) -> Generator: """Return all text from each caption of a caption file.""" - if str(caption_path).endswith('vtt'): - # VTT - for caption in webvtt.read(caption_path): - text = str(caption.text).strip() - yield text - else: - # Finally, try SRT - with open(caption_path, 'rt') as fh: - contents = fh.read() - for subtitle in srt.parse(contents): - if any(subtitle.content): - yield subtitle.content + for chunk in _parse_caption_file(caption_path): + yield chunk['text'] def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator: @@ -41,25 +84,37 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator: def read_captions(caption_path: Path): - """Parse video captions from the video's captions file.""" + """Parse video captions from the video's captions file. Returns deduplicated caption text as a string.""" try: lines = get_unique_caption_lines(str(caption_path)) block = '\n'.join(lines) return block - except UnicodeDecodeError as e: - # Failed to decode the caption file - # TODO handle this error + except UnicodeDecodeError: pass - except webvtt.errors.MalformedFileError as e: - # File format is broken + except webvtt.errors.MalformedFileError: pass - except webvtt.errors.MalformedCaptionError as e: - # Captions form is broken somehow + except webvtt.errors.MalformedCaptionError: pass - logger.debug(f'Failed to parse caption file {caption_path}') +def read_captions_with_timestamps(caption_path: Path) -> Optional[List[dict]]: + """Parse video captions preserving timestamps. Returns a list of dicts with 'start_seconds' and 'text' keys. + Overlapping and duplicate lines are deduplicated.""" + try: + raw_chunks = _parse_caption_file(caption_path) + chunks = _deduplicate_caption_chunks(raw_chunks) + return chunks if chunks else None + except UnicodeDecodeError: + pass + except webvtt.errors.MalformedFileError: + pass + except webvtt.errors.MalformedCaptionError: + pass + logger.debug(f'Failed to parse caption file {caption_path}') + return None + + def extract_captions(path: pathlib.Path) -> str | None: """Extract captions that are embedded in a video file.""" with tempfile.TemporaryDirectory() as directory: diff --git a/wrolpi/test/test_captions.py b/wrolpi/test/test_captions.py index a0d2fecb..04df3bbb 100644 --- a/wrolpi/test/test_captions.py +++ b/wrolpi/test/test_captions.py @@ -1,3 +1,5 @@ +import pytest + from wrolpi import captions @@ -47,3 +49,21 @@ def test_get_unique_caption_lines(vtt_file1, srt_file3): 'of background information we come from a', 'company called le code from finland and'] assert list(captions.get_unique_caption_lines(srt_file3)) == expected + + +EXPECTED_CAPTION_CHUNKS = [ + {'start_seconds': 5.269, 'text': 'okay welcome to this session this is'}, + {'start_seconds': 5.279, 'text': 'called the kinetic bunny need to meet'}, + {'start_seconds': 7.77, 'text': 'the virtual bonnie emilio stop this is'}, + {'start_seconds': 11.46, 'text': 'jana car who just to give a little bit'}, + {'start_seconds': 15.42, 'text': 'of background information we come from a'}, + {'start_seconds': 18.869, 'text': 'company called le code from finland and'}, +] + + +@pytest.mark.parametrize('fixture_name', ['vtt_file1', 'srt_file3']) +def test_read_captions_with_timestamps(fixture_name, request): + """read_captions_with_timestamps returns timestamped caption chunks with overlapping lines deduplicated.""" + caption_file = request.getfixturevalue(fixture_name) + chunks = captions.read_captions_with_timestamps(caption_file) + assert chunks == EXPECTED_CAPTION_CHUNKS diff --git a/wrolpi/version.txt b/wrolpi/version.txt index 539c147a..bbc79db2 100644 --- a/wrolpi/version.txt +++ b/wrolpi/version.txt @@ -1 +1 @@ -0.18.1-beta +0.19.0-beta