wrolpi/modules/videos/schema.py

305 lines
6.3 KiB
Python
Raw Permalink Normal View History

from dataclasses import dataclass, field
from typing import List, Optional
2022-01-31 21:18:49 -07:00
from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT
from wrolpi.errors import ValidationError
2022-01-31 21:18:49 -07:00
@dataclass
class ChannelPostRequest:
name: str
directory: str
2022-01-31 21:18:49 -07:00
calculate_duration: Optional[bool] = None
download_missing_data: Optional[bool] = True
2022-01-31 21:18:49 -07:00
generate_posters: Optional[bool] = None
source_id: Optional[str] = None
2024-10-04 16:32:48 -06:00
tag_name: Optional[str] = None
url: Optional[str] = None
def __post_init__(self):
self.name = self.name.strip() or None
self.directory = self.directory.strip() or None
self.url = self.url.strip() if self.url else None
self.source_id = self.source_id.strip() if self.source_id else None
@dataclass
class ChannelPutRequest:
2022-01-31 21:18:49 -07:00
calculate_duration: Optional[bool] = None
directory: Optional[str] = None
download_missing_data: Optional[bool] = True
2022-01-31 21:18:49 -07:00
generate_posters: Optional[bool] = None
mkdir: Optional[bool] = None
name: Optional[str] = None
url: Optional[str] = None
def __post_init__(self):
self.name = self.name.strip() or None
self.directory = self.directory.strip() or None
self.url = self.url.strip() if self.url else None
@dataclass
class ChannelTagRequest:
tag_name: Optional[str] = None
directory: Optional[str] = None
def __post_init__(self):
if self.directory and self.directory.startswith('/'):
raise ValidationError('Directory must be relative to media directory')
@dataclass
class ChannelTagInfoRequest:
channel_id: Optional[int] = None
tag_name: Optional[str] = None
@dataclass
class ChannelDownloadRequest:
url: str
frequency: int
settings: dict = field(default_factory=dict)
def __post_init__(self):
self.url = self.url.strip()
if not self.url:
raise ValidationError('url cannot be empty')
# Validate settings contents. Remove empty values.
from wrolpi.schema import DownloadSettings
self.settings = {k: v for k, v in DownloadSettings(**self.settings).__dict__.items() if v not in ([], None)}
@dataclass
class RefreshRequest:
channel_name: str
@dataclass
class ChannelModel:
id: int
url: str
name: str
directory: str
@dataclass
class ChannelResponse:
channel = ChannelModel
@dataclass
class ChannelPostResponse:
success: str
@dataclass
class ChannelsChannelModel:
id: int
name: str
@dataclass
class ChannelsResponse:
channels: List[ChannelsChannelModel]
@dataclass
class StreamResponse:
success: str
stream_url: str
@dataclass
class Video:
id: int
description_path: str
ext: str
poster_path: str
source_id: str
title: str
video_path: str
name: str
caption_path: str
info_json_path: str
channel_id: int
@dataclass
class VideoWithChannel:
id: int
description_path: str
ext: str
poster_path: str
source_id: str
title: str
video_path: str
name: str
caption_path: str
info_json_path: str
channel_id: int
channel = ChannelModel
@dataclass
class ExtendedVideo(Video):
info_json: dict
@dataclass
class VideoResponse:
file_group = ExtendedVideo
Fixing JS warnings. Handling all combinations of toggle in light and dark. Changing CPUUsageProgress colors to be more readable in dark mode. Moving theme handling into Theme.js. Importing element wrappers from Theme.js to handle element theming. Changing video menu to dark mode. Creating ThemeContext to share Dark Mode setting. Removing duplicate code for search functions. Displaying X when downloads are disabled. Using react context to share settings across components. Refusing to start workers when WROL Mode is enabled. Changing warning to debug. Queuing recurring downloads first. Adding Video.url index. Adding special handling for youtube shorts to videos. Reworking Downloads to have one worker per process. Testing creating downloads and getting downloaders. Testing video validation and poster generation. Testing video statistics. Increasing inventory test coverage. Adding pytest.ini to automatically detect async test. Adding more archive test coverage. Testing Archive CRUD. Catching and logging when encountering error during indexing. Using FileIcon when Archive has no screenshot. A Video will be assigned to a Channel by its info json first, then to its directory. Including suffix in File json. Adding index for File.suffix. Gathering suffix during indexing. Passing entire file down to FileIcon. Moving logging up. Returning File.associated in file json. Associated files are last in file search. Removing duplicate refreshing code by creating new _refresh_directory function. Changing File.idempotency to a datetime to avoid having to clear idempotency. Truncating indexes to avoid error with large files. Don't overwrite video data when info json is empty. Using already_downloaded for channel videos. Awaiting refresh in Channel.refresh. Testing that videos already downloaded aren't downloaded again. Adding indexes for video/archive referenced paths. Changing grid width for mobile or computer. Adding button to refresh specific directories. Restoring Channel edit link. Fixing caption path in video_modeler. Adding ZIP files to mimetype colors. Using model instead of first file's model. Fetching File models in batches to avoid issuing a SELECT for every File with a model. Flushing generated poster file. Getting Channel in VideoPage, relying on Video.channl_id, falling back to URL's channel_id. Attaching Video to it's Channel during download. Using ExternalCardLink for ArchivePage singlefile link. Validating and indexing videos after download. Handling glob when file name contains matching characters. Reworking and testing split_path_stem_and_suffix and glob_shared_stem. Adding associated video files to DB during upsert. Adding pending downloads to dashboard. Hiding status when download is open. Handling when load is empty. Adding load statistics to Dashboard. Limiting concurrent after_refresh functions. Handling when no onSelect is declared. Adding ImageRow for FileTable. Moving image date into Card.Meta. Using ExternalLink for FileCard. Using ExternalLink for ImageCard. Handling when error occurs during refresh of a directory. Committing during refresh to avoid losing all files if an error occurs. Removing api health check to avoid spamming logs. Creating ImageCard to display image posters when they aren't too large. Removing archive.domain_id not null. Reworking file refresh to process all files in a single directory. Bumping version for major file rework. No longer checking for video corruption. Adding duration by default to speed up tests. Using register_indexer for all indexers. Adding CPU usage progress bar in Welcome. Modifying TextIndexer to ignore large text files. Counting all files during refresh. Adding refresh_logger to clarify logging. Checking for wrol_mode before refresh functions. Removing unused idempotency from video/channel tables. Using magic module to speed up mimetype identification. All tests pass. Archive tests pass. Changing video get endpoint to return 'file' to avoid ambiguity. Video tests pass. Simplifying regex filtering. Using regex to ignore case when filtering. Converting all channel classes to functions. Restoring channel refresh and download buttons. Matching format of Archive and Video pages. Adding sorting to Archives search. Reworking video orders for file search. Using SearchInput for videos search. Archives can be searched on the Archives page. Domains can be filtered. Reworking Archive search to use Files. Videos, Archives and Global search use the FilesView. Videos are claimed by their Channel based on the Channel's directory. Prev and Next video cards are displayed on the VideoPage. Creating useVideo for reworked VideoWrapper and VideoPage. Adding Paginator to FilesView. Moving file refresh button out of manage tab. Removing Files page tabs. Limiting refresh to one at a time. Adding class for card icon to match poster height. Video and Archives always re-claim their files on refresh. Archive files are indexed and claimed. FileView table is compact and striped. Videos page uses FileView. Updated navlinks to router v6. Removing ! character from file names. Disabling select button when cards are visible. Adding show/hide button for FileTable selection. The Archive Page is displayed. Adding mimetype dropdown. Upgrading to React Router v6 for useSearchParams. Validating video during modeling. Removing screenshot during corruption check because it was very slow. Updating app to use video files as search results. Updating video search to return Files. Video and WROLPi tests pass. Creating function to refresh files within a channel's directory. Removing troubleshooting print. Updating result order in test. Reworking video_factory to handle a specific video file location. Returning video model on file search. Getting session using video_file. Screenshotting when checking for video corruption. Reducing ffmpeg calls by gathering duration during poster generation. Moving video file matching into model. VideoIndexer is moved into video module. Video files are validated. Moving files module into wrolpi in preparation to use it in modules. Title and contents are indexed for text file. Files are indexed on refresh by using their indexer. Zip files are indexed using their contents. Testing multiple zip files. Adding unique to file.path. Checking that the correct indexer is used. Adding index to file.modification_datetime and file.size. Handling file info gathering in File.do_index. Only reading contents of file if file.indexed is False. Video model is found during file refresh. A Video record is created that has the associated files. Videos without associated files are found. Videos are found using registered video_modeler() rather than in refresh code. Removing old video columns. Handling video info json files. Trying to import each module individually. Will continue to import modules even if one fails. Reworking refresh so it performs less DB calls. Converting file.path to primary key. file.path is always a pathlib.Path. Updating video_factory to use new file system, testing it. Bump sanic from 21.9.1 to 21.12.2 in /docker/archive Bumps [sanic](https://github.com/sanic-org/sanic) from 21.9.1 to 21.12.2. - [Release notes](https://github.com/sanic-org/sanic/releases) - [Changelog](https://github.com/sanic-org/sanic/blob/main/CHANGELOG.rst) - [Commits](https://github.com/sanic-org/sanic/compare/v21.9.1...v21.12.2) --- updated-dependencies: - dependency-name: sanic dependency-type: direct:production ... Signed-off-by: dependabot[bot] <support@github.com>
2022-08-06 05:22:07 +00:00
prev = ExtendedVideo
next = ExtendedVideo
@dataclass
class VideoCommentsResponse:
comments: list[dict]
@dataclass
class VideoCaptionsResponse:
captions: list
Fix API stalls: non-blocking switch worker, threaded searches, indexed Videos page (#678) * Fix API stalls: non-blocking switch worker, threaded searches, indexed Videos page Three response-time fixes found by benchmarking the read-only API endpoints hot and cold on real libraries: * `switch_worker` runs on the Sanic worker's event loop every 0.1s and called the blocking `multiprocessing.Event.wait(timeout=1)`. With no switch pending (the normal state) this froze the loop for a second at a time; about a third of all requests, including trivial ones, took ~1,000 ms. It now polls `is_set()` and returns immediately. * The search handlers (videos, files, archives, docs, zims) and `/api/statistics` ran synchronous SQLite queries on the event loop, so one slow search stalled every other request on that worker. They now run via `asyncio.to_thread`. * The unfiltered Videos page (`POST /api/videos/search` with only an order) used `mimetype LIKE 'video/%' OR ...` plus `COUNT(*) OVER()`, which visits every video's index entry before LIMIT applies: seconds cold on a large library on a spinning disk, and ~10 s hot for a deep page. It now seeks each exact mimetype on the (mimetype, effective_datetime) index for `limit + offset` rows and merges them, with the total from a separate count cached for 60 s. Results are identical to the generic query. Also adds `scripts/api_perf.py` (times every read-only endpoint against a host, optionally restarting the API over SSH for cold numbers) and `scripts/api_perf_compare.py` (merges per-host results). Host profiles live in a gitignored JSON file; an example is included. Measured on an 8-core docker host with a 10 GB library: median hot request 1,007 ms -> 22 ms; Videos page 1 hot 287 -> 108 ms, page 50 1,345 -> 120 ms. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Serve the video description from its own endpoint instead of Video.__json__ `Video.__json__` computed `description` as `c_text or get_video_description()`, where the fallback opens and parses the video's info json from disk. The list endpoints serialize every result with `__json__`, so any page containing videos without an indexed description (common: many videos have none) read their info json files on every request, cold, for a field the list never displays. The description is not part of search either: search runs against the FTS copy of `c_text`, so the fallback never fed the index. Drop `description` from `__json__` and serve it from `GET /api/videos/<id>/description`, like comments and captions. The endpoint returns the indexed `c_text`, falling back to the info json only there, for a row indexed before descriptions were stored. The player page fetches it through `useVideoExtras` alongside the comments. Tests assert the list and detail responses never touch the info json, and cover the endpoint's indexed, fallback, and not-found cases. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> * Address review: thread-local zim sessions, fast-path equivalence test, comments * Zim search, single-zim search, and search estimates run libzim off the event loop, each opening its own session inside the worker thread rather than passing the request's SQLAlchemy Session across threads. * The Videos fast-path test now runs the same requests through the generic query and asserts identical rows and paging for both date orders. The one intentional difference is documented: past the end, the fast path still reports the true total. * switch_worker returns before its try block so the empty path does not clear the event; its comment states the invariant only. The copied "synchronous SQLite" comments are reduced to one line in one place. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-29 16:17:41 -06:00
@dataclass
class VideoDescriptionResponse:
description: Optional[str]
@dataclass
class VideoSearchRequest:
2022-01-31 21:18:49 -07:00
search_str: Optional[str] = None
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
tags: List[str] = field(default_factory=list)
offset: Optional[int] = None
2022-01-31 21:18:49 -07:00
limit: Optional[int] = VIDEO_QUERY_LIMIT
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
channel_id: Optional[int] = None
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
tag_names: List[str] = field(default_factory=list)
headline: bool = False
censored: bool = False
Search: fast ABC mode by default, opt-in deep search The file_group.textsearch tsvector indexes all four text columns; d_text (captions, document text, zip contents) is ~90% of the data, making ts_rank searches take seconds on large libraries. Add a textsearch_abc generated column (title/author/description only, ~8x smaller) and default all searches to it, with deep=true opting into the full column. Database: - Migration adds the generated column and a GIN index (rewrites the table; a few minutes on large libraries). Backend: - FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and DocSearchRequest accept `deep`; each search function picks the column. - /api/search_file_estimates returns both counts in one index-bound query (COUNT FILTER on the abc subset): `file_groups` (fast) and `file_groups_deep`. UI (one coherent story across the count surfaces): - The typing dropdown always suggests the deep count (what is findable). - The tab badge shows the active mode's count so it matches the result list. - The gap is reconciled by a hint offering deep search: count-aware on the global search ("No results in fast search. N results available in deep search." / "N more results available..."), generic on the module pages (no deep estimate there). The hint resets pagination. - "Search Depth" toggle in the search filter modals, persisted as ?deep=true. Tests that relied on d_text matches now pass deep=True, with fast-mode assertions alongside. Also skip flaky test_download_channel on CircleCI. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
deep: bool = False # When True, search all text including captions (d_text)
Jobs queue, video Edit dropdown: transcode/remux, edit title & description (#711) * Jobs queue and video transcode from the video page Add an in-memory FIFO Job queue (wrolpi/jobs.py): functions register by name, callers enqueue by name with JSON kwargs, and the perpetual-tasks owner runs them one at a time. Logging emitted inside a Job and the output of its subprocesses are captured into a per-Job log; Jobs report progress and can be cancelled (a running Job's subprocess is killed). Jobs are not persisted; an API restart forgets them. Add a transcode Job and POST /api/videos/<id>/transcode. ffmpeg reports progress via -progress pipe:1. The existing FileGroup is pointed at the new file before the refresh so tags and history survive a container change. Replace the video page Refresh button with an Edit dropdown (Refresh, Transcode). The Transcode modal pre-fills from the preferred codecs in the Videos settings and shows the Job's progress, log tail and Cancel. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: move Delete into the Edit dropdown The Edit dropdown now holds Refresh, Transcode and Delete, so the action row is Download / Edit / Add to Playlist. Delete keeps its confirmation (a Confirm dialog) and the tagged-files prompt, and is disabled in WROL Mode like the other items. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: Edit > Edit to change a video's title PUT /api/videos/<file_group_id> writes the title (and fulltitle) into the Video's info json, creating the file when the video has none, then validates the Video again so its title and search text follow. Files stay the source of truth: the edit survives a refresh, and a re-download of the metadata replaces it (the modal says so). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: edit the description in Edit > Edit PUT /api/videos/<file_group_id> now takes optional title and description and changes only the fields given; an empty description clears it. The modal sends only what changed and refetches the description after saving. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode modal: allow a container-only remux Keeping both streams is no longer refused: ffmpeg copies them into the chosen container with fast start. The container select starts at the file's own container, the hint explains what a remux does, and the button reads Remux when nothing is re-encoded. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * WROL Mode: refuse video edits and transcodes past the API The routes already returned 403, but a transcode Job queued before WROL Mode was enabled still ran, update_video could be called from anywhere, and the Edit modal's Save button stayed live. The Job now fails before ffmpeg starts, update_video carries wrol_mode_check, and the modal disables Save with a notice. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: verify the output before replacing the original After ffmpeg exits, the output is probed and compared with the source (container, presence of video/audio streams, codec per stream, duration within 1s or 1%) and its last 30 seconds are decoded to a null output. Any failure removes the output and keeps the original. The tail decode replaces the download-time completeness check, which grabbed a single frame near the end: one ffmpeg_video_complete (sync for the downloader, async for Jobs) now serves both, and is stricter, since every frame of the tail plus the audio must decode. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Jobs: claim atomically, cap pending, own-session subprocesses; retarget before delete Review fixes. The worker read a Job's status and wrote RUNNING as two steps, so a cancel landing between them was overwritten and the Job ran; _claim_job now does both under the lock. enqueue_job refuses past JOB_PENDING_LIMIT pending Jobs. JobContext.run_command starts the subprocess in its own session and raises CancelledError when the command was killed, so a handler cannot mistake it for a normal exit. The transcode Job now retargets the FileGroup (committed) while the original file still exists, and deletes the original afterwards (transcode_video_file gained keep_original). A crash in between leaves both files with the database pointing at a real one. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: the Edit menu owns the transcode Job; polling survives Close and errors Review fixes. The transcode Job's id and polling (useJob) move from the modal to VideoEditMenu, so closing the modal no longer stops the page from learning the Job finished (and refetching the video, whose path may have changed). While a Job runs the menu shows Transcoding..., reopening shows its progress instead of the form, and Delete waits. A failed poll request is retried instead of ending the polling. getJobs checks the response like its siblings. Design narration trimmed from comments. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: audio files, and Remove video to turn a video into an audio file video_codec 'none' (REMOVE_VIDEO) drops the video stream (-vn, audio required); an audio-only source is audio-only output whatever the video target says, and a video target on it is refused. Audio output gets an audio container (m4a/ogg/mp3) checked against the codec it will hold, and a video container asked for audio output is replaced by the codec's natural one. faststart only for mp4/m4a. Verification learned the audio-only shape and ffprobe's container names (mkv reports matroska). The modal offers 'Remove video (audio only)', switches to audio containers and follows the audio codec, reads Extract Audio, and opens for audio files (no video codec select). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: yellow Edit button Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: remove the temp output on cancel; tell the user when a closed-modal Job fails or is lost Review fixes. The temp output was only removed on Exception, but a cancelled Job's run_command raises CancelledError (a BaseException), so a cancel left the partial file for a refresh to index. Closing the modal during a Job that then failed showed nothing; it now toasts the error (and a cancellation). Polling gives up on a 404 or five failures in a row (the API restarted and forgot its Jobs), warning the user and unlocking Transcode and Delete instead of reading Transcoding... forever. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Jobs: _claim_job docstring states the invariant, not the old race Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: fragmented mp4 option for very long videos A whole-file mp4 index grows with the frame count; for a many-hour video it reaches tens of MB, which a phone must fetch and parse before the first frame, and every seek walks it. `fragmented` writes the file with -movflags +frag_keyframe+empty_moov+default_base_moof+global_sidx and 10s fragments: a ~1 KB moov plus a global seek index, so playback starts and seeks quickly. mp4/m4a only; a remux, so it takes seconds per GB. Measured on an 11-hour h264/aac file: moov 40 MB -> 1.4 KB, seek index 2 x 120 KB, 10k fragments, duration and codecs unchanged, remux in 25s. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode modal: the Fragmented label says audio for audio output Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video edits keep the original: custom_title/custom_description in the info json's wrolpi section Editing a title or description no longer overwrites yt-dlp's keys. The values go into the info json's `wrolpi` section (custom_title, custom_description); the modeler prefers them when deriving the Video, so a refresh keeps the edit while the original title and description stay in the file. An empty custom description means none. Both re-download paths carry the wrolpi section over: the metadata-only refresh merges it into the fresh info json, and a full re-download snapshots it before yt-dlp overwrites the file and restores it (then re-validates) in finalize_download. FileGroup.get_wrolpi_json reads it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 19:25:23 -06:00
@dataclass
class VideoUpdateRequest:
# Only the fields given are changed.
title: Optional[str] = None
description: Optional[str] = None
@dataclass
class VideoTranscodeRequest:
video_codec: Optional[str] = None # A key of TRANSCODE_VIDEO_TARGETS, REMOVE_VIDEO, or None to copy the stream.
audio_codec: Optional[str] = None # A key of TRANSCODE_AUDIO_TARGETS, REMOVE_AUDIO, or None to copy the stream.
Jobs queue, video Edit dropdown: transcode/remux, edit title & description (#711) * Jobs queue and video transcode from the video page Add an in-memory FIFO Job queue (wrolpi/jobs.py): functions register by name, callers enqueue by name with JSON kwargs, and the perpetual-tasks owner runs them one at a time. Logging emitted inside a Job and the output of its subprocesses are captured into a per-Job log; Jobs report progress and can be cancelled (a running Job's subprocess is killed). Jobs are not persisted; an API restart forgets them. Add a transcode Job and POST /api/videos/<id>/transcode. ffmpeg reports progress via -progress pipe:1. The existing FileGroup is pointed at the new file before the refresh so tags and history survive a container change. Replace the video page Refresh button with an Edit dropdown (Refresh, Transcode). The Transcode modal pre-fills from the preferred codecs in the Videos settings and shows the Job's progress, log tail and Cancel. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: move Delete into the Edit dropdown The Edit dropdown now holds Refresh, Transcode and Delete, so the action row is Download / Edit / Add to Playlist. Delete keeps its confirmation (a Confirm dialog) and the tagged-files prompt, and is disabled in WROL Mode like the other items. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: Edit > Edit to change a video's title PUT /api/videos/<file_group_id> writes the title (and fulltitle) into the Video's info json, creating the file when the video has none, then validates the Video again so its title and search text follow. Files stay the source of truth: the edit survives a refresh, and a re-download of the metadata replaces it (the modal says so). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: edit the description in Edit > Edit PUT /api/videos/<file_group_id> now takes optional title and description and changes only the fields given; an empty description clears it. The modal sends only what changed and refetches the description after saving. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode modal: allow a container-only remux Keeping both streams is no longer refused: ffmpeg copies them into the chosen container with fast start. The container select starts at the file's own container, the hint explains what a remux does, and the button reads Remux when nothing is re-encoded. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * WROL Mode: refuse video edits and transcodes past the API The routes already returned 403, but a transcode Job queued before WROL Mode was enabled still ran, update_video could be called from anywhere, and the Edit modal's Save button stayed live. The Job now fails before ffmpeg starts, update_video carries wrol_mode_check, and the modal disables Save with a notice. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: verify the output before replacing the original After ffmpeg exits, the output is probed and compared with the source (container, presence of video/audio streams, codec per stream, duration within 1s or 1%) and its last 30 seconds are decoded to a null output. Any failure removes the output and keeps the original. The tail decode replaces the download-time completeness check, which grabbed a single frame near the end: one ffmpeg_video_complete (sync for the downloader, async for Jobs) now serves both, and is stricter, since every frame of the tail plus the audio must decode. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Jobs: claim atomically, cap pending, own-session subprocesses; retarget before delete Review fixes. The worker read a Job's status and wrote RUNNING as two steps, so a cancel landing between them was overwritten and the Job ran; _claim_job now does both under the lock. enqueue_job refuses past JOB_PENDING_LIMIT pending Jobs. JobContext.run_command starts the subprocess in its own session and raises CancelledError when the command was killed, so a handler cannot mistake it for a normal exit. The transcode Job now retargets the FileGroup (committed) while the original file still exists, and deletes the original afterwards (transcode_video_file gained keep_original). A crash in between leaves both files with the database pointing at a real one. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: the Edit menu owns the transcode Job; polling survives Close and errors Review fixes. The transcode Job's id and polling (useJob) move from the modal to VideoEditMenu, so closing the modal no longer stops the page from learning the Job finished (and refetching the video, whose path may have changed). While a Job runs the menu shows Transcoding..., reopening shows its progress instead of the form, and Delete waits. A failed poll request is retried instead of ending the polling. getJobs checks the response like its siblings. Design narration trimmed from comments. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: audio files, and Remove video to turn a video into an audio file video_codec 'none' (REMOVE_VIDEO) drops the video stream (-vn, audio required); an audio-only source is audio-only output whatever the video target says, and a video target on it is refused. Audio output gets an audio container (m4a/ogg/mp3) checked against the codec it will hold, and a video container asked for audio output is replaced by the codec's natural one. faststart only for mp4/m4a. Verification learned the audio-only shape and ffprobe's container names (mkv reports matroska). The modal offers 'Remove video (audio only)', switches to audio containers and follows the audio codec, reads Extract Audio, and opens for audio files (no video codec select). Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video page: yellow Edit button Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: remove the temp output on cancel; tell the user when a closed-modal Job fails or is lost Review fixes. The temp output was only removed on Exception, but a cancelled Job's run_command raises CancelledError (a BaseException), so a cancel left the partial file for a refresh to index. Closing the modal during a Job that then failed showed nothing; it now toasts the error (and a cancellation). Polling gives up on a 404 or five failures in a row (the API restarted and forgot its Jobs), warning the user and unlocking Transcode and Delete instead of reading Transcoding... forever. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Jobs: _claim_job docstring states the invariant, not the old race Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode: fragmented mp4 option for very long videos A whole-file mp4 index grows with the frame count; for a many-hour video it reaches tens of MB, which a phone must fetch and parse before the first frame, and every seek walks it. `fragmented` writes the file with -movflags +frag_keyframe+empty_moov+default_base_moof+global_sidx and 10s fragments: a ~1 KB moov plus a global seek index, so playback starts and seeks quickly. mp4/m4a only; a remux, so it takes seconds per GB. Measured on an 11-hour h264/aac file: moov 40 MB -> 1.4 KB, seek index 2 x 120 KB, 10k fragments, duration and codecs unchanged, remux in 25s. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Transcode modal: the Fragmented label says audio for audio output Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Video edits keep the original: custom_title/custom_description in the info json's wrolpi section Editing a title or description no longer overwrites yt-dlp's keys. The values go into the info json's `wrolpi` section (custom_title, custom_description); the modeler prefers them when deriving the Video, so a refresh keeps the edit while the original title and description stay in the file. An empty custom description means none. Both re-download paths carry the wrolpi section over: the metadata-only refresh merges it into the fresh info json, and a full re-download snapshots it before yt-dlp overwrites the file and restores it (then re-validates) in finalize_download. FileGroup.get_wrolpi_json reads it. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 19:25:23 -06:00
container: str = 'mp4'
fragmented: bool = False # Fragmented mp4: quick start/seek for very long videos (mp4/m4a only).
@dataclass
class VideoTranscodeResponse:
job_id: str
@dataclass
class VideoSearchResponse:
videos: List[VideoWithChannel]
tsquery: str
@dataclass
class PaginationQuery:
offset: int
@dataclass
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
class TagRequest:
video_id: int
Making example label a tag. Tags can be created and deleted in UI. Adding tag_file using id to avoid session error. Using UTC in test. Matching tag_names style in test. Tag icon is displayed on file rows. Archives, Videos and Files can be filtered by their tags. Tags can be added and removed from UI. Tags are displayed in VideoPlayer. The tags of a file are returned in it's JSON. Video favorites are converted to Favorite tag on channel config import. Updating tests to match frontend usage. PDF creator and modification time are displayed. Reworking file search to use Video.__json__ for complex video relations. Archives can be viewed. Removing ebook.title. Reworking alembic migration to be an upgrade rather than a replace. Skipping test on circleci. Clearing out video caption index only if extract_subtitles if off. Channel tests pass. Some video tests pass. Missing videos can be found. A video can be downloaded. Orphaned files can be found. Videos can be tagged. Videos can be searched. Videos can be modeled. Converting some tests to use FileGroup.primary_path. Adding archive and domains to alembic migration. Testing cascading Archive titles. Archives now uses FileGroup. archive.lib tests pass. Archive is partially converted to FileGroup. Calculating FileGroup.primary file on refresh, rather than on select. wrolpi tests pass. Using FilesType to convert between str/pathlib.Path for FileGroup.files. Using list of files in FileGroup.files instead of dictionary. Storing files in dedicated FileGroup.files to avoid overwriting them. Adding comment explaining CASE. Not overwriting FileGroup.data on refresh if no change has been detected. Removing File model in favor of FileGroup.data. Adding typing to Files methods. Files are indexed using the primary mimetype of the FileGroup. files test_lib tests pass. Removing file_group_file in favor of simple one-to-many relationship. PDF files are indexed during refresh. Adding Tag and TagFile to tag files with specific tags. Only iterating when full_stem_ids has contents. Testing that FileGroups are related to their Files. Files and file_groups are created and linked. Bump webpack from 5.75.0 to 5.76.1 in /app (#160) Bumps [webpack](https://github.com/webpack/webpack) from 5.75.0 to 5.76.1. - [Release notes](https://github.com/webpack/webpack/releases) - [Commits](https://github.com/webpack/webpack/compare/v5.75.0...v5.76.1) --- updated-dependencies: - dependency-name: webpack dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2023-03-24 08:58:18 -06:00
tag_name: str
@dataclass
class WROLModeRequest:
enabled: bool
@dataclass
class VideoStatistics:
videos: int
week: int
month: int
year: int
sum_duration: int
sum_size: int
max_size: int
have_comments: int
missing_comments: int
failed_comments: int
@dataclass
class ChannelStatistics:
channels: int
tagged_channels: int
@dataclass
class VideosStatisticsResponse:
videos = VideoStatistics
channels = ChannelStatistics
@dataclass
class CensoredVideoRequest:
limit: int
offset: int
@dataclass
class CensoredVideoResponse:
videos: List[VideoWithChannel]
@dataclass
class ChannelSearchRequest:
tag_names: List[str] = field(default_factory=list)
search_str: Optional[str] = None
@dataclass
class VideoFileFormatRequest:
video_file_format: str
@dataclass
class CookiesUploadRequest:
"""Request to upload and encrypt cookies."""
cookies_content: str
password: str
def __post_init__(self):
if not self.cookies_content:
raise ValidationError('cookies_content cannot be empty')
if not self.password or len(self.password) < 8:
raise ValidationError('password must be at least 8 characters')
@dataclass
class CookiesUnlockRequest:
"""Request to unlock cookies with password."""
password: str
def __post_init__(self):
if not self.password:
raise ValidationError('password cannot be empty')
@dataclass
class CookiesStatusResponse:
"""Response with cookies status."""
cookies_exist: bool
cookies_unlocked: bool