2022-01-13 09:29:58 -07:00
|
|
|
from dataclasses import dataclass, field
|
|
|
|
|
from typing import List, Optional
|
2021-11-01 16:42:52 -06:00
|
|
|
|
2022-01-31 21:18:49 -07:00
|
|
|
from modules.videos.video.lib import DEFAULT_VIDEO_ORDER, VIDEO_QUERY_LIMIT
|
2024-06-25 22:12:13 -06:00
|
|
|
from wrolpi.errors import ValidationError
|
2022-01-31 21:18:49 -07:00
|
|
|
|
2021-11-01 16:42:52 -06:00
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelPostRequest:
|
2022-01-13 09:29:58 -07:00
|
|
|
name: str
|
|
|
|
|
directory: str
|
2022-01-31 21:18:49 -07:00
|
|
|
calculate_duration: Optional[bool] = None
|
2025-02-19 14:52:50 -07:00
|
|
|
download_missing_data: Optional[bool] = True
|
2022-01-31 21:18:49 -07:00
|
|
|
generate_posters: Optional[bool] = None
|
2022-04-08 09:57:04 -06:00
|
|
|
source_id: Optional[str] = None
|
2024-10-04 16:32:48 -06:00
|
|
|
tag_name: Optional[str] = None
|
|
|
|
|
url: Optional[str] = None
|
2022-04-08 09:57:04 -06:00
|
|
|
|
|
|
|
|
def __post_init__(self):
|
|
|
|
|
self.name = self.name.strip() or None
|
|
|
|
|
self.directory = self.directory.strip() or None
|
|
|
|
|
self.url = self.url.strip() if self.url else None
|
|
|
|
|
self.source_id = self.source_id.strip() if self.source_id else None
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelPutRequest:
|
2022-01-31 21:18:49 -07:00
|
|
|
calculate_duration: Optional[bool] = None
|
|
|
|
|
directory: Optional[str] = None
|
2025-02-19 14:52:50 -07:00
|
|
|
download_missing_data: Optional[bool] = True
|
2022-01-31 21:18:49 -07:00
|
|
|
generate_posters: Optional[bool] = None
|
|
|
|
|
mkdir: Optional[bool] = None
|
|
|
|
|
name: Optional[str] = None
|
|
|
|
|
url: Optional[str] = None
|
2022-01-13 09:29:58 -07:00
|
|
|
|
2022-04-08 09:57:04 -06:00
|
|
|
def __post_init__(self):
|
|
|
|
|
self.name = self.name.strip() or None
|
|
|
|
|
self.directory = self.directory.strip() or None
|
|
|
|
|
self.url = self.url.strip() if self.url else None
|
2024-06-25 22:12:13 -06:00
|
|
|
|
|
|
|
|
|
2024-08-06 18:13:21 -06:00
|
|
|
@dataclass
|
|
|
|
|
class ChannelTagRequest:
|
|
|
|
|
tag_name: Optional[str] = None
|
|
|
|
|
directory: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
def __post_init__(self):
|
|
|
|
|
if self.directory and self.directory.startswith('/'):
|
|
|
|
|
raise ValidationError('Directory must be relative to media directory')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class ChannelTagInfoRequest:
|
2024-08-15 19:32:01 -06:00
|
|
|
channel_id: Optional[int] = None
|
2024-08-06 18:13:21 -06:00
|
|
|
tag_name: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
|
2024-06-25 22:12:13 -06:00
|
|
|
@dataclass
|
|
|
|
|
class ChannelDownloadRequest:
|
|
|
|
|
url: str
|
|
|
|
|
frequency: int
|
|
|
|
|
settings: dict = field(default_factory=dict)
|
|
|
|
|
|
|
|
|
|
def __post_init__(self):
|
|
|
|
|
self.url = self.url.strip()
|
|
|
|
|
if not self.url:
|
|
|
|
|
raise ValidationError('url cannot be empty')
|
|
|
|
|
|
2026-05-31 12:43:56 -06:00
|
|
|
# Validate settings contents. Remove empty values.
|
2024-06-25 22:12:13 -06:00
|
|
|
from wrolpi.schema import DownloadSettings
|
|
|
|
|
self.settings = {k: v for k, v in DownloadSettings(**self.settings).__dict__.items() if v not in ([], None)}
|
2022-04-08 09:57:04 -06:00
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
|
|
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class RefreshRequest:
|
2022-01-13 09:29:58 -07:00
|
|
|
channel_name: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelModel:
|
2022-01-13 09:29:58 -07:00
|
|
|
id: int
|
|
|
|
|
url: str
|
|
|
|
|
name: str
|
|
|
|
|
directory: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelResponse:
|
|
|
|
|
channel = ChannelModel
|
|
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelPostResponse:
|
2022-01-13 09:29:58 -07:00
|
|
|
success: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelsChannelModel:
|
2022-01-13 09:29:58 -07:00
|
|
|
id: int
|
|
|
|
|
name: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelsResponse:
|
2022-01-13 09:29:58 -07:00
|
|
|
channels: List[ChannelsChannelModel]
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class StreamResponse:
|
2022-01-13 09:29:58 -07:00
|
|
|
success: str
|
|
|
|
|
stream_url: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class Video:
|
2022-01-13 09:29:58 -07:00
|
|
|
id: int
|
|
|
|
|
description_path: str
|
|
|
|
|
ext: str
|
|
|
|
|
poster_path: str
|
|
|
|
|
source_id: str
|
|
|
|
|
title: str
|
|
|
|
|
video_path: str
|
|
|
|
|
name: str
|
|
|
|
|
caption_path: str
|
|
|
|
|
info_json_path: str
|
|
|
|
|
channel_id: int
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class VideoWithChannel:
|
2022-01-13 09:29:58 -07:00
|
|
|
id: int
|
|
|
|
|
description_path: str
|
|
|
|
|
ext: str
|
|
|
|
|
poster_path: str
|
|
|
|
|
source_id: str
|
|
|
|
|
title: str
|
|
|
|
|
video_path: str
|
|
|
|
|
name: str
|
|
|
|
|
caption_path: str
|
|
|
|
|
info_json_path: str
|
|
|
|
|
channel_id: int
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
channel = ChannelModel
|
|
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ExtendedVideo(Video):
|
2022-01-13 09:29:58 -07:00
|
|
|
info_json: dict
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class VideoResponse:
|
2023-10-24 10:20:06 -06:00
|
|
|
file_group = ExtendedVideo
|
2022-08-06 05:22:07 +00:00
|
|
|
prev = ExtendedVideo
|
|
|
|
|
next = ExtendedVideo
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2025-01-26 12:44:16 -07:00
|
|
|
@dataclass
|
|
|
|
|
class VideoCommentsResponse:
|
|
|
|
|
comments: list[dict]
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class VideoCaptionsResponse:
|
2026-03-21 16:50:27 -06:00
|
|
|
captions: list
|
2025-01-26 12:44:16 -07:00
|
|
|
|
|
|
|
|
|
Fix API stalls: non-blocking switch worker, threaded searches, indexed Videos page (#678)
* Fix API stalls: non-blocking switch worker, threaded searches, indexed Videos page
Three response-time fixes found by benchmarking the read-only API endpoints
hot and cold on real libraries:
* `switch_worker` runs on the Sanic worker's event loop every 0.1s and called
the blocking `multiprocessing.Event.wait(timeout=1)`. With no switch pending
(the normal state) this froze the loop for a second at a time; about a third
of all requests, including trivial ones, took ~1,000 ms. It now polls
`is_set()` and returns immediately.
* The search handlers (videos, files, archives, docs, zims) and `/api/statistics`
ran synchronous SQLite queries on the event loop, so one slow search stalled
every other request on that worker. They now run via `asyncio.to_thread`.
* The unfiltered Videos page (`POST /api/videos/search` with only an order)
used `mimetype LIKE 'video/%' OR ...` plus `COUNT(*) OVER()`, which visits
every video's index entry before LIMIT applies: seconds cold on a large
library on a spinning disk, and ~10 s hot for a deep page. It now seeks each
exact mimetype on the (mimetype, effective_datetime) index for `limit + offset`
rows and merges them, with the total from a separate count cached for 60 s.
Results are identical to the generic query.
Also adds `scripts/api_perf.py` (times every read-only endpoint against a host,
optionally restarting the API over SSH for cold numbers) and
`scripts/api_perf_compare.py` (merges per-host results). Host profiles live in a
gitignored JSON file; an example is included.
Measured on an 8-core docker host with a 10 GB library: median hot request
1,007 ms -> 22 ms; Videos page 1 hot 287 -> 108 ms, page 50 1,345 -> 120 ms.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* Serve the video description from its own endpoint instead of Video.__json__
`Video.__json__` computed `description` as `c_text or get_video_description()`,
where the fallback opens and parses the video's info json from disk. The list
endpoints serialize every result with `__json__`, so any page containing videos
without an indexed description (common: many videos have none) read their info
json files on every request, cold, for a field the list never displays. The
description is not part of search either: search runs against the FTS copy of
`c_text`, so the fallback never fed the index.
Drop `description` from `__json__` and serve it from
`GET /api/videos/<id>/description`, like comments and captions. The endpoint
returns the indexed `c_text`, falling back to the info json only there, for a
row indexed before descriptions were stored. The player page fetches it through
`useVideoExtras` alongside the comments.
Tests assert the list and detail responses never touch the info json, and cover
the endpoint's indexed, fallback, and not-found cases.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* Address review: thread-local zim sessions, fast-path equivalence test, comments
* Zim search, single-zim search, and search estimates run libzim off the event
loop, each opening its own session inside the worker thread rather than
passing the request's SQLAlchemy Session across threads.
* The Videos fast-path test now runs the same requests through the generic
query and asserts identical rows and paging for both date orders. The one
intentional difference is documented: past the end, the fast path still
reports the true total.
* switch_worker returns before its try block so the empty path does not clear
the event; its comment states the invariant only. The copied "synchronous
SQLite" comments are reduced to one line in one place.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
2026-08-29 16:17:41 -06:00
|
|
|
@dataclass
|
|
|
|
|
class VideoDescriptionResponse:
|
|
|
|
|
description: Optional[str]
|
|
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class VideoSearchRequest:
|
2022-01-31 21:18:49 -07:00
|
|
|
search_str: Optional[str] = None
|
2023-03-24 08:58:18 -06:00
|
|
|
tags: List[str] = field(default_factory=list)
|
2022-01-13 09:29:58 -07:00
|
|
|
offset: Optional[int] = None
|
2022-01-31 21:18:49 -07:00
|
|
|
limit: Optional[int] = VIDEO_QUERY_LIMIT
|
|
|
|
|
order_by: Optional[str] = DEFAULT_VIDEO_ORDER
|
2022-04-08 09:57:04 -06:00
|
|
|
channel_id: Optional[int] = None
|
2023-03-24 08:58:18 -06:00
|
|
|
tag_names: List[str] = field(default_factory=list)
|
2023-04-12 18:51:26 -06:00
|
|
|
headline: bool = False
|
2026-06-27 09:48:21 -06:00
|
|
|
censored: bool = False
|
Search: fast ABC mode by default, opt-in deep search
The file_group.textsearch tsvector indexes all four text columns;
d_text (captions, document text, zip contents) is ~90% of the data,
making ts_rank searches take seconds on large libraries. Add a
textsearch_abc generated column (title/author/description only, ~8x
smaller) and default all searches to it, with deep=true opting into the
full column.
Database:
- Migration adds the generated column and a GIN index (rewrites the
table; a few minutes on large libraries).
Backend:
- FilesSearchRequest, VideoSearchRequest, ArchiveSearchRequest, and
DocSearchRequest accept `deep`; each search function picks the column.
- /api/search_file_estimates returns both counts in one index-bound
query (COUNT FILTER on the abc subset): `file_groups` (fast) and
`file_groups_deep`.
UI (one coherent story across the count surfaces):
- The typing dropdown always suggests the deep count (what is findable).
- The tab badge shows the active mode's count so it matches the result
list.
- The gap is reconciled by a hint offering deep search: count-aware on
the global search ("No results in fast search. N results available in
deep search." / "N more results available..."), generic on the module
pages (no deep estimate there). The hint resets pagination.
- "Search Depth" toggle in the search filter modals, persisted as
?deep=true.
Tests that relied on d_text matches now pass deep=True, with fast-mode
assertions alongside. Also skip flaky test_download_channel on
CircleCI.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-03 10:09:08 -06:00
|
|
|
deep: bool = False # When True, search all text including captions (d_text)
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
Jobs queue, video Edit dropdown: transcode/remux, edit title & description (#711)
* Jobs queue and video transcode from the video page
Add an in-memory FIFO Job queue (wrolpi/jobs.py): functions register by
name, callers enqueue by name with JSON kwargs, and the perpetual-tasks
owner runs them one at a time. Logging emitted inside a Job and the
output of its subprocesses are captured into a per-Job log; Jobs report
progress and can be cancelled (a running Job's subprocess is killed).
Jobs are not persisted; an API restart forgets them.
Add a transcode Job and POST /api/videos/<id>/transcode. ffmpeg reports
progress via -progress pipe:1. The existing FileGroup is pointed at the
new file before the refresh so tags and history survive a container
change.
Replace the video page Refresh button with an Edit dropdown (Refresh,
Transcode). The Transcode modal pre-fills from the preferred codecs in
the Videos settings and shows the Job's progress, log tail and Cancel.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: move Delete into the Edit dropdown
The Edit dropdown now holds Refresh, Transcode and Delete, so the action
row is Download / Edit / Add to Playlist. Delete keeps its confirmation
(a Confirm dialog) and the tagged-files prompt, and is disabled in WROL
Mode like the other items.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: Edit > Edit to change a video's title
PUT /api/videos/<file_group_id> writes the title (and fulltitle) into the
Video's info json, creating the file when the video has none, then
validates the Video again so its title and search text follow. Files
stay the source of truth: the edit survives a refresh, and a re-download
of the metadata replaces it (the modal says so).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: edit the description in Edit > Edit
PUT /api/videos/<file_group_id> now takes optional title and description
and changes only the fields given; an empty description clears it. The
modal sends only what changed and refetches the description after saving.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode modal: allow a container-only remux
Keeping both streams is no longer refused: ffmpeg copies them into the
chosen container with fast start. The container select starts at the
file's own container, the hint explains what a remux does, and the button
reads Remux when nothing is re-encoded.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* WROL Mode: refuse video edits and transcodes past the API
The routes already returned 403, but a transcode Job queued before WROL
Mode was enabled still ran, update_video could be called from anywhere,
and the Edit modal's Save button stayed live. The Job now fails before
ffmpeg starts, update_video carries wrol_mode_check, and the modal
disables Save with a notice.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: verify the output before replacing the original
After ffmpeg exits, the output is probed and compared with the source
(container, presence of video/audio streams, codec per stream, duration
within 1s or 1%) and its last 30 seconds are decoded to a null output.
Any failure removes the output and keeps the original.
The tail decode replaces the download-time completeness check, which
grabbed a single frame near the end: one ffmpeg_video_complete (sync for
the downloader, async for Jobs) now serves both, and is stricter, since
every frame of the tail plus the audio must decode.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Jobs: claim atomically, cap pending, own-session subprocesses; retarget before delete
Review fixes. The worker read a Job's status and wrote RUNNING as two
steps, so a cancel landing between them was overwritten and the Job ran;
_claim_job now does both under the lock. enqueue_job refuses past
JOB_PENDING_LIMIT pending Jobs. JobContext.run_command starts the
subprocess in its own session and raises CancelledError when the command
was killed, so a handler cannot mistake it for a normal exit.
The transcode Job now retargets the FileGroup (committed) while the
original file still exists, and deletes the original afterwards
(transcode_video_file gained keep_original). A crash in between leaves
both files with the database pointing at a real one.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: the Edit menu owns the transcode Job; polling survives Close and errors
Review fixes. The transcode Job's id and polling (useJob) move from the
modal to VideoEditMenu, so closing the modal no longer stops the page
from learning the Job finished (and refetching the video, whose path may
have changed). While a Job runs the menu shows Transcoding..., reopening
shows its progress instead of the form, and Delete waits. A failed poll
request is retried instead of ending the polling. getJobs checks the
response like its siblings. Design narration trimmed from comments.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: audio files, and Remove video to turn a video into an audio file
video_codec 'none' (REMOVE_VIDEO) drops the video stream (-vn, audio
required); an audio-only source is audio-only output whatever the video
target says, and a video target on it is refused. Audio output gets an
audio container (m4a/ogg/mp3) checked against the codec it will hold, and
a video container asked for audio output is replaced by the codec's
natural one. faststart only for mp4/m4a. Verification learned the
audio-only shape and ffprobe's container names (mkv reports matroska).
The modal offers 'Remove video (audio only)', switches to audio
containers and follows the audio codec, reads Extract Audio, and opens
for audio files (no video codec select).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: yellow Edit button
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: remove the temp output on cancel; tell the user when a closed-modal Job fails or is lost
Review fixes. The temp output was only removed on Exception, but a
cancelled Job's run_command raises CancelledError (a BaseException), so a
cancel left the partial file for a refresh to index. Closing the modal
during a Job that then failed showed nothing; it now toasts the error
(and a cancellation). Polling gives up on a 404 or five failures in a
row (the API restarted and forgot its Jobs), warning the user and
unlocking Transcode and Delete instead of reading Transcoding... forever.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Jobs: _claim_job docstring states the invariant, not the old race
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: fragmented mp4 option for very long videos
A whole-file mp4 index grows with the frame count; for a many-hour video
it reaches tens of MB, which a phone must fetch and parse before the
first frame, and every seek walks it. `fragmented` writes the file with
-movflags +frag_keyframe+empty_moov+default_base_moof+global_sidx and
10s fragments: a ~1 KB moov plus a global seek index, so playback starts
and seeks quickly. mp4/m4a only; a remux, so it takes seconds per GB.
Measured on an 11-hour h264/aac file: moov 40 MB -> 1.4 KB, seek index
2 x 120 KB, 10k fragments, duration and codecs unchanged, remux in 25s.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode modal: the Fragmented label says audio for audio output
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video edits keep the original: custom_title/custom_description in the info json's wrolpi section
Editing a title or description no longer overwrites yt-dlp's keys. The
values go into the info json's `wrolpi` section (custom_title,
custom_description); the modeler prefers them when deriving the Video, so
a refresh keeps the edit while the original title and description stay in
the file. An empty custom description means none.
Both re-download paths carry the wrolpi section over: the metadata-only
refresh merges it into the fresh info json, and a full re-download
snapshots it before yt-dlp overwrites the file and restores it (then
re-validates) in finalize_download. FileGroup.get_wrolpi_json reads it.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 19:25:23 -06:00
|
|
|
@dataclass
|
|
|
|
|
class VideoUpdateRequest:
|
|
|
|
|
# Only the fields given are changed.
|
|
|
|
|
title: Optional[str] = None
|
|
|
|
|
description: Optional[str] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class VideoTranscodeRequest:
|
2026-09-26 17:49:00 -06:00
|
|
|
video_codec: Optional[str] = None # A key of TRANSCODE_VIDEO_TARGETS, REMOVE_VIDEO, or None to copy the stream.
|
|
|
|
|
audio_codec: Optional[str] = None # A key of TRANSCODE_AUDIO_TARGETS, REMOVE_AUDIO, or None to copy the stream.
|
Jobs queue, video Edit dropdown: transcode/remux, edit title & description (#711)
* Jobs queue and video transcode from the video page
Add an in-memory FIFO Job queue (wrolpi/jobs.py): functions register by
name, callers enqueue by name with JSON kwargs, and the perpetual-tasks
owner runs them one at a time. Logging emitted inside a Job and the
output of its subprocesses are captured into a per-Job log; Jobs report
progress and can be cancelled (a running Job's subprocess is killed).
Jobs are not persisted; an API restart forgets them.
Add a transcode Job and POST /api/videos/<id>/transcode. ffmpeg reports
progress via -progress pipe:1. The existing FileGroup is pointed at the
new file before the refresh so tags and history survive a container
change.
Replace the video page Refresh button with an Edit dropdown (Refresh,
Transcode). The Transcode modal pre-fills from the preferred codecs in
the Videos settings and shows the Job's progress, log tail and Cancel.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: move Delete into the Edit dropdown
The Edit dropdown now holds Refresh, Transcode and Delete, so the action
row is Download / Edit / Add to Playlist. Delete keeps its confirmation
(a Confirm dialog) and the tagged-files prompt, and is disabled in WROL
Mode like the other items.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: Edit > Edit to change a video's title
PUT /api/videos/<file_group_id> writes the title (and fulltitle) into the
Video's info json, creating the file when the video has none, then
validates the Video again so its title and search text follow. Files
stay the source of truth: the edit survives a refresh, and a re-download
of the metadata replaces it (the modal says so).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: edit the description in Edit > Edit
PUT /api/videos/<file_group_id> now takes optional title and description
and changes only the fields given; an empty description clears it. The
modal sends only what changed and refetches the description after saving.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode modal: allow a container-only remux
Keeping both streams is no longer refused: ffmpeg copies them into the
chosen container with fast start. The container select starts at the
file's own container, the hint explains what a remux does, and the button
reads Remux when nothing is re-encoded.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* WROL Mode: refuse video edits and transcodes past the API
The routes already returned 403, but a transcode Job queued before WROL
Mode was enabled still ran, update_video could be called from anywhere,
and the Edit modal's Save button stayed live. The Job now fails before
ffmpeg starts, update_video carries wrol_mode_check, and the modal
disables Save with a notice.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: verify the output before replacing the original
After ffmpeg exits, the output is probed and compared with the source
(container, presence of video/audio streams, codec per stream, duration
within 1s or 1%) and its last 30 seconds are decoded to a null output.
Any failure removes the output and keeps the original.
The tail decode replaces the download-time completeness check, which
grabbed a single frame near the end: one ffmpeg_video_complete (sync for
the downloader, async for Jobs) now serves both, and is stricter, since
every frame of the tail plus the audio must decode.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Jobs: claim atomically, cap pending, own-session subprocesses; retarget before delete
Review fixes. The worker read a Job's status and wrote RUNNING as two
steps, so a cancel landing between them was overwritten and the Job ran;
_claim_job now does both under the lock. enqueue_job refuses past
JOB_PENDING_LIMIT pending Jobs. JobContext.run_command starts the
subprocess in its own session and raises CancelledError when the command
was killed, so a handler cannot mistake it for a normal exit.
The transcode Job now retargets the FileGroup (committed) while the
original file still exists, and deletes the original afterwards
(transcode_video_file gained keep_original). A crash in between leaves
both files with the database pointing at a real one.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: the Edit menu owns the transcode Job; polling survives Close and errors
Review fixes. The transcode Job's id and polling (useJob) move from the
modal to VideoEditMenu, so closing the modal no longer stops the page
from learning the Job finished (and refetching the video, whose path may
have changed). While a Job runs the menu shows Transcoding..., reopening
shows its progress instead of the form, and Delete waits. A failed poll
request is retried instead of ending the polling. getJobs checks the
response like its siblings. Design narration trimmed from comments.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: audio files, and Remove video to turn a video into an audio file
video_codec 'none' (REMOVE_VIDEO) drops the video stream (-vn, audio
required); an audio-only source is audio-only output whatever the video
target says, and a video target on it is refused. Audio output gets an
audio container (m4a/ogg/mp3) checked against the codec it will hold, and
a video container asked for audio output is replaced by the codec's
natural one. faststart only for mp4/m4a. Verification learned the
audio-only shape and ffprobe's container names (mkv reports matroska).
The modal offers 'Remove video (audio only)', switches to audio
containers and follows the audio codec, reads Extract Audio, and opens
for audio files (no video codec select).
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video page: yellow Edit button
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: remove the temp output on cancel; tell the user when a closed-modal Job fails or is lost
Review fixes. The temp output was only removed on Exception, but a
cancelled Job's run_command raises CancelledError (a BaseException), so a
cancel left the partial file for a refresh to index. Closing the modal
during a Job that then failed showed nothing; it now toasts the error
(and a cancellation). Polling gives up on a 404 or five failures in a
row (the API restarted and forgot its Jobs), warning the user and
unlocking Transcode and Delete instead of reading Transcoding... forever.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Jobs: _claim_job docstring states the invariant, not the old race
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode: fragmented mp4 option for very long videos
A whole-file mp4 index grows with the frame count; for a many-hour video
it reaches tens of MB, which a phone must fetch and parse before the
first frame, and every seek walks it. `fragmented` writes the file with
-movflags +frag_keyframe+empty_moov+default_base_moof+global_sidx and
10s fragments: a ~1 KB moov plus a global seek index, so playback starts
and seeks quickly. mp4/m4a only; a remux, so it takes seconds per GB.
Measured on an 11-hour h264/aac file: moov 40 MB -> 1.4 KB, seek index
2 x 120 KB, 10k fragments, duration and codecs unchanged, remux in 25s.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Transcode modal: the Fragmented label says audio for audio output
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
* Video edits keep the original: custom_title/custom_description in the info json's wrolpi section
Editing a title or description no longer overwrites yt-dlp's keys. The
values go into the info json's `wrolpi` section (custom_title,
custom_description); the modeler prefers them when deriving the Video, so
a refresh keeps the edit while the original title and description stay in
the file. An empty custom description means none.
Both re-download paths carry the wrolpi section over: the metadata-only
refresh merges it into the fresh info json, and a full re-download
snapshots it before yt-dlp overwrites the file and restores it (then
re-validates) in finalize_download. FileGroup.get_wrolpi_json reads it.
Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
2026-09-16 19:25:23 -06:00
|
|
|
container: str = 'mp4'
|
|
|
|
|
fragmented: bool = False # Fragmented mp4: quick start/seek for very long videos (mp4/m4a only).
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class VideoTranscodeResponse:
|
|
|
|
|
job_id: str
|
|
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class VideoSearchResponse:
|
2022-01-13 09:29:58 -07:00
|
|
|
videos: List[VideoWithChannel]
|
|
|
|
|
tsquery: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class PaginationQuery:
|
2022-01-13 09:29:58 -07:00
|
|
|
offset: int
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2023-03-24 08:58:18 -06:00
|
|
|
class TagRequest:
|
2022-01-13 09:29:58 -07:00
|
|
|
video_id: int
|
2023-03-24 08:58:18 -06:00
|
|
|
tag_name: str
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class WROLModeRequest:
|
2022-01-13 09:29:58 -07:00
|
|
|
enabled: bool
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class VideoStatistics:
|
2022-01-13 09:29:58 -07:00
|
|
|
videos: int
|
|
|
|
|
week: int
|
|
|
|
|
month: int
|
|
|
|
|
year: int
|
|
|
|
|
sum_duration: int
|
|
|
|
|
sum_size: int
|
|
|
|
|
max_size: int
|
2024-06-25 22:12:13 -06:00
|
|
|
have_comments: int
|
|
|
|
|
missing_comments: int
|
|
|
|
|
failed_comments: int
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class ChannelStatistics:
|
2022-01-13 09:29:58 -07:00
|
|
|
channels: int
|
2024-08-15 19:32:01 -06:00
|
|
|
tagged_channels: int
|
2021-11-01 16:42:52 -06:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-11-01 16:42:52 -06:00
|
|
|
class VideosStatisticsResponse:
|
|
|
|
|
videos = VideoStatistics
|
|
|
|
|
channels = ChannelStatistics
|
2021-12-23 09:38:12 -07:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-12-23 09:38:12 -07:00
|
|
|
class CensoredVideoRequest:
|
2022-01-13 09:29:58 -07:00
|
|
|
limit: int
|
|
|
|
|
offset: int
|
2021-12-23 09:38:12 -07:00
|
|
|
|
|
|
|
|
|
2022-01-13 09:29:58 -07:00
|
|
|
@dataclass
|
2021-12-23 09:38:12 -07:00
|
|
|
class CensoredVideoResponse:
|
2022-01-13 09:29:58 -07:00
|
|
|
videos: List[VideoWithChannel]
|
2024-08-15 19:32:01 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class ChannelSearchRequest:
|
|
|
|
|
tag_names: List[str] = field(default_factory=list)
|
2026-10-07 18:22:05 -06:00
|
|
|
search_str: Optional[str] = None
|
2024-11-29 17:01:16 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class VideoFileFormatRequest:
|
|
|
|
|
video_file_format: str
|
2026-02-20 14:37:58 -07:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class CookiesUploadRequest:
|
|
|
|
|
"""Request to upload and encrypt cookies."""
|
|
|
|
|
cookies_content: str
|
|
|
|
|
password: str
|
|
|
|
|
|
|
|
|
|
def __post_init__(self):
|
|
|
|
|
if not self.cookies_content:
|
|
|
|
|
raise ValidationError('cookies_content cannot be empty')
|
|
|
|
|
if not self.password or len(self.password) < 8:
|
|
|
|
|
raise ValidationError('password must be at least 8 characters')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class CookiesUnlockRequest:
|
|
|
|
|
"""Request to unlock cookies with password."""
|
|
|
|
|
password: str
|
|
|
|
|
|
|
|
|
|
def __post_init__(self):
|
|
|
|
|
if not self.password:
|
|
|
|
|
raise ValidationError('password cannot be empty')
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@dataclass
|
|
|
|
|
class CookiesStatusResponse:
|
|
|
|
|
"""Response with cookies status."""
|
|
|
|
|
cookies_exist: bool
|
|
|
|
|
cookies_unlocked: bool
|