mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
* videos: codec download preferences with transcode and strict-fail options
Add video_codecs/audio_codecs ordered preference lists to the video,
channel, and RSS downloaders, alongside the existing resolution
selector. Codec preference is codec-major: a preferred codec at a
lower resolution beats another codec at a higher resolution.
When 'transcode' is enabled, a downloaded file whose streams do not
match the preferences is re-encoded with ffmpeg after download (only
the mismatching stream; targets are extensible maps, currently h264
and aac/opus/mp3). When 'strict_codecs' is enabled and transcode is
not, a download with no matching format fails permanently instead of
falling back.
Empty codec lists produce a byte-identical yt-dlp command to before.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* videos: honor strict_codecs when transcode cannot produce the preference
Review fixes:
- strict_codecs is now suppressed only when transcoding can produce a
preferred codec for every non-empty preference list (new
transcode_can_satisfy_codecs); av1-only preferences with transcode
enabled fail under strict instead of silently keeping the file.
- No partial transcode: an unfixable video mismatch blocks an
audio-only conversion (and vice versa); the mismatch falls through
to strict/keep.
- fsync the transcode output before rename, and the directory after,
so power loss on a Pi cannot leave a truncated final file.
- mp3 yt-dlp pattern matches mp4a.6B as well as mp4a.6b.
- ffmpeg maps use 0:V:0 (excludes attached pictures) and 0🅰️0?
(tolerates missing audio).
- Strict toggle is no longer disabled by transcode in the UI; popup
wording corrected.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* videos: serialize transcodes machine-wide via shared_ctx lock
Sanic runs multiple worker processes, so an asyncio primitive cannot
serialize transcodes across download workers. A multiprocessing.Lock
in shared_ctx now guarantees only one ffmpeg transcode runs at a time
on the machine; waiting workers poll the lock asynchronously so the
event loop stays responsive while a transcode holds it for hours.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
* videos: transcode-lock review fixes
- Document that a crashed lock holder requires an API restart; this is
accepted rather than adding owner-death recovery.
- Move the 'Transcoding' user notification to after the lock is
acquired, so it fires when work begins rather than before a
potentially hours-long wait.
- Soften the lock test docstring to what it measures (in-process
wait/release); trim the acquire docstring to the polling rationale.
- Give transcode_lock its own section header in contexts.py.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
506 lines
15 KiB
Python
506 lines
15 KiB
Python
import enum
|
|
import pathlib
|
|
from dataclasses import dataclass, field
|
|
from typing import Optional, List
|
|
|
|
from wrolpi.common import get_relative_to_media_directory, unique_by_predicate
|
|
from wrolpi.errors import InvalidDownload, ValidationError
|
|
|
|
|
|
@dataclass
|
|
class SettingsObject:
|
|
media_directory: str
|
|
|
|
|
|
@dataclass
|
|
class SettingsResponse:
|
|
download_manager_disabled: bool
|
|
download_manager_stopped: bool
|
|
download_daily_limit_global: Optional[int]
|
|
download_daily_limit_per_domain: Optional[int]
|
|
download_on_startup: bool
|
|
download_timeout: int
|
|
download_wait: int
|
|
download_window_start: Optional[str]
|
|
download_window_end: Optional[str]
|
|
hotspot_device: str
|
|
hotspot_on_startup: bool
|
|
hotspot_password: str
|
|
hotspot_ssid: str
|
|
# `hotspot_status` and `throttle_status` were declared here long after the handler stopped
|
|
# returning them -- both moved to the Controller (/controller/api/hotspot/status and
|
|
# /controller/api/throttle/status). This is what /docs publishes, so a field it names and
|
|
# the endpoint never sends is a promise to a caller that cannot be kept;
|
|
# test/test_api_fixtures.py now holds the two together.
|
|
check_for_upgrades: bool
|
|
ignore_outdated_zims: bool
|
|
ignored_directories: List[str]
|
|
# A Python logging integer (INFO is 20), not a name. Declared `str` here while
|
|
# SettingsRequest.log_level already said int, and the frontend converts it numerically --
|
|
# so the fixture was carrying 'info', which fromApiLogLevel() maps to undefined.
|
|
log_level: int
|
|
map_default_location: Optional[dict]
|
|
nav_color: str
|
|
media_directory: str
|
|
require_cookies_unlocked: bool
|
|
require_media_mounted: bool
|
|
save_ffprobe_json: bool
|
|
tags_directory: bool
|
|
throttle_on_startup: bool
|
|
timezone: Optional[str]
|
|
version: str
|
|
wrol_mode: bool
|
|
archive_destination: str = 'archive/%(domain_tag)s/%(domain)s'
|
|
map_destination: str = 'map'
|
|
playlists_destination: str = 'playlists'
|
|
videos_destination: str = 'videos/%(channel_tag)s/%(channel_name)s'
|
|
zims_destination: str = 'zims'
|
|
|
|
|
|
class NavColors(enum.StrEnum):
|
|
"""The colors a user may choose for the navigation bar.
|
|
|
|
The same twelve as `navColorHexMap` in app/src/components/Vars.js, which holds the hex
|
|
each one is drawn as, and as `navColorNames` in app/src/themes/navColors.ts. A color
|
|
not on this list has no favicon file to go with it.
|
|
"""
|
|
red = enum.auto()
|
|
orange = enum.auto()
|
|
yellow = enum.auto()
|
|
olive = enum.auto()
|
|
green = enum.auto()
|
|
teal = enum.auto()
|
|
blue = enum.auto()
|
|
violet = enum.auto()
|
|
purple = enum.auto()
|
|
pink = enum.auto()
|
|
brown = enum.auto()
|
|
grey = enum.auto()
|
|
|
|
|
|
@dataclass
|
|
class SettingsRequest:
|
|
archive_destination: Optional[str] = None
|
|
download_daily_limit_global: Optional[int] = None
|
|
download_daily_limit_per_domain: Optional[int] = None
|
|
download_on_startup: Optional[bool] = None
|
|
download_timeout: Optional[int] = None
|
|
download_wait: Optional[int] = None
|
|
download_window_start: Optional[str] = None
|
|
download_window_end: Optional[str] = None
|
|
hotspot_device: Optional[str] = None
|
|
hotspot_on_startup: Optional[bool] = None
|
|
hotspot_password: Optional[str] = None
|
|
hotspot_ssid: Optional[str] = None
|
|
hotspot_status: Optional[bool] = None
|
|
check_for_upgrades: Optional[bool] = None
|
|
ignore_outdated_zims: Optional[bool] = None
|
|
log_level: Optional[int] = None
|
|
map_destination: Optional[str] = None
|
|
nav_color: Optional[str] = None
|
|
media_directory: Optional[str] = None
|
|
playlists_destination: Optional[str] = None
|
|
tags_directory: Optional[bool] = None
|
|
throttle_on: Optional[bool] = None
|
|
throttle_on_startup: Optional[bool] = None
|
|
videos_destination: Optional[str] = None
|
|
wrol_mode: Optional[bool] = None
|
|
zims_destination: Optional[str] = None
|
|
require_cookies_unlocked: Optional[bool] = None
|
|
require_media_mounted: Optional[bool] = None
|
|
save_ffprobe_json: Optional[bool] = None
|
|
timezone: Optional[str] = None
|
|
|
|
def __post_init__(self):
|
|
self.nav_color = self.nav_color.lower() if self.nav_color else None
|
|
if self.nav_color and self.nav_color not in NavColors.__members__:
|
|
raise ValidationError('Nav color is invalid')
|
|
|
|
|
|
@dataclass
|
|
class FlagsStatusResponse:
|
|
db_up: bool
|
|
file_worker_busy: bool
|
|
file_worker_cleanup: bool
|
|
file_worker_counting: bool
|
|
file_worker_discovery: bool
|
|
file_worker_indexing: bool
|
|
file_worker_modeling: bool
|
|
global_refresh_active: bool
|
|
have_internet: bool
|
|
media_mounted: bool
|
|
outdated_zims: bool
|
|
refresh_complete: bool
|
|
|
|
|
|
@dataclass
|
|
class CPUStatusResponse:
|
|
cores: int
|
|
cur_frequency: int
|
|
max_frequency: int
|
|
min_frequency: int
|
|
percent: int
|
|
temperature: int
|
|
high_temperature: int
|
|
critical_temperature: int
|
|
|
|
|
|
@dataclass
|
|
class LoadStatusStat:
|
|
minute_1: int
|
|
minute_5: int
|
|
minute_15: int
|
|
|
|
|
|
@dataclass
|
|
class DriveStatusStat:
|
|
mount: str
|
|
percent: int
|
|
size: int
|
|
used: int
|
|
|
|
|
|
@dataclass
|
|
class ProcessStatusStat:
|
|
pid: int
|
|
percent_cpu: int
|
|
percent_mem: int
|
|
command: str
|
|
|
|
|
|
@dataclass
|
|
class MemoryStatusStat:
|
|
total: int
|
|
used: int
|
|
free: int
|
|
cached: int
|
|
|
|
|
|
@dataclass
|
|
class NicBandwidthStatusStat:
|
|
name: str
|
|
now: float
|
|
bytes_recv: int
|
|
bytes_sent: int
|
|
speed: int
|
|
bytes_recv_ps: int
|
|
bytes_sent_ps: int
|
|
elapsed: int
|
|
|
|
|
|
@dataclass
|
|
class DiskBandwidthStatusStat:
|
|
bytes_read: int
|
|
bytes_read_ps: int
|
|
bytes_write: int
|
|
bytes_write_ps: int
|
|
elapsed: int
|
|
max_read_ps: int
|
|
max_write_ps: int
|
|
name: str
|
|
now: float
|
|
speed: int
|
|
|
|
|
|
@dataclass
|
|
class DownloadsSummaryResponse:
|
|
"""`DownloadManager.get_summary()`, which /api/status returns as `downloads`.
|
|
|
|
It is an empty object while the database is down.
|
|
"""
|
|
pending: int
|
|
recurring: int
|
|
disabled: bool
|
|
stopped: bool
|
|
outside_download_window: bool
|
|
daily_limit_reached: bool
|
|
|
|
|
|
@dataclass
|
|
class StatusResponse:
|
|
"""The response of GET /api/status.
|
|
|
|
Half of this is assembled from `shared_ctx.status`, which the status worker fills in. Its
|
|
keys are not open-ended: `initialize_configs_contexts` seeds exactly the stats and upgrade
|
|
keys below and the worker only ever updates those, which is what makes this declarable at
|
|
all. Add a key to that seed and it belongs here too -- test/test_api_fixtures.py compares
|
|
the two.
|
|
|
|
`hotspot_ssid`, `hotspot_status`, `throttle_status`, `last_status`, `drive_status`,
|
|
`disk_bandwidth_stats` and `nic_bandwidth_stats` were declared here after the handler had
|
|
stopped returning them; hotspot and throttle moved to the Controller.
|
|
"""
|
|
cpu_stats: CPUStatusResponse
|
|
dockerized: bool
|
|
# Declared `int` for a long time while the handler sent the summary object, so the published
|
|
# contract promised a generated client a number where it gets a dict.
|
|
downloads: DownloadsSummaryResponse
|
|
drives_stats: List[DriveStatusStat]
|
|
flags: FlagsStatusResponse
|
|
is_rpi4: bool
|
|
is_rpi5: bool
|
|
is_rpi: bool
|
|
load_stats: LoadStatusStat
|
|
local_time: str
|
|
memory_stats: MemoryStatusStat
|
|
processes_stats: List[ProcessStatusStat]
|
|
sanic_workers: dict
|
|
version: str
|
|
# `str` here was wrong too; wrol_mode_enabled() returns a bool.
|
|
wrol_mode: bool
|
|
# Upgrade info, seeded in contexts.py and refreshed by the upgrade check.
|
|
commits_behind: int
|
|
current_commit: Optional[str]
|
|
git_branch: Optional[str]
|
|
latest_commit: Optional[str]
|
|
update_available: bool
|
|
|
|
|
|
@dataclass
|
|
class RegexRequest:
|
|
regex: str
|
|
|
|
|
|
@dataclass
|
|
class RegexResponse:
|
|
regex: str
|
|
valid: bool
|
|
|
|
|
|
@dataclass
|
|
class EchoResponse:
|
|
form: dict
|
|
headers: dict
|
|
json: str
|
|
method: str
|
|
|
|
|
|
def _coerce_optional_number(value, kind=float):
|
|
"""Coerce a value (possibly str from forms) to int/float or None.
|
|
Empty strings and unparsable values become None (and are filtered by callers).
|
|
"""
|
|
if value in (None, ''):
|
|
return None
|
|
try:
|
|
return kind(value)
|
|
except (ValueError, TypeError):
|
|
return None
|
|
|
|
|
|
@dataclass
|
|
class DownloadSettings:
|
|
channel_id: Optional[int] = None
|
|
channel_tag_name: List[str] = field(default_factory=list)
|
|
channel_url: Optional[str] = None
|
|
collection_id: Optional[int] = None
|
|
depth: Optional[int] = None
|
|
destination: Optional[str] = None # Legacy support
|
|
download_metadata_only: bool = False
|
|
download_order: Optional[str] = None
|
|
excluded_urls: Optional[str] = None
|
|
max_pages: Optional[int] = None
|
|
maximum_duration: Optional[int] = None
|
|
minimum_duration: Optional[int] = None
|
|
suffix: Optional[str] = None
|
|
tag_names: Optional[List[str]] = None # Legacy support
|
|
title_exclude: Optional[str] = None
|
|
title_include: Optional[str] = None
|
|
video_count_limit: Optional[int] = None
|
|
video_format: Optional[str] = None
|
|
video_resolutions: List[str] = field(default_factory=list)
|
|
parent_download_url: Optional[str] = None
|
|
skip_already_downloaded: Optional[bool] = None
|
|
compress_singlefile: Optional[bool] = None
|
|
# Inheritable video settings (Optional so they're only set when user overrides)
|
|
writesubtitles: Optional[bool] = None
|
|
writeautomaticsub: Optional[bool] = None
|
|
writethumbnail: Optional[bool] = None
|
|
writeinfojson: Optional[bool] = None
|
|
yt_dlp_extra_args: Optional[str] = None
|
|
sleep_requests: Optional[float] = None
|
|
user_agent: Optional[str] = None
|
|
continue_dl: Optional[bool] = None
|
|
nooverwrites: Optional[bool] = None
|
|
audio_only: Optional[bool] = None
|
|
audio_format: Optional[str] = None
|
|
video_codecs: Optional[List[str]] = None
|
|
audio_codecs: Optional[List[str]] = None
|
|
transcode: Optional[bool] = None
|
|
strict_codecs: Optional[bool] = None
|
|
|
|
def __post_init__(self):
|
|
if self.excluded_urls and self.excluded_urls.endswith(','):
|
|
raise ValidationError('Excluded urls cannot end with ,')
|
|
if self.excluded_urls and self.excluded_urls.startswith(','):
|
|
raise ValidationError('Excluded urls cannot start with ,')
|
|
|
|
if self.suffix and not self.suffix.startswith('.'):
|
|
raise ValidationError('suffix must start with .')
|
|
|
|
# Coerce numeric settings that may arrive as strings from forms, old YAML,
|
|
# or direct API calls. Empty/bad values become None and are filtered later.
|
|
self.depth = _coerce_optional_number(self.depth, int)
|
|
self.max_pages = _coerce_optional_number(self.max_pages, int)
|
|
self.maximum_duration = _coerce_optional_number(self.maximum_duration, int)
|
|
self.minimum_duration = _coerce_optional_number(self.minimum_duration, int)
|
|
self.video_count_limit = _coerce_optional_number(self.video_count_limit, int)
|
|
self.sleep_requests = _coerce_optional_number(self.sleep_requests, float)
|
|
|
|
self.title_exclude = self.title_exclude or None
|
|
self.title_include = self.title_include or None
|
|
if not self.download_metadata_only:
|
|
del self.download_metadata_only
|
|
|
|
if self.download_order not in (None, 'newest', 'oldest', 'views'):
|
|
raise ValidationError(f'Download order must be one of newest, oldest, views, or null.')
|
|
|
|
if self.video_format not in (None, 'mp4', 'mkv', 'ogg', 'webm'):
|
|
raise ValidationError(f'Video format must be one of mp4, mkv, ogg, webm, or null.')
|
|
|
|
if self.audio_format not in (None, 'mp3', 'opus', 'm4a', 'flac', 'vorbis', 'wav'):
|
|
raise ValidationError(f'Audio format must be one of mp3, opus, m4a, flac, vorbis, wav, or null.')
|
|
|
|
valid_resolutions = ('360p', '480p', '720p', '1080p', '1440p', '2160p', 'maximum')
|
|
if not all(i in valid_resolutions for i in self.video_resolutions):
|
|
raise ValidationError(f'Download order must be one of 360p, 480p, 720p, 1080p, 1440p, 2160p, or maximum.')
|
|
|
|
valid_video_codecs = ('h264', 'vp9', 'av1', 'vp8', 'hevc')
|
|
if not all(i in valid_video_codecs for i in self.video_codecs or []):
|
|
raise ValidationError(f'Video codecs must be of: {", ".join(valid_video_codecs)}')
|
|
|
|
valid_audio_codecs = ('aac', 'opus', 'mp3', 'vorbis')
|
|
if not all(i in valid_audio_codecs for i in self.audio_codecs or []):
|
|
raise ValidationError(f'Audio codecs must be of: {", ".join(valid_audio_codecs)}')
|
|
|
|
|
|
@dataclass
|
|
class DownloadRequest:
|
|
urls: List[str]
|
|
downloader: str
|
|
destination: Optional[str] = None
|
|
tag_names: Optional[List[str]] = field(default_factory=list)
|
|
frequency: Optional[int] = None
|
|
sub_downloader: Optional[str] = None
|
|
settings: Optional[dict] = field(default_factory=dict)
|
|
collection_id: Optional[int] = None
|
|
|
|
def __post_init__(self):
|
|
urls = [j for i in self.urls if (j := i.strip())]
|
|
if not urls:
|
|
raise InvalidDownload(f'urls cannot be empty')
|
|
# Get unique URLs, preserve order.
|
|
self.urls = unique_by_predicate(urls)
|
|
|
|
if self.destination:
|
|
destination = pathlib.Path(self.destination)
|
|
if destination.is_absolute():
|
|
destination = get_relative_to_media_directory(destination)
|
|
self.destination = str(destination)
|
|
|
|
self.tag_names = self.tag_names or []
|
|
|
|
# Validate settings contents. Remove empty values.
|
|
settings = self.settings or dict()
|
|
self.settings = {k: v for k, v in DownloadSettings(**settings).__dict__.items() if v not in ([], None)}
|
|
|
|
|
|
@dataclass
|
|
class DownloadBatchRequest:
|
|
"""Request for batch operations on downloads."""
|
|
download_ids: List[int]
|
|
|
|
def __post_init__(self):
|
|
if not self.download_ids:
|
|
raise ValidationError('download_ids cannot be empty')
|
|
|
|
|
|
@dataclass
|
|
class JSONErrorResponse:
|
|
error: str
|
|
|
|
|
|
@dataclass
|
|
class EventsRequest:
|
|
after: Optional[str] = None
|
|
|
|
|
|
@dataclass
|
|
class TagRequest:
|
|
name: str
|
|
color: str
|
|
|
|
|
|
@dataclass
|
|
class DeleteTagRequest:
|
|
name: str
|
|
|
|
|
|
@dataclass
|
|
class NotifyRequest:
|
|
message: str
|
|
url: str = None
|
|
|
|
|
|
@dataclass
|
|
class VINDecoderRequest:
|
|
vin_number: str
|
|
|
|
|
|
@dataclass
|
|
class VIN:
|
|
country: str
|
|
manufacturer: str
|
|
region: str
|
|
years: str
|
|
body: str = None
|
|
engine: str = None
|
|
model: str = None
|
|
plant: str = None
|
|
transmission: str = None
|
|
serial: str = None
|
|
|
|
def __json__(self) -> dict:
|
|
d = dict(
|
|
country=self.country,
|
|
manufacturer=self.manufacturer,
|
|
region=self.region,
|
|
years=self.years,
|
|
body=self.body,
|
|
engine=self.engine,
|
|
model=self.model,
|
|
plant=self.plant,
|
|
transmission=self.transmission,
|
|
serial=self.serial,
|
|
)
|
|
return d
|
|
|
|
|
|
@dataclass
|
|
class VINDecoderResponse:
|
|
vin: VIN
|
|
|
|
|
|
@dataclass
|
|
class SearchSuggestionsRequest:
|
|
search_str: Optional[str] = None
|
|
order_by_video_count: bool = True
|
|
|
|
|
|
@dataclass
|
|
class SearchFileEstimateRequest:
|
|
search_str: Optional[str] = None
|
|
mimetypes: List[str] = field(default_factory=list)
|
|
tag_names: List[str] = field(default_factory=list)
|
|
months: Optional[List[int]] = None
|
|
from_year: Optional[int] = None
|
|
to_year: Optional[int] = None
|
|
any_tag: bool = False
|
|
|
|
def __post_init__(self):
|
|
if self.any_tag and self.tag_names:
|
|
raise ValidationError('Cannot search for any tag, and list of tags.')
|
|
|
|
|
|
@dataclass
|
|
class SearchOtherEstimateRequest:
|
|
tag_names: List[str] = field(default_factory=list)
|