wrolpi/wrolpi/schema.py
lrnselfreliance 93d878952c
Add a Repos module that mirrors git repositories (#783)
Repos keeps local mirrors of git repositories, cloned over https and kept up to date on a schedule.

- A Repo is a Collection (kind='repo') plus a `repository` row.  The new `git` downloader clones it, then updates it
  (weekly by default, or Never: the clone is kept as it is and only updated on request).
- Repos are saved in `repos/<tag>/<name>` (the Repos Directory, ignored by default) or a directory the user chooses.
  An existing clone can be imported where it is.  repos.yaml is the source of truth.
- A GitHub-like page shows each repo's README, files, and history, with a ZIP download.  READMEs are searchable
  under the global search's Other tab.
- git is hardened against hostile repos and configs: https only, pinned addresses with no private addresses (for the
  repo and its submodules), sanitized configs, no hooks or links, and only clones WROLPi made or imported are
  updated.
- Tagging a Repo moves its clone as one directory.  Repo changes wait for repos.yaml to be imported at startup.
- The file mover groups unindexed files with no primary file as one FileGroup, and never replaces a file at the
  destination.  A failed tag move restores the Collection's tag.  A Collection's directory cannot leave the media
  directory.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-07 15:16:58 -06:00

509 lines
15 KiB
Python

import enum
import pathlib
from dataclasses import dataclass, field
from typing import Optional, List
from wrolpi.common import get_relative_to_media_directory, unique_by_predicate
from wrolpi.errors import InvalidDownload, ValidationError
@dataclass
class SettingsObject:
media_directory: str
@dataclass
class SettingsResponse:
download_manager_disabled: bool
download_manager_stopped: bool
download_daily_limit_global: Optional[int]
download_daily_limit_per_domain: Optional[int]
download_on_startup: bool
download_timeout: int
download_wait: int
download_window_start: Optional[str]
download_window_end: Optional[str]
hotspot_device: str
hotspot_on_startup: bool
hotspot_password: str
hotspot_ssid: str
# `hotspot_status` and `throttle_status` were declared here long after the handler stopped
# returning them -- both moved to the Controller (/controller/api/hotspot/status and
# /controller/api/throttle/status). This is what /docs publishes, so a field it names and
# the endpoint never sends is a promise to a caller that cannot be kept;
# test/test_api_fixtures.py now holds the two together.
check_for_upgrades: bool
ignore_outdated_zims: bool
ignored_directories: List[str]
# A Python logging integer (INFO is 20), not a name. Declared `str` here while
# SettingsRequest.log_level already said int, and the frontend converts it numerically --
# so the fixture was carrying 'info', which fromApiLogLevel() maps to undefined.
log_level: int
map_default_location: Optional[dict]
nav_color: str
media_directory: str
require_cookies_unlocked: bool
require_media_mounted: bool
save_ffprobe_json: bool
tags_directory: bool
throttle_on_startup: bool
timezone: Optional[str]
version: str
wrol_mode: bool
archive_destination: str = 'archive/%(domain_tag)s/%(domain)s'
map_destination: str = 'map'
playlists_destination: str = 'playlists'
repos_destination: str = 'repos/%(repo_tag)s/%(repo_name)s'
videos_destination: str = 'videos/%(channel_tag)s/%(channel_name)s'
zims_destination: str = 'zims'
class NavColors(enum.StrEnum):
"""The colors a user may choose for the navigation bar.
The same twelve as `navColorHexMap` in app/src/components/Vars.js, which holds the hex
each one is drawn as, and as `navColorNames` in app/src/themes/navColors.ts. A color
not on this list has no favicon file to go with it.
"""
red = enum.auto()
orange = enum.auto()
yellow = enum.auto()
olive = enum.auto()
green = enum.auto()
teal = enum.auto()
blue = enum.auto()
violet = enum.auto()
purple = enum.auto()
pink = enum.auto()
brown = enum.auto()
grey = enum.auto()
@dataclass
class SettingsRequest:
archive_destination: Optional[str] = None
download_daily_limit_global: Optional[int] = None
download_daily_limit_per_domain: Optional[int] = None
download_on_startup: Optional[bool] = None
download_timeout: Optional[int] = None
download_wait: Optional[int] = None
download_window_start: Optional[str] = None
download_window_end: Optional[str] = None
hotspot_device: Optional[str] = None
hotspot_on_startup: Optional[bool] = None
hotspot_password: Optional[str] = None
hotspot_ssid: Optional[str] = None
hotspot_status: Optional[bool] = None
check_for_upgrades: Optional[bool] = None
ignore_outdated_zims: Optional[bool] = None
log_level: Optional[int] = None
map_destination: Optional[str] = None
nav_color: Optional[str] = None
media_directory: Optional[str] = None
playlists_destination: Optional[str] = None
repos_destination: Optional[str] = None
tags_directory: Optional[bool] = None
throttle_on: Optional[bool] = None
throttle_on_startup: Optional[bool] = None
videos_destination: Optional[str] = None
wrol_mode: Optional[bool] = None
zims_destination: Optional[str] = None
require_cookies_unlocked: Optional[bool] = None
require_media_mounted: Optional[bool] = None
save_ffprobe_json: Optional[bool] = None
timezone: Optional[str] = None
def __post_init__(self):
self.nav_color = self.nav_color.lower() if self.nav_color else None
if self.nav_color and self.nav_color not in NavColors.__members__:
raise ValidationError('Nav color is invalid')
@dataclass
class FlagsStatusResponse:
db_up: bool
file_worker_busy: bool
file_worker_cleanup: bool
file_worker_counting: bool
file_worker_discovery: bool
file_worker_indexing: bool
file_worker_modeling: bool
global_refresh_active: bool
have_internet: bool
media_mounted: bool
outdated_zims: bool
refresh_complete: bool
@dataclass
class CPUStatusResponse:
cores: int
cur_frequency: int
max_frequency: int
min_frequency: int
percent: int
temperature: int
high_temperature: int
critical_temperature: int
@dataclass
class LoadStatusStat:
minute_1: int
minute_5: int
minute_15: int
@dataclass
class DriveStatusStat:
mount: str
percent: int
size: int
used: int
@dataclass
class ProcessStatusStat:
pid: int
percent_cpu: int
percent_mem: int
command: str
@dataclass
class MemoryStatusStat:
total: int
used: int
free: int
cached: int
@dataclass
class NicBandwidthStatusStat:
name: str
now: float
bytes_recv: int
bytes_sent: int
speed: int
bytes_recv_ps: int
bytes_sent_ps: int
elapsed: int
@dataclass
class DiskBandwidthStatusStat:
bytes_read: int
bytes_read_ps: int
bytes_write: int
bytes_write_ps: int
elapsed: int
max_read_ps: int
max_write_ps: int
name: str
now: float
speed: int
@dataclass
class DownloadsSummaryResponse:
"""`DownloadManager.get_summary()`, which /api/status returns as `downloads`.
It is an empty object while the database is down.
"""
pending: int
recurring: int
disabled: bool
stopped: bool
outside_download_window: bool
daily_limit_reached: bool
@dataclass
class StatusResponse:
"""The response of GET /api/status.
Half of this is assembled from `shared_ctx.status`, which the status worker fills in. Its
keys are not open-ended: `initialize_configs_contexts` seeds exactly the stats and upgrade
keys below and the worker only ever updates those, which is what makes this declarable at
all. Add a key to that seed and it belongs here too -- test/test_api_fixtures.py compares
the two.
`hotspot_ssid`, `hotspot_status`, `throttle_status`, `last_status`, `drive_status`,
`disk_bandwidth_stats` and `nic_bandwidth_stats` were declared here after the handler had
stopped returning them; hotspot and throttle moved to the Controller.
"""
cpu_stats: CPUStatusResponse
dockerized: bool
# Declared `int` for a long time while the handler sent the summary object, so the published
# contract promised a generated client a number where it gets a dict.
downloads: DownloadsSummaryResponse
drives_stats: List[DriveStatusStat]
flags: FlagsStatusResponse
is_rpi4: bool
is_rpi5: bool
is_rpi: bool
load_stats: LoadStatusStat
local_time: str
memory_stats: MemoryStatusStat
processes_stats: List[ProcessStatusStat]
sanic_workers: dict
version: str
# `str` here was wrong too; wrol_mode_enabled() returns a bool.
wrol_mode: bool
# Upgrade info, seeded in contexts.py and refreshed by the upgrade check.
commits_behind: int
current_commit: Optional[str]
git_branch: Optional[str]
latest_commit: Optional[str]
update_available: bool
@dataclass
class RegexRequest:
regex: str
@dataclass
class RegexResponse:
regex: str
valid: bool
@dataclass
class EchoResponse:
form: dict
headers: dict
json: str
method: str
def _coerce_optional_number(value, kind=float):
"""Coerce a value (possibly str from forms) to int/float or None.
Empty strings and unparsable values become None (and are filtered by callers).
"""
if value in (None, ''):
return None
try:
return kind(value)
except (ValueError, TypeError):
return None
@dataclass
class DownloadSettings:
channel_id: Optional[int] = None
channel_tag_name: List[str] = field(default_factory=list)
channel_url: Optional[str] = None
collection_id: Optional[int] = None
depth: Optional[int] = None
destination: Optional[str] = None # Legacy support
download_metadata_only: bool = False
download_order: Optional[str] = None
excluded_urls: Optional[str] = None
max_pages: Optional[int] = None
maximum_duration: Optional[int] = None
minimum_duration: Optional[int] = None
suffix: Optional[str] = None
tag_names: Optional[List[str]] = None # Legacy support
title_exclude: Optional[str] = None
title_include: Optional[str] = None
video_count_limit: Optional[int] = None
video_format: Optional[str] = None
video_resolutions: List[str] = field(default_factory=list)
parent_download_url: Optional[str] = None
skip_already_downloaded: Optional[bool] = None
compress_singlefile: Optional[bool] = None
# Inheritable video settings (Optional so they're only set when user overrides)
writesubtitles: Optional[bool] = None
writeautomaticsub: Optional[bool] = None
writethumbnail: Optional[bool] = None
writeinfojson: Optional[bool] = None
yt_dlp_extra_args: Optional[str] = None
sleep_requests: Optional[float] = None
user_agent: Optional[str] = None
continue_dl: Optional[bool] = None
nooverwrites: Optional[bool] = None
audio_only: Optional[bool] = None
audio_format: Optional[str] = None
video_codecs: Optional[List[str]] = None
audio_codecs: Optional[List[str]] = None
transcode: Optional[bool] = None
strict_codecs: Optional[bool] = None
def __post_init__(self):
if self.excluded_urls and self.excluded_urls.endswith(','):
raise ValidationError('Excluded urls cannot end with ,')
if self.excluded_urls and self.excluded_urls.startswith(','):
raise ValidationError('Excluded urls cannot start with ,')
if self.suffix and not self.suffix.startswith('.'):
raise ValidationError('suffix must start with .')
# Coerce numeric settings that may arrive as strings from forms, old YAML,
# or direct API calls. Empty/bad values become None and are filtered later.
self.depth = _coerce_optional_number(self.depth, int)
self.max_pages = _coerce_optional_number(self.max_pages, int)
self.maximum_duration = _coerce_optional_number(self.maximum_duration, int)
self.minimum_duration = _coerce_optional_number(self.minimum_duration, int)
self.video_count_limit = _coerce_optional_number(self.video_count_limit, int)
self.sleep_requests = _coerce_optional_number(self.sleep_requests, float)
self.title_exclude = self.title_exclude or None
self.title_include = self.title_include or None
if not self.download_metadata_only:
del self.download_metadata_only
if self.download_order not in (None, 'newest', 'oldest', 'views'):
raise ValidationError(f'Download order must be one of newest, oldest, views, or null.')
if self.video_format not in (None, 'mp4', 'mkv', 'ogg', 'webm'):
raise ValidationError(f'Video format must be one of mp4, mkv, ogg, webm, or null.')
if self.audio_format not in (None, 'mp3', 'opus', 'm4a', 'flac', 'vorbis', 'wav'):
raise ValidationError(f'Audio format must be one of mp3, opus, m4a, flac, vorbis, wav, or null.')
valid_resolutions = ('360p', '480p', '720p', '1080p', '1440p', '2160p', 'maximum')
if not all(i in valid_resolutions for i in self.video_resolutions):
raise ValidationError(f'Download order must be one of 360p, 480p, 720p, 1080p, 1440p, 2160p, or maximum.')
valid_video_codecs = ('h264', 'vp9', 'av1', 'vp8', 'hevc')
if not all(i in valid_video_codecs for i in self.video_codecs or []):
raise ValidationError(f'Video codecs must be of: {", ".join(valid_video_codecs)}')
valid_audio_codecs = ('aac', 'opus', 'mp3', 'vorbis')
if not all(i in valid_audio_codecs for i in self.audio_codecs or []):
raise ValidationError(f'Audio codecs must be of: {", ".join(valid_audio_codecs)}')
@dataclass
class DownloadRequest:
urls: List[str]
downloader: str
destination: Optional[str] = None
tag_names: Optional[List[str]] = field(default_factory=list)
frequency: Optional[int] = None
sub_downloader: Optional[str] = None
settings: Optional[dict] = field(default_factory=dict)
collection_id: Optional[int] = None
def __post_init__(self):
urls = [j for i in self.urls if (j := i.strip())]
if not urls:
raise InvalidDownload(f'urls cannot be empty')
# Get unique URLs, preserve order.
self.urls = unique_by_predicate(urls)
if self.destination:
destination = pathlib.Path(self.destination)
if destination.is_absolute():
destination = get_relative_to_media_directory(destination)
self.destination = str(destination)
self.tag_names = self.tag_names or []
# Validate settings contents. Remove empty values.
settings = self.settings or dict()
self.settings = {k: v for k, v in DownloadSettings(**settings).__dict__.items() if v not in ([], None)}
@dataclass
class DownloadBatchRequest:
"""Request for batch operations on downloads."""
download_ids: List[int]
def __post_init__(self):
if not self.download_ids:
raise ValidationError('download_ids cannot be empty')
@dataclass
class JSONErrorResponse:
error: str
@dataclass
class EventsRequest:
after: Optional[str] = None
@dataclass
class TagRequest:
name: str
color: str
@dataclass
class DeleteTagRequest:
name: str
@dataclass
class NotifyRequest:
message: str
url: str = None
@dataclass
class VINDecoderRequest:
vin_number: str
@dataclass
class VIN:
country: str
manufacturer: str
region: str
years: str
body: str = None
engine: str = None
model: str = None
plant: str = None
transmission: str = None
serial: str = None
def __json__(self) -> dict:
d = dict(
country=self.country,
manufacturer=self.manufacturer,
region=self.region,
years=self.years,
body=self.body,
engine=self.engine,
model=self.model,
plant=self.plant,
transmission=self.transmission,
serial=self.serial,
)
return d
@dataclass
class VINDecoderResponse:
vin: VIN
@dataclass
class SearchSuggestionsRequest:
search_str: Optional[str] = None
order_by_video_count: bool = True
@dataclass
class SearchFileEstimateRequest:
search_str: Optional[str] = None
mimetypes: List[str] = field(default_factory=list)
tag_names: List[str] = field(default_factory=list)
months: Optional[List[int]] = None
from_year: Optional[int] = None
to_year: Optional[int] = None
any_tag: bool = False
def __post_init__(self):
if self.any_tag and self.tag_names:
raise ValidationError('Cannot search for any tag, and list of tags.')
@dataclass
class SearchOtherEstimateRequest:
tag_names: List[str] = field(default_factory=list)
search_str: Optional[str] = None