diff --git a/app/src/api.js b/app/src/api.js index 15129c84..06762b9b 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -34,11 +34,6 @@ export class ApiDownError extends Error { const apiDownError = new ApiDownError('API is down'); -// A secondary WROLPi refuses requests while it copies its library from its primary. Like a down API, callers -// treat it quietly; the app shows the sync's progress instead (see SyncingOverlay). -export class SyncingError extends ApiDownError { -} - async function apiCall(url, method, body, ms = 60_000) { let init = {method}; if (body !== undefined) { @@ -64,9 +59,6 @@ async function apiCall(url, method, body, ms = 60_000) { let copy = response.clone(); const content = await copy.json(); const code = content['code']; - if (response.status === 503 && code === 'SYNCING') { - throw new SyncingError('This WROLPi is syncing from its primary'); - } console.error('API error response json', content); if (response.status === 403 && code === 'WROL_MODE_ENABLED') { toast({ diff --git a/app/src/test-fixtures.js b/app/src/test-fixtures.js index 7c19f160..2168a019 100644 --- a/app/src/test-fixtures.js +++ b/app/src/test-fixtures.js @@ -96,33 +96,6 @@ export const settingsFixture = (overrides = {}) => ({ ...overrides, }); -/** The `sync` of GET /api/status: a secondary that has never synced. */ -export const syncStatusFixture = (overrides = {}) => ({ - state: 'idle', - phase: null, - requested: false, - force: false, - cancel_requested: false, - started_at: null, - finished_at: null, - last_success_at: null, - last_error: null, - last_skip_reason: null, - primary_url: null, - primary_version: null, - groups_seen: 0, - max_id: 0, - files_checked: 0, - files_to_download: 0, - files_downloaded: 0, - bytes_downloaded: 0, - files_failed: 0, - files_excluded: 0, - configs_imported: [], - disk_full: false, - ...overrides, -}); - /** GET /controller/api/sync/status: a secondary's sync that has never run (controller/lib/sync.py). */ export const controllerSyncStatusFixture = (overrides = {}) => ({ state: 'idle', @@ -198,8 +171,6 @@ export const statusFixture = (overrides = {}) => ({ memory_stats: {total: 4000000000, used: 1000000000, free: 3000000000, cached: 500000000}, processes_stats: [], sanic_workers: {}, - // A secondary's sync (wrolpi.sync); idle on a primary. - sync: syncStatusFixture(), // A backup to a second drive (wrolpi.sync.backup); idle without one. backup: backupStatusFixture(), version: '1.0.0', diff --git a/wrolpi/common.py b/wrolpi/common.py index 65b7422f..da64fdfb 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -2183,10 +2183,9 @@ def enable_wrol_mode(): get_wrolpi_config().wrol_mode = True from wrolpi.downloader import download_manager download_manager.stop() - # A secondary in WROL Mode is in use; a running sync stops between pages. A backup stops too. - from wrolpi.sync.lib import request_sync_cancel + # A backup stops. (A secondary's sync is the Controller's, which stops at its next step when it finds WROL Mode + # enabled; its API is stopped while it copies, so WROL Mode cannot be enabled then.) from wrolpi.sync.backup import request_backup_cancel - request_sync_cancel() request_backup_cancel() diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index 4397e85e..a977f019 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -5,7 +5,7 @@ import queue from sanic import Sanic from wrolpi.common import logger -from wrolpi.sync.status import default_sync_status, default_backup_status +from wrolpi.sync.status import default_backup_status from wrolpi.vars import LOG_LEVEL_INT logger = logger.getChild(__name__) @@ -68,9 +68,6 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.download_manager_stopped.set() app.shared_ctx.single_tasks_started = multiprocessing.Event() - # Set while a secondary syncs (wrolpi.sync); the API refuses requests. An Event so the request middleware can - # check it without a manager round trip. - app.shared_ctx.sync_api_locked = multiprocessing.Event() app.shared_ctx.flags_initialized = multiprocessing.Event() # Transcode @@ -158,7 +155,6 @@ def reset_shared_contexts(app: Sanic): current_commit=None, commits_behind=0, git_branch=None, - sync=default_sync_status(), backup=default_backup_status(), )) app.shared_ctx.cache.clear() @@ -248,7 +244,6 @@ def reset_shared_contexts(app: Sanic): # Clear events_history in place to preserve references held by fixtures del app.shared_ctx.events_history[:] app.shared_ctx.single_tasks_started.clear() - app.shared_ctx.sync_api_locked.clear() app.shared_ctx.flags_initialized.clear() if hasattr(app.shared_ctx, 'flag_holders'): app.shared_ctx.flag_holders.clear() diff --git a/wrolpi/errors.py b/wrolpi/errors.py index c2852ad6..8f01ccd2 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -45,12 +45,6 @@ class WROLModeEnabled(APIError): status_code = HTTPStatus.FORBIDDEN -class SyncInProgress(APIError): - code = 'SYNCING' - summary = 'This WROLPi is copying its library from its primary. Try again when the sync finishes.' - status_code = HTTPStatus.SERVICE_UNAVAILABLE - - class SecondaryRole(APIError): code = 'SECONDARY_ROLE' summary = 'This method is disabled on a secondary WROLPi.' diff --git a/wrolpi/flags.py b/wrolpi/flags.py index 88f3c2a5..bfb400d1 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -161,8 +161,6 @@ map_search_building = Flag('map_search_building') # The global refresh has been performed. This is False on a fresh instance of WROLPi. refresh_complete = Flag('refresh_complete', store_db=True) -# A secondary WROLPi is copying its library from its primary (see wrolpi.sync). Its API refuses requests meanwhile. -sync_running = Flag('sync_running') # This WROLPi is copying its media directory to its backup drive (see wrolpi.sync.backup). backup_running = Flag('backup_running') @@ -190,7 +188,6 @@ def get_flags() -> dict: media_mounted=media_mounted.is_set(), outdated_zims=outdated_zims.is_set(), refresh_complete=refresh_complete.is_set(), - sync_running=sync_running.is_set(), ) return flags diff --git a/wrolpi/perpetual.py b/wrolpi/perpetual.py index 80c40d17..c791f859 100644 --- a/wrolpi/perpetual.py +++ b/wrolpi/perpetual.py @@ -55,8 +55,7 @@ SHARED_LOCK_HEAL_TIMEOUT = 10 # when the recorded holder pid is dead, because a live worker may be inside the block right now. # file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and # process_queue already drops a stale set on its next idle tick. -PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active', 'sync_running', - 'backup_running') +PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active', 'backup_running') SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup', 'map_search_building') diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 161e86a1..14d82982 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -141,7 +141,6 @@ class FlagsStatusResponse: media_mounted: bool outdated_zims: bool refresh_complete: bool - sync_running: bool @dataclass @@ -201,33 +200,6 @@ class DownloadsSummaryResponse: daily_limit_reached: bool -@dataclass -class SyncStatusResponse: - """The progress of a secondary WROLPi copying its library from its primary (see wrolpi.sync).""" - state: str # 'idle', 'running', or 'error' - phase: Optional[str] # 'files', 'configs', or None - requested: bool - force: bool - cancel_requested: bool - started_at: Optional[str] - finished_at: Optional[str] - last_success_at: Optional[str] - last_error: Optional[str] - last_skip_reason: Optional[str] - primary_url: Optional[str] - primary_version: Optional[str] - groups_seen: int - max_id: int - files_checked: int - files_to_download: int - files_downloaded: int - bytes_downloaded: int - files_failed: int - files_excluded: int - configs_imported: List[str] - disk_full: bool - - @dataclass class BackupStatusResponse: """The progress of copying this WROLPi's media directory to its backup drive (see wrolpi.sync.backup).""" @@ -277,7 +249,6 @@ class StatusResponse: memory_stats: MemoryStatusStat processes_stats: List[ProcessStatusStat] sanic_workers: dict - sync: SyncStatusResponse version: str # `str` here was wrong too; wrol_mode_enabled() returns a bool. wrol_mode: bool diff --git a/wrolpi/sync/api.py b/wrolpi/sync/api.py index ccbabe3a..19cbfb78 100644 --- a/wrolpi/sync/api.py +++ b/wrolpi/sync/api.py @@ -4,54 +4,30 @@ import re from http import HTTPStatus from sanic import Request, Blueprint, response -from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi import flags -from wrolpi.api_utils import api_app, json_response, perpetual_signal -from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled, \ +from wrolpi.api_utils import json_response, perpetual_signal +from wrolpi.common import logger, is_secondary, wrol_mode_check, wrol_mode_enabled, \ CONTROLLER_URL, aiohttp_post from wrolpi.db import get_db_session -from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled -from wrolpi.sync import backup, lib, schema, snapshot +from wrolpi.errors import InvalidConfig, SecondaryRole, WROLModeEnabled +from wrolpi.sync import backup, lib, notify, schema, snapshot from wrolpi.sync.config import get_sync_config -from wrolpi.sync.status import get_backup_status, get_snapshot_status, get_sync_status +from wrolpi.sync.status import get_backup_status, get_snapshot_status from wrolpi.vars import DOCKERIZED sync_bp = Blueprint('Sync', '/api/sync') logger = logger.getChild(__name__) -# Requests a secondary answers while it syncs: enough for the UI to show the sync's progress, and to cancel it. -SYNC_ALLOWED_REQUESTS = frozenset({ - ('GET', '/api/echo'), - ('GET', '/api/events/feed'), - ('GET', '/api/settings'), - ('GET', '/api/status'), - ('GET', '/api/sync/info'), - ('GET', '/api/sync/status'), - ('POST', '/api/sync/cancel'), -}) - - -@api_app.middleware('request') -async def check_sync_api_lock(request: Request): - """A secondary refuses requests while it replaces its library with its primary's (see wrolpi.sync).""" - # The Event costs nothing to check; the Flag confirms it (the Event survives a perpetual process that died). - if not api_app.shared_ctx.sync_api_locked.is_set() or not flags.sync_running.is_set(): - return - if request.method == 'OPTIONS' or (request.method, request.path.rstrip('/')) in SYNC_ALLOWED_REQUESTS: - return - # Answered here rather than raised: the UI polls while it waits, and each refusal would be logged as an error. - return json_response(dict(error=SyncInProgress.summary, code=SyncInProgress.code), - SyncInProgress.status_code) - - @perpetual_signal(sleep=5) async def perpetual_sync_worker(): # A secondary's sync is the Controller's (controller/lib/sync.py), which stops this API while it runs. await backup.backup_tick() await snapshot.snapshot_tick() + # The Controller cannot send this API's events; announce the syncs it finished. + notify.notify_tick() @sync_bp.get('/info') @@ -64,22 +40,11 @@ async def get_info(_: Request): return json_response(await asyncio.to_thread(info)) -@sync_bp.get('/manifest') -@openapi.definition(summary="One page of this WROLPi's files, for a secondary to compare with its own") -@validate(query=schema.ManifestRequest) -async def get_manifest(_: Request, query: schema.ManifestRequest): - def page(): - with get_db_session() as session: - return lib.build_manifest_page(session, query.after_id, query.limit) - - return json_response(await asyncio.to_thread(page)) - - @sync_bp.get('/status') @openapi.definition(summary="The progress of this secondary's sync") async def get_status(_: Request): - return json_response(dict(sync=get_sync_status(), sync_running=bool(flags.sync_running.is_set()), - backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()), + # A secondary's sync is the Controller's: /controller/api/sync/status. + return json_response(dict(backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()), snapshot=get_snapshot_status())) diff --git a/wrolpi/sync/config.py b/wrolpi/sync/config.py index cdeab874..1007b22d 100644 --- a/wrolpi/sync/config.py +++ b/wrolpi/sync/config.py @@ -13,7 +13,7 @@ from wrolpi.sync.status import get_backup_status, update_backup_status @dataclass class SyncConfigValidator: version: int = None - # 'primary' or 'secondary'; a secondary copies its library from `primary_url` (see wrolpi.sync.lib). + # 'primary' or 'secondary'; a secondary copies its library from `primary_url` (see controller/lib/sync.py). role: str = None primary_url: str = None # The daily window ('HH:MM') in which a secondary syncs; none means only "Sync Now". diff --git a/wrolpi/sync/conftest.py b/wrolpi/sync/conftest.py index 0f2f09f9..479bfa93 100644 --- a/wrolpi/sync/conftest.py +++ b/wrolpi/sync/conftest.py @@ -7,12 +7,11 @@ import os import pathlib import tempfile import threading -from typing import Callable, List +from typing import Callable import pytest -from wrolpi.common import get_wrolpi_config, set_test_media_directory -from wrolpi.sync import lib +from wrolpi.common import set_test_media_directory from wrolpi.sync.config import get_sync_config @@ -125,33 +124,6 @@ def make_secondary(test_wrolpi_config): return make -@pytest.fixture -def fake_primary_api(monkeypatch): - """Answer the primary's sync API (info and manifest) from pages captured on the "primary" beforehand; media still - comes from the `primary_media_server`.""" - - def fake(pages: List[dict], media_directory: pathlib.Path, role: str = 'primary', version: str = None): - async def get_info(self): - from wrolpi.version import __version__ - return dict(role=role, version=version or __version__, media_directory=str(media_directory), - max_file_group_id=pages[-1]['max_id'] if pages else 0) - - async def iter_manifest(self, limit: int = lib.MANIFEST_PAGE_LIMIT): - for page in pages: - yield page - - monkeypatch.setattr(lib.PrimaryClient, 'get_info', get_info) - monkeypatch.setattr(lib.PrimaryClient, 'iter_manifest', iter_manifest) - - return fake - - -def manifest_page(*files: tuple, next_after_id=None, max_id: int = 1) -> dict: - """A manifest page of one group holding `files`, each (path, size, mtime).""" - return dict(groups=[dict(id=1, paths=[dict(path=p, size=s, mtime=m) for p, s, m in files])], - next_after_id=next_after_id, max_id=max_id, limit=lib.MANIFEST_PAGE_LIMIT) - - @contextlib.contextmanager def media_directory(path: pathlib.Path): set_test_media_directory(path) diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py index 0c843bff..b5b63ee4 100644 --- a/wrolpi/sync/lib.py +++ b/wrolpi/sync/lib.py @@ -1,86 +1,54 @@ -"""Copy a primary WROLPi's library to a secondary WROLPi. +"""The API's side of a secondary WROLPi copying its primary. -A secondary pulls; the primary only answers reads. The library is the media files plus the config YAMLs, because the -database is rebuilt from both, so a sync is: +The Controller runs a sync (controller/lib/sync.py): it stops this API, copies the primary's whole media directory +with rsync from the primary's daemon (controller/rsyncd.conf), puts the primary's database snapshot +(wrolpi.sync.snapshot) in place, and starts this API again. What is left here: -1. Walk the primary's manifest (`/api/sync/manifest`): every file of every FileGroup, paged by FileGroup id. -2. Download each file the secondary lacks, or has with a different size or modification time, from the primary's - `/media/` (Caddy). A file is written to a hidden partial (refresh never indexes hidden files), then renamed into - place with the primary's modification time, so a later comparison is exact and an interrupted download resumes. -3. Refresh the downloaded files, a page at a time. -4. Copy the synced configs (`SYNCED_CONFIG_FILE_NAMES`) and import them. Files come first so the configs never - refer to files the secondary does not have yet. - -Files deleted on the primary are never deleted on the secondary. - -While a sync runs the secondary's API refuses requests (`check_sync_api_lock`), so nobody edits a library that is -being replaced. +- what a primary tells a secondary about itself (`get_sync_info`), +- the secondary's settings: its primary's address and the folders it does not copy, +- the primary's library settings, which a secondary's API takes from the primary's wrolpi.yaml when it starts + (`apply_primary_wrolpi_config`), +- the daily window, which the backup drive uses too. """ -import asyncio -import contextlib import json import os import pathlib import posixpath -import re -import shutil import ssl -import time -import uuid -from dataclasses import dataclass from datetime import datetime from http import HTTPStatus -from typing import AsyncGenerator, Callable, Dict, List, Optional -from urllib.parse import quote +from typing import List, Optional +from urllib.parse import quote, urlparse import aiohttp import yaml -from sqlalchemy import func, select, text +from sqlalchemy import text from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session from wrolpi import flags -from wrolpi.collections.models import Collection -from wrolpi.collections.sync import get_playlists_directory, MANAGED_NAME -from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \ - get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES -from wrolpi.dates import now, is_within_daily_window -from wrolpi.db import get_db_session +from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary +from wrolpi.dates import is_within_daily_window from wrolpi.errors import InvalidConfig -from wrolpi.events import Events from wrolpi.sync.config import get_sync_config -from wrolpi.sync.status import get_sync_status, update_sync_status -from wrolpi.tags import get_tags_directory -from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST +from wrolpi.vars import DEFAULT_HTTP_HEADERS from wrolpi.version import __version__ logger = logger.getChild(__name__) -MANIFEST_PAGE_LIMIT = 500 -# Files downloaded at once. Small files are dominated by round trips; four keeps a Pi's disk and CPU busy. -DOWNLOAD_CONCURRENCY = 4 -DOWNLOAD_CHUNK_SIZE = 1024 * 1024 -# Never fill the secondary's drive; leave this much free. -MIN_FREE_BYTES = 1024 ** 3 -# exFAT stores modification times in 2-second steps. -MTIME_TOLERANCE_SECONDS = 2.0 -# How often a running sync reports its progress. +# How often a running backup reports its progress. PROGRESS_INTERVAL_SECONDS = 2.0 -# Within the sync window, sync again this long after the last sync finished (an idle sync only compares sizes). -SYNC_WINDOW_REPEAT_SECONDS = 15 * 60 -REFRESH_TIMEOUT_SECONDS = 60 * 60 -# Configs and directory listings are small; never read more than this from the primary. -MAX_CONFIG_BYTES = 64 * 1024 * 1024 +# Answers and directory listings are small; never read more than this from the primary. +MAX_RESPONSE_BYTES = 64 * 1024 * 1024 +READ_CHUNK_SIZE = 1024 * 1024 # Enough for any real library; keeps a pasted list from growing without bound. MAX_EXCLUDED_DIRECTORIES = 500 -# Top-level directories never written by a sync: the configs and database, and the Tags Directory (hard links that -# the secondary rebuilds from tags.yaml). +# Top-level folders that cannot be excluded: the configs and database, and the Tags Directory (hard links that the +# secondary rebuilds). PROTECTED_DIRECTORIES = frozenset({'config', 'tags'}) # Top-level folders never offered for exclusion: the protected ones, and the root-only folder of every ext4 drive. UNLISTED_DIRECTORIES = PROTECTED_DIRECTORIES | {'lost+found'} -# Files that are still being downloaded on the primary: aria2 and yt-dlp partials, and yt-dlp format files. -INCOMPLETE_FILE = re.compile(r'(\.part|\.aria2|\.ytdl|\.temp)$|\.f\d{2,4}\.[^.]+$') # wrolpi.yaml keys that describe the library rather than this machine (hotspot, timezone, etc. stay local; the role # and other sync settings are in sync.yaml, which is never synced). @@ -103,57 +71,8 @@ class SyncError(Exception): pass -class SyncDiskFull(SyncError): - pass - - -class SyncCancelled(SyncError): - pass - - -@dataclass(frozen=True) -class ManifestFile: - path: str # Relative to the media directory, '/' separated. - size: int - mtime: float - - # --------------------------------------------------------------------------------------------------------------------- -# Primary: the manifest. - - -def build_manifest_page(session: Session, after_id: int = 0, limit: int = MANIFEST_PAGE_LIMIT) -> dict: - """One page of the primary's library: the files of the FileGroups after `after_id`, in id order. - - Sizes and modification times come from the files themselves, not the database, which does not record them for - every file of a group.""" - from wrolpi.files.models import FileGroup - - media_directory = get_media_directory().resolve() - rows = session.execute( - select(FileGroup.id, FileGroup.directory, FileGroup.files) - .where(FileGroup.id > after_id) - .order_by(FileGroup.id) - .limit(limit) - ).all() - max_id = session.execute(select(func.max(FileGroup.id))).scalar() or 0 - - groups = list() - for id_, directory, files in rows: - paths = list() - for entry in files or []: - if not isinstance(entry, dict) or not entry.get('path'): - continue - path = pathlib.Path(entry['path']) - if not path.is_absolute(): - # Files are stored relative to their FileGroup's directory; older rows stored absolute paths. - path = pathlib.Path(directory) / path - if manifest_file := _manifest_file(media_directory, path): - paths.append(manifest_file) - groups.append(dict(id=id_, paths=[dict(path=i.path, size=i.size, mtime=i.mtime) for i in paths])) - - next_after_id = rows[-1][0] if len(rows) == limit else None - return dict(groups=groups, next_after_id=next_after_id, max_id=max_id, limit=limit) +# Primary: what it tells a secondary. def _is_protected(name: str) -> bool: @@ -166,32 +85,13 @@ def _is_unlisted(name: str) -> bool: return name.casefold() in UNLISTED_DIRECTORIES -def _manifest_file(media_directory: pathlib.Path, path: pathlib.Path) -> Optional[ManifestFile]: - try: - relative = path.relative_to(media_directory) - except ValueError: - return None - parts = relative.parts - if not parts or any(i.startswith('.') for i in parts) or _is_protected(parts[0]): - return None - if INCOMPLETE_FILE.search(path.name): - return None - try: - stat = path.stat() - except (FileNotFoundError, NotADirectoryError): - return None - if not path.is_file(): - return None - return ManifestFile(relative.as_posix(), stat.st_size, stat.st_mtime) - - def get_sync_info(session: Session) -> dict: - from wrolpi.files.models import FileGroup + """What a secondary's Controller checks before it replaces its library and database with this WROLPi's.""" return dict( version=__version__.strip(), role=get_sync_config().role, + # The database names files by their full path. media_directory=str(get_media_directory()), - max_file_group_id=session.execute(select(func.max(FileGroup.id))).scalar() or 0, # A secondary that replaces its database with this one must run code of the same migration. database_revision=_database_revision(session), # An unmounted media directory looks like an empty library. @@ -207,12 +107,12 @@ def _database_revision(session: Session) -> Optional[str]: # --------------------------------------------------------------------------------------------------------------------- -# Secondary: comparing and writing files. +# Secondary: its primary, and the folders it does not copy. def resolve_destination(media_directory: pathlib.Path, path: str) -> Optional[pathlib.Path]: - """The secondary's path for a manifest path, or None if the path must not be written: outside the media directory, - hidden, protected, or reached through a symlink.""" + """The path in this WROLPi's media directory for `path` (relative, '/'-separated), or None if it may not be used: + outside the media directory, hidden, protected, or reached through a symlink.""" if not path or path.startswith('/') or '\\' in path: return None parts = pathlib.PurePosixPath(path).parts @@ -290,45 +190,17 @@ async def list_primary_directories(path: str) -> List[dict]: return sorted(directories, key=lambda i: i['name'].lower()) -def needs_download(destination: pathlib.Path, file: ManifestFile) -> bool: - try: - stat = destination.stat() - except FileNotFoundError: - return True - return stat.st_size != file.size or abs(stat.st_mtime - file.mtime) > MTIME_TOLERANCE_SECONDS - - -def partial_path(destination: pathlib.Path) -> pathlib.Path: - """Hidden, so a refresh never indexes a file that is still downloading.""" - return destination.with_name(f'.{destination.name}.partial') - - -def _cpu_lacks_aes() -> bool: - """True when the CPU has no AES instructions (a Raspberry Pi 4). ChaCha20 is then about twice as fast.""" - try: - cpuinfo = pathlib.Path('/proc/cpuinfo').read_text() - except OSError: - return False - features = [i for i in cpuinfo.splitlines() if i.lower().startswith(('features', 'flags'))] - return bool(features) and not any(' aes' in f' {i.split(":", 1)[-1]} ' for i in features) - - def make_sync_ssl_context() -> ssl.SSLContext: """WROLPi serves HTTPS with its own certificate authority; on a trusted LAN (see the Bookshelf threat model) the - primary is not verified. Pinning it would load the primary's `/ca.crt` here with `load_verify_locations`.""" + primary is not verified.""" context = ssl.create_default_context() context.check_hostname = False context.verify_mode = ssl.CERT_NONE - if _cpu_lacks_aes(): - # Python cannot order TLS 1.3 ciphers, and TLS 1.3 picks AES-GCM, which a Pi 4 decrypts at half the speed. - context.maximum_version = ssl.TLSVersion.TLSv1_2 - context.set_ciphers('ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE+AESGCM') return context def normalize_primary_url(url: str) -> str: """'https://host[:port]' with no path; raises ValueError otherwise.""" - from urllib.parse import urlparse url = (url or '').strip().rstrip('/') parsed = urlparse(url) if parsed.scheme not in ('http', 'https') or not parsed.hostname: @@ -343,15 +215,16 @@ def normalize_primary_url(url: str) -> str: async def _read_limited(response: aiohttp.ClientResponse, path: str) -> bytes: data = bytearray() - async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE): + async for chunk in response.content.iter_chunked(READ_CHUNK_SIZE): data.extend(chunk) - if len(data) > MAX_CONFIG_BYTES: - raise SyncError(f'{path} is larger than {MAX_CONFIG_BYTES} bytes') + if len(data) > MAX_RESPONSE_BYTES: + raise SyncError(f'{path} is larger than {MAX_RESPONSE_BYTES} bytes') return bytes(data) class PrimaryClient: - """Reads from a primary WROLPi: its sync API, and its media directory through Caddy.""" + """Asks a primary WROLPi about itself (Test Connection) and lists its folders (to choose which are not copied). + The sync itself is the Controller's, by rsync.""" def __init__(self, url: str): self.url = normalize_primary_url(url) @@ -359,48 +232,24 @@ class PrimaryClient: async def __aenter__(self): ssl_context = make_sync_ssl_context() if self.url.startswith('https') else False - connector = aiohttp.TCPConnector(ssl=ssl_context, limit=DOWNLOAD_CONCURRENCY * 2) - # A large file takes minutes; only a stalled read is an error. - timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=120) - self.session = aiohttp.ClientSession(connector=connector, timeout=timeout, headers=DEFAULT_HTTP_HEADERS) + timeout = aiohttp.ClientTimeout(total=60, connect=30) + self.session = aiohttp.ClientSession(connector=aiohttp.TCPConnector(ssl=ssl_context), timeout=timeout, + headers=DEFAULT_HTTP_HEADERS) return self async def __aexit__(self, *exc): await self.session.close() - def media_url(self, path: str) -> str: - return f'{self.url}/media/{quote(path)}' - - async def get_json(self, path: str, params: dict = None, headers: dict = None): - async with self.session.get(f'{self.url}{path}', params=params, headers=headers) as response: + async def get_info(self) -> dict: + path = '/api/sync/info' + async with self.session.get(f'{self.url}{path}') as response: if response.status != HTTPStatus.OK: raise SyncError(f'{path} returned HTTP {response.status}') return json.loads(await _read_limited(response, path)) - async def get_info(self) -> dict: - return await self.get_json('/api/sync/info') - - async def iter_manifest(self, limit: int = MANIFEST_PAGE_LIMIT) -> AsyncGenerator[dict, None]: - after_id = 0 - while True: - page = await self.get_json('/api/sync/manifest', params=dict(after_id=after_id, limit=limit)) - yield page - if page.get('next_after_id') is None: - return - after_id = page['next_after_id'] - - async def get_bytes(self, path: str) -> Optional[bytes]: - """A file in the primary's media directory, or None if it does not exist.""" - async with self.session.get(self.media_url(path)) as response: - if response.status == HTTPStatus.NOT_FOUND: - return None - if response.status != HTTPStatus.OK: - raise SyncError(f'{path} returned HTTP {response.status}') - return await _read_limited(response, path) - async def list_directory(self, path: str) -> List[dict]: """The entries of a directory in the primary's media directory (Caddy's browse listing), [] if missing.""" - url = self.media_url(path.rstrip('/') + '/') + url = f'{self.url}/media/{quote(path.rstrip("/") + "/")}' async with self.session.get(url, headers={'Accept': 'application/json'}) as response: if response.status == HTTPStatus.NOT_FOUND: return [] @@ -408,218 +257,9 @@ class PrimaryClient: raise SyncError(f'{path} returned HTTP {response.status}') return json.loads(await _read_limited(response, path)) - @contextlib.asynccontextmanager - async def open_media(self, path: str, offset: int = 0): - # Sizes and ranges count the file's bytes, so the file must not be compressed in transit. - headers = {'Accept-Encoding': 'identity'} - if offset: - headers['Range'] = f'bytes={offset}-' - async with self.session.get(self.media_url(path), headers=headers) as response: - yield response - - -async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path, - on_bytes=None, before_replace: Callable[[pathlib.Path], None] = None) -> int: - """Download one file to `destination` with the primary's modification time. Returns the bytes received. - - A partial file left by an earlier attempt is resumed. `before_replace` is called with the complete new version - just before it takes the place of `destination`.""" - partial = partial_path(destination) - destination.parent.mkdir(parents=True, exist_ok=True) - - offset = partial.stat().st_size if partial.is_file() else 0 - if offset > file.size: - offset = 0 - remaining = file.size - offset - if shutil.disk_usage(destination.parent).free - remaining < MIN_FREE_BYTES: - raise SyncDiskFull(f'Not enough free space for {file.path}') - - received = 0 - if remaining > 0 or not partial.is_file(): - async with client.open_media(file.path, offset) as response: - if response.status == HTTPStatus.PARTIAL_CONTENT and offset: - mode = 'ab' - elif response.status == HTTPStatus.OK: - mode, offset = 'wb', 0 - elif response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE: - # The partial does not match the primary's file; start over next time. - partial.unlink(missing_ok=True) - raise SyncError(f'{file.path} changed on the primary') - else: - raise SyncError(f'{file.path} returned HTTP {response.status}') - - if response.content_length is not None and response.content_length != file.size - offset: - # The file changed on the primary after the manifest was read; the next sync gets it. - raise SyncError(f'{file.path} changed on the primary') - - with open(partial, mode) as fh: - async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE): - if offset + received + len(chunk) > file.size: - # Never write past the size the free-space check allowed for. - fh.close() - partial.unlink(missing_ok=True) - raise SyncError(f'{file.path} is larger than the primary said') - await asyncio.to_thread(fh.write, chunk) - received += len(chunk) - if on_bytes: - on_bytes(len(chunk)) - - if (size := partial.stat().st_size) != file.size: - raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}') - os.utime(partial, (file.mtime, file.mtime)) - if before_replace: - before_replace(partial) - os.replace(partial, destination) - return received - - -def link_directories() -> List[tuple]: - """Where hard links of library files are kept, each as (directory, recursive): the Tags Directory, the - playlists directory, and the directory of every playlist that has its own.""" - directories = [(get_tags_directory(), True), (get_playlists_directory(), True)] - with get_db_session() as session: - custom = session.execute( - select(Collection.directory).where(Collection.kind == 'playlist', Collection.directory.isnot(None)) - ).scalars().all() - # A playlist's own directory may be any folder of the library; only the links directly in it are its own. - directories.extend((pathlib.Path(i), False) for i in custom) - return directories - - -def _is_link_name(link_name: str, file_name: str) -> bool: - """True if `link_name` is what the Tags Directory (the file's name) or a playlist (the name after its position) - calls a link to a file named `file_name`.""" - return link_name == file_name or bool(MANAGED_NAME.match(link_name)) and link_name[5:] == file_name - - -def find_links(files: List[pathlib.Path]) -> Dict[pathlib.Path, List[pathlib.Path]]: - """The Tags Directory and playlist links of each of `files`: the files in `link_directories` that are the same - inode as the file and are named for it. A file without other hard links has none and is left out.""" - inodes = dict() - for file in files: - try: - stat = file.lstat() - except FileNotFoundError: - continue - if stat.st_nlink > 1: - # Library files may be hard links of each other, so several files can share an inode. - inodes.setdefault((stat.st_dev, stat.st_ino), []).append(file) - links = dict() - if not inodes: - return links - - seen = set() - for directory, recursive in link_directories(): - roots = (i[0] for i in os.walk(directory)) if recursive else (str(directory),) - for root in roots: - if root in seen: - continue - seen.add(root) - try: - names = os.listdir(root) - except OSError: - continue - for name in names: - path = pathlib.Path(root, name) - try: - stat = path.lstat() - except OSError: - continue - for file in inodes.get((stat.st_dev, stat.st_ino), ()): - if _is_link_name(name, file.name): - links.setdefault(file, []).append(path) - return links - - -def move_links(links: List[pathlib.Path], old_version: pathlib.Path, new_version: pathlib.Path): - """Make each of `links` that is still a link to `old_version` a link to `new_version`. - - The Tags Directory and a playlist refuse to delete a file that has no other link, because it may be a last - copy. A link left on a replaced version is exactly that: once it is no longer wanted (the primary changed the - file's tags, or reordered the playlist) it would stay forever.""" - try: - old = old_version.lstat() - except FileNotFoundError: - return - for link in links: - # Short, so it fits wherever the link's own name does. - temporary = link.with_name(f'.{uuid.uuid4().hex}.sync') - try: - stat = link.lstat() - if (stat.st_dev, stat.st_ino) != (old.st_dev, old.st_ino): - continue - # Linked beside it, then renamed over it: the link is never missing. - os.link(new_version, temporary) - os.replace(temporary, link) - except OSError as e: - logger.error(f'Failed to move the link {repr(str(link))}: {e}') - temporary.unlink(missing_ok=True) - # --------------------------------------------------------------------------------------------------------------------- -# Secondary: configs. - - -def _import_tags(): - from wrolpi.tags import import_tags_config - import_tags_config() - - -def _import_downloads(): - from wrolpi.downloader import get_download_manager_config - get_download_manager_config().import_config() - - -def _import_channels(): - from modules.videos.lib import import_channels_config - import_channels_config() - - -def _import_domains(): - from modules.archive.lib import import_domains_config - import_domains_config() - - -def _import_playlists(): - from wrolpi.collections.config import playlists_config - from wrolpi.collections.sync import sync_playlists_directory - playlists_config.import_config() - sync_playlists_directory.activate_switch() - - -def _import_zims(): - from modules.zim.lib import import_zims_config - import_zims_config() - - -def _import_plain(file_name: str): - def import_(): - get_config_by_file_name(file_name).import_config() - - return import_ - - -def _import_inventories(): - from modules.inventory.common import import_inventories_config - import_inventories_config() - - -# In the order of `import_all_db_configs`: Collections refer to Tags, and link to Downloads. -SYNCED_CONFIG_IMPORTS = ( - ('tags.yaml', _import_tags), - ('download_manager.yaml', _import_downloads), - ('channels.yaml', _import_channels), - ('domains.yaml', _import_domains), - ('playlists.yaml', _import_playlists), - ('zims.yaml', _import_zims), - ('map_pins.yaml', _import_plain('map_pins.yaml')), - ('maps.yaml', _import_plain('maps.yaml')), - ('bookmarks.yaml', _import_plain('bookmarks.yaml')), - ('videos_downloader.yaml', _import_plain('videos_downloader.yaml')), - ('archives_downloader.yaml', _import_plain('archives_downloader.yaml')), -) -INVENTORY_DIRECTORY = 'config/inventory' -assert {i for i, _ in SYNCED_CONFIG_IMPORTS} | {'inventory/catalog.yaml'} == SYNCED_CONFIG_FILE_NAMES +# Secondary: the primary's library settings. def _parse_config(data: bytes, name: str) -> dict: @@ -629,66 +269,6 @@ def _parse_config(data: bytes, name: str) -> dict: return config -def write_config_bytes(file: pathlib.Path, data: bytes, backup: pathlib.Path = None) -> bool: - """Replace `file` with the primary's bytes (keeping its version). Returns False if they are already the same. - - The replaced file is copied to `backup` first (today's config backup), unless today's backup exists: like a save, - the first copy of the day is the one kept, so a sync can be undone with a config backup import.""" - if file.is_file() and file.read_bytes() == data: - return False - if backup and file.is_file() and not backup.exists(): - backup.parent.mkdir(parents=True, exist_ok=True) - shutil.copy2(file, backup) - file.parent.mkdir(parents=True, exist_ok=True) - temporary = file.with_name(f'.{file.name}.sync') - temporary.write_bytes(data) - os.replace(temporary, file) - return True - - -async def sync_configs(client: PrimaryClient, primary_media_directory: str, reimport: bool) -> List[str]: - """Copy the primary's synced configs, then import those that changed (all of them when `reimport`, because newly - downloaded files may let a config import entries it skipped before). Returns the imported configs.""" - imported = list() - for file_name, import_ in SYNCED_CONFIG_IMPORTS: - data = await client.get_bytes(f'config/{file_name}') - if data is None: - continue - config = get_config_by_file_name(file_name) - _parse_config(data, file_name) - changed = write_config_bytes(config.get_file(), data, config._get_backup_filename()) - if changed or reimport: - logger.info(f'Importing {file_name} from the primary') - await asyncio.to_thread(import_) - imported.append(file_name) - - if await _sync_inventory_directory(client) or reimport: - logger.info('Importing inventories from the primary') - await asyncio.to_thread(_import_inventories) - imported.append('inventory') - - await _merge_wrolpi_config(client, primary_media_directory) - return imported - - -async def _sync_inventory_directory(client: PrimaryClient) -> bool: - from modules.inventory.common import get_inventory_configs - inventories = get_inventory_configs() - directory = inventories.get_directory() - changed = False - for entry in await client.list_directory(INVENTORY_DIRECTORY): - name = entry.get('name') or '' - if entry.get('is_dir') or not name.endswith('.yaml') or '/' in name or name.startswith('.'): - continue - data = await client.get_bytes(f'{INVENTORY_DIRECTORY}/{name}') - if data is None: - continue - _parse_config(data, name) - backup = inventories._get_backup_filename(name.removesuffix('.yaml')) - changed = write_config_bytes(directory / name, data, backup) or changed - return changed - - def merge_wrolpi_config(primary_config: dict, primary_media_directory: str) -> dict: """The library settings of the primary's wrolpi.yaml that differ from this WROLPi's.""" wrolpi_config = get_wrolpi_config() @@ -743,226 +323,8 @@ def _relative_directories(directories: List[str], primary_media_directory: str) return relative -async def _merge_wrolpi_config(client: PrimaryClient, primary_media_directory: str): - data = await client.get_bytes('config/wrolpi.yaml') - if data is None: - return - if changes := merge_wrolpi_config(_parse_config(data, 'wrolpi.yaml'), primary_media_directory): - logger.info(f'Applying library settings from the primary: {sorted(changes)}') - get_wrolpi_config().update(changes) - - # --------------------------------------------------------------------------------------------------------------------- -# Secondary: the sync. - - -def sync_skip_reason() -> Optional[str]: - """Why a sync cannot start now, or None.""" - config = get_sync_config() - if not is_secondary(): - return 'This WROLPi is not a secondary' - if wrol_mode_enabled(): - return 'WROL Mode is enabled' - if not config.primary_url: - return 'No primary is configured' - if not PYTEST: - if not flags.db_up.is_set(): - return 'The database is not up' - if not flags.refresh_complete.is_set(): - return 'The first refresh has not finished' - if get_wrolpi_config().require_media_mounted and not flags.media_mounted.is_set(): - return 'The media directory is not mounted' - if flags.sync_running.is_set(): - return 'A sync is already running' - return None - - -def request_sync(force: bool = False): - update_sync_status(requested=True, force=force) - - -def request_sync_cancel(): - """Stop a running sync between pages, and forget a requested one.""" - from wrolpi.api_utils import api_app - if 'sync' not in api_app.shared_ctx.status: - return - status = get_sync_status() - if status['state'] == 'running' or status['requested']: - update_sync_status(cancel_requested=status['state'] == 'running', requested=False) - - -def _check_cancel(): - if get_sync_status()['cancel_requested']: - raise SyncCancelled('The sync was cancelled') - if wrol_mode_enabled(): - raise SyncCancelled('WROL Mode was enabled') - - -@contextlib.contextmanager -def lock_api(): - """Refuse API requests (see `check_sync_api_lock`) while the library is replaced.""" - from wrolpi.api_utils import api_app - with flags.sync_running: - api_app.shared_ctx.sync_api_locked.set() - try: - yield - finally: - api_app.shared_ctx.sync_api_locked.clear() - - -async def run_sync(force: bool = False) -> dict: - """Copy the primary's library to this secondary. Returns the final status.""" - if reason := sync_skip_reason(): - logger.info(f'Not syncing: {reason}') - return update_sync_status(last_skip_reason=reason, force=False) - - url = get_sync_config().primary_url - started_at = now() - update_sync_status( - state='running', phase='files', force=False, cancel_requested=False, started_at=started_at.isoformat(), - finished_at=None, last_error=None, last_skip_reason=None, primary_url=url, primary_version=None, - groups_seen=0, max_id=0, files_checked=0, files_to_download=0, files_downloaded=0, bytes_downloaded=0, - files_failed=0, files_excluded=0, configs_imported=[], disk_full=False, - ) - logger.warning(f'Syncing from the primary {url}') - try: - with lock_api(): - async with PrimaryClient(url) as client: - info = await client.get_info() - if info.get('role') != 'primary': - raise SyncError(f'{url} is not a primary WROLPi') - if (info.get('version') or '').strip() != __version__.strip(): - logger.warning(f'The primary runs WROLPi {info.get("version")}, this is {__version__.strip()}') - update_sync_status(primary_version=(info.get('version') or '').strip() or None) - - downloaded = await _sync_files(client) - _check_cancel() - - update_sync_status(phase='configs') - primary_media_directory = info.get('media_directory') or '' - imported = await sync_configs(client, primary_media_directory, reimport=bool(downloaded) or force) - except SyncCancelled as e: - logger.warning(f'Sync stopped: {e}') - return update_sync_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e), - cancel_requested=False) - except Exception as e: - logger.error('Sync failed', exc_info=e) - Events.send_sync_failed(f'Sync from the primary failed: {e}') - return update_sync_status(state='error', phase=None, finished_at=now().isoformat(), last_error=str(e), - cancel_requested=False) - - status = get_sync_status() - finished_at = now() - failed = status['files_failed'] - message = f'Synced {len(downloaded)} files from the primary' - if failed: - message += f'; {failed} failed' - if status['disk_full']: - message += '; the drive is full' - logger.warning(f'{message} in {(finished_at - started_at).total_seconds():.0f}s') - if downloaded or failed or imported: - Events.send_sync_completed(message) - last_success_at = finished_at.isoformat() if not failed and not status['disk_full'] else status['last_success_at'] - return update_sync_status(state='idle', phase=None, finished_at=finished_at.isoformat(), - last_success_at=last_success_at, configs_imported=imported, - last_error=f'{failed} files failed' if failed else None) - - -async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]: - """Download the files this secondary lacks, and refresh them a page at a time. Returns the files downloaded.""" - media_directory = get_media_directory() - downloaded = list() - groups_seen = files_checked = files_to_download = files_failed = files_excluded = 0 - excluded_directories = set(get_sync_config().sync_excluded_directories) - bytes_downloaded = 0 - disk_full = False - - last_report = 0.0 - - def report(force: bool = False): - nonlocal last_report - if force or time.monotonic() - last_report >= PROGRESS_INTERVAL_SECONDS: - last_report = time.monotonic() - update_sync_status(groups_seen=groups_seen, files_checked=files_checked, - files_to_download=files_to_download, files_downloaded=len(downloaded), - bytes_downloaded=bytes_downloaded, files_failed=files_failed, - files_excluded=files_excluded, disk_full=disk_full) - - def count_bytes(n: int): - nonlocal bytes_downloaded - bytes_downloaded += n - report() - - async for page in client.iter_manifest(): - _check_cancel() - wanted = list() - for group in page['groups']: - groups_seen += 1 - for entry in group['paths']: - files_checked += 1 - file = ManifestFile(entry['path'], int(entry['size']), float(entry['mtime'])) - if excluded_directories and is_excluded(file.path, excluded_directories): - files_excluded += 1 - continue - destination = resolve_destination(media_directory, file.path) - if destination is None: - logger.warning(f'Refusing to sync {repr(file.path)}') - files_failed += 1 - continue - if needs_download(destination, file): - wanted.append((file, destination)) - files_to_download += len(wanted) - update_sync_status(max_id=page.get('max_id') or 0) - report(force=True) - - page_downloaded = list() - if wanted and not disk_full: - semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY) - # A file's links move to its new version before the new version takes its place, so a sync that is - # interrupted in between downloads nothing again and still leaves no link on the old version. - links = await asyncio.to_thread(find_links, [i for _, i in wanted]) - - async def download(file_: ManifestFile, destination_: pathlib.Path): - nonlocal disk_full - async with semaphore: - if disk_full: - return None - try: - before_replace = None - if destination_ in links: - def before_replace(new_version: pathlib.Path): - move_links(links[destination_], destination_, new_version) - await download_file(client, file_, destination_, count_bytes, before_replace) - downloaded.append(destination_) - return destination_ - except SyncDiskFull as e: - disk_full = True - logger.error(str(e)) - except Exception as e: - logger.error(f'Failed to sync {repr(file_.path)}: {e}') - return False - - results = await asyncio.gather(*(download(f, d) for f, d in wanted)) - page_downloaded = [i for i in results if i] - files_failed += sum(1 for i in results if i is False) - - update_sync_status(max_id=page.get('max_id') or 0) - report(force=True) - - if page_downloaded: - await _refresh(page_downloaded) - - return downloaded - - -async def _refresh(paths: List[pathlib.Path]): - from wrolpi.files.worker import file_worker, FileWorkerJobFailed - job_id = file_worker.queue_refresh(paths, send_events=False) - try: - await file_worker.wait_for_job(job_id, timeout=REFRESH_TIMEOUT_SECONDS) - except (FileWorkerJobFailed, TimeoutError) as e: - # The files are in place; a later refresh indexes them. - logger.error(f'Failed to refresh synced files: {e}') +# The daily window (the backup drive's; the Controller has its own for the sync). def window_is_due(status: dict, local_now: datetime, start: Optional[str], end: Optional[str], @@ -979,24 +341,3 @@ def window_is_due(status: dict, local_now: datetime, start: Optional[str], end: return (local_now - finished_at).total_seconds() >= repeat_seconds -def sync_is_due(status: dict, local_now: datetime) -> bool: - """True if the sync window is open and the last sync finished long enough ago.""" - config = get_sync_config() - return window_is_due(status, local_now, config.window_start, config.window_end, SYNC_WINDOW_REPEAT_SECONDS) - - -async def sync_tick(): - """Start a sync when one was requested, or when it is due.""" - if not is_secondary(): - return - status = get_sync_status() - if wrol_mode_enabled(): - if status['requested']: - update_sync_status(requested=False, last_skip_reason='WROL Mode is enabled') - return - from wrolpi.downloader import DownloadManager - if not status['requested'] and not sync_is_due(status, DownloadManager._get_local_now()): - return - force = status['force'] - update_sync_status(requested=False) - await run_sync(force=force) diff --git a/wrolpi/sync/notify.py b/wrolpi/sync/notify.py new file mode 100644 index 00000000..f3d5ba2b --- /dev/null +++ b/wrolpi/sync/notify.py @@ -0,0 +1,73 @@ +"""Notifications for a secondary's syncs. + +The Controller runs a sync (controller/lib/sync.py) and stops this API while it copies, so it cannot send the API's +events itself. It records each sync's outcome in `config/sync_state.yaml`; the API reads that record and sends one +event per finished sync. What it has announced is kept in its own file, which a sync never touches, so a restart +(and every sync restarts the API) neither repeats nor loses an announcement. +""" +from typing import Optional + +import yaml + +from wrolpi.common import logger, get_media_directory, is_secondary +from wrolpi.events import Events + +logger = logger.getChild(__name__) + +STATE_FILE = 'config/sync_state.yaml' +# Under config/.sync, which a sync never copies over or deletes. +ANNOUNCED_FILE = 'config/.sync/announced.yaml' +# A sync that was stopped on purpose is not news. +STOPPED = ('The sync was cancelled', 'WROL Mode was enabled') + + +def _read(path: str) -> dict: + try: + data = yaml.safe_load((get_media_directory() / path).read_text()) + except (OSError, yaml.YAMLError): + return {} + return data if isinstance(data, dict) else {} + + +def sync_message(state: dict) -> Optional[tuple]: + """The event a finished sync deserves, as (kind, message), or None.""" + error = state.get('last_error') + if state.get('state') == 'error': + return 'failed', f'Sync from the primary failed: {error}' + if error in STOPPED: + return None + copied, deleted = int(state.get('files_transferred') or 0), int(state.get('files_deleted') or 0) + warnings = state.get('warnings') or [] + if not copied and not deleted and not warnings: + # Within the window a sync runs every 15 minutes; one that changed nothing is not news. + return None + message = f'Synced {copied} files from the primary' + if deleted: + message += f'; {deleted} moved to the trash' + if warnings: + message += f'; {warnings[0]}' + return 'completed', message + + +def notify_tick(): + """Send the event of a sync that finished since the last one announced.""" + if not is_secondary(): + return + state = _read(STATE_FILE) + finished_at = state.get('finished_at') + if not finished_at or state.get('state') == 'running': + return + if _read(ANNOUNCED_FILE).get('finished_at') == finished_at: + return + if found := sync_message(state): + kind, message = found + if kind == 'failed': + Events.send_sync_failed(message) + else: + Events.send_sync_completed(message) + announced = get_media_directory() / ANNOUNCED_FILE + try: + announced.parent.mkdir(parents=True, exist_ok=True) + announced.write_text(yaml.safe_dump(dict(finished_at=finished_at))) + except OSError as e: + logger.warning(f'Could not record the announced sync: {e}') diff --git a/wrolpi/sync/schema.py b/wrolpi/sync/schema.py index e9f9daee..0c1f0ab8 100644 --- a/wrolpi/sync/schema.py +++ b/wrolpi/sync/schema.py @@ -4,20 +4,6 @@ from typing import List, Optional from wrolpi.errors import ValidationError -@dataclass -class ManifestRequest: - after_id: int = 0 - limit: int = 500 - - def __post_init__(self): - self.after_id = int(self.after_id) - self.limit = int(self.limit) - if self.after_id < 0: - raise ValidationError('after_id must not be negative') - if not 1 <= self.limit <= 5000: - raise ValidationError('limit must be between 1 and 5000') - - @dataclass class SyncDirectoriesRequest: # The folder whose sub-folders are listed, relative to the media directory ('' is the media directory). diff --git a/wrolpi/sync/status.py b/wrolpi/sync/status.py index 52f28f5c..7cf89061 100644 --- a/wrolpi/sync/status.py +++ b/wrolpi/sync/status.py @@ -1,53 +1,11 @@ -"""The shared progress records of a secondary WROLPi's sync, `api_app.shared_ctx.status['sync']`, and of a backup to -a second drive, `api_app.shared_ctx.status['backup']`. +"""The shared progress records of a backup to a second drive, `api_app.shared_ctx.status['backup']`, and of a +primary's database snapshot for its secondaries. (A secondary's sync is the Controller's: controller/lib/sync.py.) -They live in a manager dict, so every read returns a copy: change them only with `update_sync_status` and -`update_backup_status`. +They live in a manager dict, so every read returns a copy: change them only with `update_backup_status` and +`update_snapshot_status`. """ -def default_sync_status() -> dict: - return dict( - state='idle', - phase=None, - requested=False, - force=False, - cancel_requested=False, - started_at=None, - finished_at=None, - last_success_at=None, - last_error=None, - last_skip_reason=None, - primary_url=None, - primary_version=None, - groups_seen=0, - max_id=0, - files_checked=0, - files_to_download=0, - files_downloaded=0, - bytes_downloaded=0, - files_failed=0, - files_excluded=0, - configs_imported=[], - disk_full=False, - ) - - -def get_sync_status() -> dict: - from wrolpi.api_utils import api_app - return dict(api_app.shared_ctx.status.get('sync') or default_sync_status()) - - -def update_sync_status(**changes) -> dict: - """Merge `changes` into the shared sync status. Only the perpetual process and the sync endpoints write it, and - the endpoints only set the request/cancel keys, so a lost update can only delay a request by one tick.""" - from wrolpi.api_utils import api_app - status = get_sync_status() - status.update(changes) - api_app.shared_ctx.status['sync'] = status - return status - - def default_snapshot_status() -> dict: return dict( state='idle', @@ -73,7 +31,8 @@ def get_snapshot_status() -> dict: def update_snapshot_status(**changes) -> dict: - """Merge `changes` into the shared snapshot status (see `update_sync_status`).""" + """Merge `changes` into the shared snapshot status. Only the perpetual process and the snapshot endpoint write + it, and the endpoint only sets `requested`, so a lost update can only delay a request by one tick.""" from wrolpi.api_utils import api_app status = get_snapshot_status() status.update(changes) @@ -106,7 +65,7 @@ def get_backup_status() -> dict: def update_backup_status(**changes) -> dict: - """Merge `changes` into the shared backup status (see `update_sync_status`).""" + """Merge `changes` into the shared backup status (see `update_snapshot_status`).""" from wrolpi.api_utils import api_app status = get_backup_status() status.update(changes) diff --git a/wrolpi/sync/test/test_api.py b/wrolpi/sync/test/test_api.py index ebb1fb6f..1ac92f36 100644 --- a/wrolpi/sync/test/test_api.py +++ b/wrolpi/sync/test/test_api.py @@ -5,38 +5,22 @@ from http import HTTPStatus import pytest import yaml -from wrolpi import flags -from wrolpi.api_utils import api_app from wrolpi.common import get_wrolpi_config, enable_wrol_mode from wrolpi.downloader import download_manager -from wrolpi.sync import api as sync_api, backup, lib +from wrolpi.sync import api as sync_api, backup, notify, snapshot from wrolpi.sync.config import get_sync_config -from wrolpi.sync.status import get_sync_status, update_sync_status @pytest.mark.asyncio -async def test_info_and_manifest(async_client, test_session, test_wrolpi_config, make_files_structure, - refresh_files): - make_files_structure({'docs/a.txt': 'a', 'docs/b.txt': 'b'}) - await refresh_files() - +async def test_info(async_client, test_session, test_wrolpi_config): request, response = await async_client.get('/api/sync/info') assert response.status_code == HTTPStatus.OK + assert set(response.json) == {'version', 'role', 'media_directory', 'database_revision', 'media_mounted'} assert response.json['role'] == 'primary' - assert response.json['max_file_group_id'] > 0 - request, response = await async_client.get('/api/sync/manifest?after_id=0&limit=1') - assert response.status_code == HTTPStatus.OK - assert len(response.json['groups']) == 1 - after_id = response.json['next_after_id'] - assert after_id - - request, response = await async_client.get(f'/api/sync/manifest?after_id={after_id}&limit=1') - paths = [p['path'] for g in response.json['groups'] for p in g['paths']] - assert len(paths) == 1 and paths[0].startswith('docs/') - - request, response = await async_client.get('/api/sync/manifest?limit=0') - assert response.status_code == HTTPStatus.BAD_REQUEST + # The old engine's manifest is gone; a secondary copies with rsync. + request, response = await async_client.get('/api/sync/manifest') + assert response.status_code == HTTPStatus.NOT_FOUND class FakeControllerResponse: @@ -91,19 +75,19 @@ async def test_run(async_client, test_wrolpi_config, make_secondary, monkeypatch @pytest.mark.asyncio -async def test_old_engine_does_not_run(async_client, test_wrolpi_config, make_secondary, monkeypatch): - """Only the Controller syncs now; the API's loop runs the backup drive and nothing else.""" - make_secondary('https://primary.example') - lib.request_sync() +async def test_worker_runs_the_backup_and_the_snapshot(async_client, test_wrolpi_config, monkeypatch): + """The API's loop makes the backup and the primary's database snapshot, and announces the syncs the Controller + finished; a secondary's sync itself is the Controller's.""" called = [] async def tick(name): called.append(name) - monkeypatch.setattr(lib, 'sync_tick', lambda: tick('sync')) monkeypatch.setattr(backup, 'backup_tick', lambda: tick('backup')) + monkeypatch.setattr(snapshot, 'snapshot_tick', lambda: tick('snapshot')) + monkeypatch.setattr(notify, 'notify_tick', lambda: called.append('notify')) await sync_api.perpetual_sync_worker() - assert called == ['backup'] + assert called == ['backup', 'snapshot', 'notify'] @pytest.mark.asyncio @@ -215,56 +199,14 @@ async def test_directories_primary_unreachable(async_client, test_wrolpi_config, @pytest.mark.asyncio -async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_lock, make_secondary, monkeypatch): - """While a secondary syncs, its API refuses everything but the sync's own progress.""" +async def test_wrol_mode_refuses_a_sync(async_client, test_wrolpi_config, test_download_manager, make_secondary, + monkeypatch): + """WROL Mode means the secondary is in use: no sync is asked of the Controller. (The Controller itself also + refuses, and stops a sync at its next step, when it finds WROL Mode enabled.)""" make_secondary() - with lib.lock_api(): - request, response = await async_client.get('/api/files/search?search_str=foo') - assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE - assert response.json['code'] == 'SYNCING' - request, response = await async_client.post('/api/tag', content=json.dumps(dict(name='one'))) - assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE - - for path in ('/api/status', '/api/sync/status', '/api/settings', '/api/echo'): - request, response = await async_client.get(path) - assert response.status_code == HTTPStatus.OK, path - fake_controller(monkeypatch, status=HTTPStatus.ACCEPTED) - request, response = await async_client.post('/api/sync/cancel') - assert response.status_code == HTTPStatus.ACCEPTED - # Nothing may change the sync while it runs, except stopping it. - request, response = await async_client.put('/api/sync/settings', content=json.dumps(dict(role='primary'))) - assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE - request, response = await async_client.post('/api/sync/run', content='{}') - assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE - - request, response = await async_client.get('/api/status') - assert response.json['flags']['sync_running'] is True - - request, response = await async_client.get('/api/settings') - assert response.status_code == HTTPStatus.OK - assert not api_app.shared_ctx.sync_api_locked.is_set() - assert not flags.sync_running.is_set() - - -@pytest.mark.asyncio -async def test_stale_lock_event_does_not_lock(async_client, test_wrolpi_config, flags_lock): - """The Event outlives a perpetual process that died mid-sync; the cleared Flag unlocks the API.""" - api_app.shared_ctx.sync_api_locked.set() - try: - request, response = await async_client.get('/api/settings') - assert response.status_code == HTTPStatus.OK - finally: - api_app.shared_ctx.sync_api_locked.clear() - - -@pytest.mark.asyncio -async def test_wrol_mode_cancels_sync(async_client, test_wrolpi_config, test_download_manager, make_secondary): - """WROL Mode means the secondary is in use: a running sync stops, and none start.""" - make_secondary() - update_sync_status(state='running') + calls = fake_controller(monkeypatch, status=HTTPStatus.ACCEPTED) enable_wrol_mode() - assert get_sync_status()['cancel_requested'] is True - assert lib.sync_skip_reason() == 'WROL Mode is enabled' request, response = await async_client.post('/api/sync/run', content='{}') assert response.status_code == HTTPStatus.FORBIDDEN + assert calls == [] diff --git a/wrolpi/sync/test/test_backup.py b/wrolpi/sync/test/test_backup.py index ba75143a..4d41c916 100644 --- a/wrolpi/sync/test/test_backup.py +++ b/wrolpi/sync/test/test_backup.py @@ -403,18 +403,15 @@ def test_backup_is_due_hourly_in_window(test_wrolpi_config): def test_backup_and_sync_windows_are_separate(test_wrolpi_config): - """The sync window (a secondary's sync) does not start backups, and the backup window does not start syncs.""" - from wrolpi.sync.lib import sync_is_due + """The sync window (a secondary's sync, which the Controller schedules) does not start backups.""" local_now = datetime(2026, 11, 5, 3, 0, tzinfo=timezone.utc) never = dict(finished_at=None) get_sync_config()._config.update(window_start='01:00', window_end='05:00') - assert sync_is_due(never, local_now) assert not backup.backup_is_due(never, local_now) get_sync_config()._config.update(window_start=None, window_end=None, backup_window_start='02:00', backup_window_end='04:00') - assert not sync_is_due(never, local_now) assert backup.backup_is_due(never, local_now) diff --git a/wrolpi/sync/test/test_engine.py b/wrolpi/sync/test/test_engine.py deleted file mode 100644 index fc5f886c..00000000 --- a/wrolpi/sync/test/test_engine.py +++ /dev/null @@ -1,698 +0,0 @@ -import os -import pathlib -from collections import namedtuple - -import pytest -import yaml - -from wrolpi.collections.models import Collection -from wrolpi.collections.sync import get_playlists_directory, _safe_unlink_managed -from wrolpi.common import get_wrolpi_config, set_test_media_directory -from wrolpi.db import get_db_session -from wrolpi.errors import InvalidConfig -from wrolpi.files.models import FileGroup -from wrolpi.sync import lib -from wrolpi.sync.config import get_sync_config -from wrolpi.sync.conftest import manifest_page -from wrolpi.sync.status import get_sync_status, update_sync_status -from wrolpi.tags import get_tags_config, Tag, TagFile, _delete_extra_tags_directory_paths - - -@pytest.mark.parametrize('path, allowed', [ - ('videos/channel/video.mp4', True), - ('a.txt', True), - ('../escape.txt', False), - ('videos/../../escape.txt', False), - ('/etc/passwd', False), - ('videos/.hidden/video.mp4', False), - ('.hidden.txt', False), - ('config/wrolpi.db', False), - ('config/tags.yaml', False), - ('tags/one/video.mp4', False), - # A drive that ignores case (exFAT, NTFS, macOS) would write these into the real config and tags folders. - ('Config/wrolpi.db', False), - ('CONFIG/tags.yaml', False), - ('Tags/one/video.mp4', False), - ('videos\\video.mp4', False), - ('', False), -]) -def test_resolve_destination(test_directory, path, allowed): - destination = lib.resolve_destination(test_directory, path) - if allowed: - assert destination == test_directory / path - else: - assert destination is None - - -def test_resolve_destination_refuses_symlinks(test_directory): - """A sync never writes through a symlink; it could point outside the media directory.""" - outside = test_directory / 'outside' - outside.mkdir() - (test_directory / 'videos').symlink_to(outside) - assert lib.resolve_destination(test_directory, 'videos/video.mp4') is None - - (test_directory / 'docs').mkdir() - (test_directory / 'docs/link.txt').symlink_to(outside / 'target.txt') - assert lib.resolve_destination(test_directory, 'docs/link.txt') is None - - -def test_needs_download(test_directory): - path = test_directory / 'a.txt' - file = lib.ManifestFile('a.txt', 5, 1_700_000_000.0) - assert lib.needs_download(path, file), 'a missing file is downloaded' - - path.write_text('12345') - os.utime(path, (file.mtime, file.mtime)) - assert not lib.needs_download(path, file) - - os.utime(path, (file.mtime + 1.5, file.mtime + 1.5)) - assert not lib.needs_download(path, file), 'within the 2 second tolerance of exFAT' - - os.utime(path, (file.mtime + 3, file.mtime + 3)) - assert lib.needs_download(path, file), 'a different modification time is downloaded' - - os.utime(path, (file.mtime, file.mtime)) - path.write_text('1234') - os.utime(path, (file.mtime, file.mtime)) - assert lib.needs_download(path, file), 'a different size is downloaded' - - -Primary = namedtuple('Primary', 'directory url state') - - -@pytest.fixture -def primary(test_directory, secondary_directory, primary_media_server) -> Primary: - """A primary whose media is `test_directory`, served like Caddy.""" - url, state = primary_media_server(test_directory) - return Primary(test_directory, url, state) - - -def write_primary_file(primary: Primary, path: str, content: bytes, mtime: float = 1_700_000_000.0) \ - -> lib.ManifestFile: - file = primary.directory / path - file.parent.mkdir(parents=True, exist_ok=True) - file.write_bytes(content) - os.utime(file, (mtime, mtime)) - return lib.ManifestFile(path, len(content), mtime) - - -@pytest.mark.asyncio -async def test_download_file(primary, secondary_directory): - """A downloaded file has the primary's bytes and modification time, and no partial is left.""" - file = write_primary_file(primary, 'videos/channel/video name.mp4', b'video bytes' * 1000) - destination = secondary_directory / file.path - - async with lib.PrimaryClient(primary.url) as client: - received = await lib.download_file(client, file, destination) - - assert received == file.size - assert destination.read_bytes() == (primary.directory / file.path).read_bytes() - assert destination.stat().st_mtime == file.mtime - assert not lib.partial_path(destination).exists() - assert not lib.needs_download(destination, file) - - -@pytest.mark.asyncio -async def test_download_file_resumes(primary, secondary_directory): - """A partial left by an interrupted download is resumed with a Range request.""" - content = bytes(range(256)) * 400 - file = write_primary_file(primary, 'big.bin', content) - destination = secondary_directory / file.path - lib.partial_path(destination).write_bytes(content[:30_000]) - - async with lib.PrimaryClient(primary.url) as client: - received = await lib.download_file(client, file, destination) - - assert received == len(content) - 30_000 - assert ('/media/big.bin', 'bytes=30000-') in primary.state['requests'] - assert destination.read_bytes() == content - - -@pytest.mark.asyncio -async def test_download_file_restarts_without_range(primary, secondary_directory): - """A server that ignores Range sends the whole file; the partial is replaced, not appended to.""" - primary.state['honor_range'] = False - content = b'abcdefghij' * 100 - file = write_primary_file(primary, 'a.bin', content) - destination = secondary_directory / file.path - lib.partial_path(destination).write_bytes(content[:500]) - - async with lib.PrimaryClient(primary.url) as client: - await lib.download_file(client, file, destination) - - assert destination.read_bytes() == content - - -@pytest.mark.asyncio -async def test_download_file_changed_on_primary(primary, secondary_directory): - """A file whose size changed after the manifest was read is not written; the next sync gets it.""" - file = write_primary_file(primary, 'a.txt', b'old') - (primary.directory / 'a.txt').write_bytes(b'new and longer') - destination = secondary_directory / file.path - - async with lib.PrimaryClient(primary.url) as client: - with pytest.raises(lib.SyncError): - await lib.download_file(client, file, destination) - - assert not destination.exists() - - -@pytest.mark.asyncio -async def test_download_file_larger_than_manifest(primary, secondary_directory): - """A primary that sends more bytes than its manifest said (no Content-Length to catch it) is not written past - the manifest's size, which is what the free-space check allowed for.""" - primary.state['omit_length'] = True - write_primary_file(primary, 'a.bin', b'x' * 5 * lib.DOWNLOAD_CHUNK_SIZE) - file = lib.ManifestFile('a.bin', 3000, 1_700_000_000.0) - destination = secondary_directory / file.path - - async with lib.PrimaryClient(primary.url) as client: - with pytest.raises(lib.SyncError, match='larger than the primary said'): - await lib.download_file(client, file, destination) - - assert not destination.exists() - assert not lib.partial_path(destination).exists() - - -@pytest.mark.asyncio -async def test_download_file_disk_full(primary, secondary_directory, monkeypatch): - """A file that would leave less than the minimum free space is not downloaded; its partial is kept.""" - file = write_primary_file(primary, 'a.bin', b'x' * 1000) - destination = secondary_directory / file.path - destination.parent.mkdir(parents=True, exist_ok=True) - lib.partial_path(destination).write_bytes(b'x' * 10) - - usage = namedtuple('usage', 'total used free') - monkeypatch.setattr(lib.shutil, 'disk_usage', lambda _: usage(10, 10, lib.MIN_FREE_BYTES)) - - async with lib.PrimaryClient(primary.url) as client: - with pytest.raises(lib.SyncDiskFull): - await lib.download_file(client, file, destination) - - assert lib.partial_path(destination).read_bytes() == b'x' * 10 - assert primary.state['requests'] == [] - - -def capture_manifest() -> list: - pages = list() - after_id = 0 - with get_db_session() as session: - while True: - page = lib.build_manifest_page(session, after_id, limit=2) - pages.append(page) - if page['next_after_id'] is None: - return pages - after_id = page['next_after_id'] - - -@pytest.mark.asyncio -async def test_run_sync(async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api, - refresh_files, tag_factory, test_tags_config, events_fixture, await_switches): - """A secondary gets the primary's files, then its tags.yaml, and its FileGroups are tagged like the primary's.""" - write_primary_file(primary, 'docs/notes.txt', b'some notes', mtime=1_700_000_001.0) - write_primary_file(primary, 'docs/more notes.txt', b'more notes', mtime=1_700_000_002.0) - write_primary_file(primary, 'videos/channel/video.mp4', b'not really a video', mtime=1_700_000_003.0) - write_primary_file(primary, 'videos/channel/video.info.json', b'{}', mtime=1_700_000_004.0) - await refresh_files() - - tag = await tag_factory('one') - notes = test_session.query(FileGroup).filter_by(primary_path=primary.directory / 'docs/notes.txt').one() - notes.add_tag(test_session, tag.id) - test_session.commit() - get_tags_config().dump_config() - await await_switches() - primary_tags = (primary.directory / 'config/tags.yaml').read_bytes() - assert b'docs/notes.txt' in primary_tags - - pages = capture_manifest() - assert len(pages) >= 2, 'the manifest is paged' - fake_primary_api(pages, primary.directory) - - # The secondary is another machine with its own (empty) database. - test_session.query(TagFile).delete() - test_session.query(Tag).delete() - test_session.query(FileGroup).delete() - test_session.commit() - - # This WROLPi becomes the secondary; it has its own (empty) tags.yaml. - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - (secondary_directory / 'config/tags.yaml').write_text('tags: {}\ntag_files: []\nversion: 1\n') - - status = await lib.run_sync() - - assert status['state'] == 'idle', status['last_error'] - assert status['files_failed'] == 0 - assert status['files_downloaded'] == 4 - assert status['last_success_at'] - for path in ('docs/notes.txt', 'docs/more notes.txt', 'videos/channel/video.mp4', - 'videos/channel/video.info.json'): - copy = secondary_directory / path - assert copy.read_bytes() == (primary.directory / path).read_bytes() - assert copy.stat().st_mtime == (primary.directory / path).stat().st_mtime - - # The secondary's own tags.yaml is kept in today's backup. - backups = list((secondary_directory / 'config/backup').glob('tags-*.yaml')) - assert len(backups) == 1 and b'docs/notes.txt' not in backups[0].read_bytes() - - # The primary's tags.yaml is copied byte for byte, and imported. - assert (secondary_directory / 'config/tags.yaml').read_bytes() == primary_tags - assert 'tags.yaml' in status['configs_imported'] - with get_db_session() as session: - copy = session.query(FileGroup).filter_by(primary_path=secondary_directory / 'docs/notes.txt').one() - assert copy.tag_names == ['one'] - # Files of a group arrive together, and are grouped again on the secondary. - video = session.query(FileGroup).filter_by(directory=secondary_directory / 'videos/channel').one() - assert {pathlib.Path(i['path']).name for i in video.files} == {'video.mp4', 'video.info.json'} - - events_fixture.assert_has_event('sync_completed') - - # Nothing changed, so nothing is downloaded or imported again. - primary.state['requests'].clear() - status = await lib.run_sync() - assert status['files_to_download'] == 0 - assert status['configs_imported'] == [] - assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')] - - -@pytest.mark.asyncio -async def test_run_sync_replaced_file_keeps_its_links(async_client, primary, secondary_directory, make_secondary, - fake_primary_api): - """The Tags Directory and playlists links of a file the sync replaces are links to the new version. A link to - the old version would be the old version's last copy, which neither directory ever deletes.""" - changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) - same = write_primary_file(primary, 'docs/same.txt', b'unchanged') - fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime), (same.path, same.size, same.mtime))], - primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - # The secondary has an older version of one file, linked into a tag and a playlist. - notes = secondary_directory / 'docs/notes.txt' - unchanged = secondary_directory / 'docs/same.txt' - notes.parent.mkdir(parents=True) - notes.write_bytes(b'old notes') - unchanged.write_bytes(b'unchanged') - os.utime(unchanged, (same.mtime, same.mtime)) - tag_link = secondary_directory / 'tags/one/notes.txt' - playlist_link = get_playlists_directory() / 'Reading/0001_notes.txt' - unchanged_link = secondary_directory / 'tags/one/same.txt' - # The only copy of a file that someone put in the Tags Directory themselves. - stranger = secondary_directory / 'tags/one/stranger.txt' - for source, link in ((notes, tag_link), (notes, playlist_link), (unchanged, unchanged_link)): - link.parent.mkdir(parents=True, exist_ok=True) - link.hardlink_to(source) - stranger.write_bytes(b'old notes') - - status = await lib.run_sync() - - assert status['state'] == 'idle', status['last_error'] - assert status['files_downloaded'] == 1 - assert notes.read_bytes() == b'the new notes' - for link in (tag_link, playlist_link): - assert link.samefile(notes), f'{link} is still the old version' - assert notes.stat().st_nlink == 3 - assert unchanged_link.samefile(unchanged) - assert stranger.read_bytes() == b'old notes' and stranger.stat().st_nlink == 1 - assert not [i for i in secondary_directory.rglob('.*') if i.is_file()], 'no temporary file is left behind' - - # The links are no longer wanted (the tag was removed, the playlist reordered): both directories can delete them. - _delete_extra_tags_directory_paths(secondary_directory / 'tags', [unchanged_link, stranger]) - assert not tag_link.exists() - assert _safe_unlink_managed(playlist_link) - assert notes.read_bytes() == b'the new notes' and notes.stat().st_nlink == 1 - - -@pytest.mark.asyncio -async def test_run_sync_replaced_file_in_a_playlist_with_its_own_directory( - async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api): - """A playlist may keep its links in any folder of the library; a replaced file's link there is moved too, and - the folder's other files are left alone.""" - changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) - fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - notes = secondary_directory / 'docs/notes.txt' - notes.parent.mkdir(parents=True) - notes.write_bytes(b'old notes') - directory = secondary_directory / 'my lists/Reading' - (directory / 'deeper').mkdir(parents=True) - link = directory / '0001_notes.txt' - link.hardlink_to(notes) - # A hard link of the file that the playlist did not make: not its name, or not directly in its folder. - renamed = directory / 'my copy of the notes.txt' - deeper = directory / 'deeper/0001_notes.txt' - renamed.hardlink_to(notes) - deeper.hardlink_to(notes) - test_session.add(Collection(name='Reading', kind='playlist', directory=directory)) - test_session.commit() - - status = await lib.run_sync() - - assert status['state'] == 'idle', status['last_error'] - assert link.samefile(notes) and link.read_bytes() == b'the new notes' - assert renamed.read_bytes() == b'old notes' and deeper.read_bytes() == b'old notes' - assert _safe_unlink_managed(link) - - -@pytest.mark.asyncio -async def test_run_sync_replaced_file_with_a_hard_linked_twin(async_client, primary, secondary_directory, - make_secondary, fake_primary_api): - """Two library files may be hard links of each other. When one of them changes, only links named for it move; - its twin's link is still its twin.""" - changed = write_primary_file(primary, 'docs/a.txt', b'the new a', mtime=1_700_000_500.0) - fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - a, b = secondary_directory / 'docs/a.txt', secondary_directory / 'docs/b.txt' - a.parent.mkdir(parents=True) - a.write_bytes(b'shared') - b.hardlink_to(a) - a_link, b_link = secondary_directory / 'tags/one/a.txt', secondary_directory / 'tags/one/b.txt' - a_link.parent.mkdir(parents=True) - a_link.hardlink_to(a) - b_link.hardlink_to(b) - - status = await lib.run_sync() - - assert status['state'] == 'idle', status['last_error'] - assert a.read_bytes() == b'the new a' and a_link.samefile(a) - assert b.read_bytes() == b'shared' and b_link.samefile(b) - - -@pytest.mark.asyncio -async def test_run_sync_interrupted_before_a_file_is_replaced(async_client, primary, secondary_directory, - make_secondary, fake_primary_api, monkeypatch): - """A sync that stops between downloading a new version and putting it in place has left no link on the old - version once a later sync finishes the job, although that sync downloads nothing.""" - changed = write_primary_file(primary, 'docs/notes.txt', b'the new notes', mtime=1_700_000_500.0) - fake_primary_api([manifest_page((changed.path, changed.size, changed.mtime))], primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - notes = secondary_directory / 'docs/notes.txt' - notes.parent.mkdir(parents=True) - notes.write_bytes(b'old notes') - link = secondary_directory / 'tags/one/notes.txt' - link.parent.mkdir(parents=True) - link.hardlink_to(notes) - - replace = os.replace - - def power_cut(source, destination): - if pathlib.Path(destination) == notes: - raise OSError('the power went out') - return replace(source, destination) - - monkeypatch.setattr(lib.os, 'replace', power_cut) - status = await lib.run_sync() - assert status['files_failed'] == 1 - assert notes.read_bytes() == b'old notes', 'the file was not replaced' - monkeypatch.setattr(lib.os, 'replace', replace) - - primary.state['requests'].clear() - status = await lib.run_sync() - - assert status['state'] == 'idle', status['last_error'] - assert status['files_downloaded'] == 1 and status['bytes_downloaded'] == 0 - assert notes.read_bytes() == b'the new notes' - assert link.samefile(notes) and notes.stat().st_nlink == 2 - assert not [i for i in secondary_directory.rglob('.*') if i.is_file()] - - -def test_move_links_of_a_long_name(test_session, test_directory): - """A link whose name is as long as a file name may be is still moved.""" - name = 'n' * 240 + '.txt' - old, new = test_directory / 'docs' / name, test_directory / 'docs/new version' - old.parent.mkdir() - old.write_bytes(b'old') - new.write_bytes(b'new') - link = get_playlists_directory() / 'Reading' / f'0001_{name}' - link.parent.mkdir(parents=True) - link.hardlink_to(old) - - assert lib.find_links([old]) == {old: [link]} - lib.move_links([link], old, new) - - assert link.samefile(new) - assert [i.name for i in link.parent.iterdir()] == [link.name] - - -@pytest.mark.asyncio -async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary, - fake_primary_api): - """A manifest cannot make a secondary write outside its library, or over its configs and database.""" - good = write_primary_file(primary, 'good.txt', b'good') - write_primary_file(primary, 'config/wrolpi.db', b'not a database') - pages = [manifest_page( - (good.path, good.size, good.mtime), - ('config/wrolpi.db', 14, 1_700_000_000.0), - ('../escape.txt', 4, 1_700_000_000.0), - ('tags/one/good.txt', 4, 1_700_000_000.0), - )] - fake_primary_api(pages, primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - status = await lib.run_sync() - - assert status['files_failed'] == 3 - assert (secondary_directory / 'good.txt').read_bytes() == b'good' - assert not (secondary_directory / 'config/wrolpi.db').exists() - assert not (secondary_directory.parent / 'escape.txt').exists() - assert not (secondary_directory / 'tags').exists() - - -@pytest.mark.asyncio -async def test_run_sync_excluded_directories(async_client, primary, secondary_directory, make_secondary, - fake_primary_api): - """A secondary never downloads files inside its excluded folders, and keeps what it already has in them.""" - wanted = [write_primary_file(primary, i, b'wanted') for i in ('Movies2/a.mp4', 'docs/Movies/b.txt', 'c.txt')] - excluded = [write_primary_file(primary, i, b'excluded') for i in ('Movies/a.mp4', 'Movies/deep/b.mp4', - 'Movies [HD]/c.mp4')] - pages = [manifest_page(*((i.path, i.size, i.mtime) for i in wanted + excluded))] - fake_primary_api(pages, primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - get_sync_config()._config['sync_excluded_directories'] = ['Movies', 'Movies [HD]'] - # Copied before the folder was excluded. - (secondary_directory / 'Movies').mkdir() - (secondary_directory / 'Movies/old.mp4').write_bytes(b'old') - - status = await lib.run_sync() - - assert status['last_error'] is None - assert status['files_excluded'] == 3 - assert status['files_downloaded'] == 3 - for file in wanted: - assert (secondary_directory / file.path).read_bytes() == b'wanted' - for file in excluded: - assert not (secondary_directory / file.path).exists() - assert (secondary_directory / 'Movies/old.mp4').read_bytes() == b'old', 'nothing is deleted' - - -@pytest.mark.parametrize('values, expected', [ - ([], []), - (['Movies'], ['Movies']), - (['/Movies/', ' Movies ', 'Movies//'], ['Movies']), - (['videos/Movies', 'videos'], ['videos']), - (['Movies', 'Movies2', 'Movies/deep'], ['Movies', 'Movies2']), - (['a//b'], ['a/b']), - (['Movies [HD]'], ['Movies [HD]']), -]) -def test_normalize_excluded_directories(values, expected): - assert lib.normalize_excluded_directories(values) == expected - - -@pytest.mark.parametrize('value', ['', '/', '..', 'a/../b', '../a', 'a/./b', '.hidden', 'a/.hidden', 'config', - 'config/x', 'tags', 'Config', 'TAGS/x', 'a\\b', None]) -def test_normalize_excluded_directories_rejects(value): - with pytest.raises(InvalidConfig): - lib.normalize_excluded_directories([value]) - - -def test_is_excluded(): - assert lib.is_excluded('Movies/a.mp4', {'Movies'}) - assert lib.is_excluded('Movies', {'Movies'}) - assert not lib.is_excluded('Movies2/a.mp4', {'Movies'}) - assert not lib.is_excluded('docs/Movies/a.mp4', {'Movies'}) - assert lib.is_excluded('docs/Movies/a.mp4', {'docs/Movies'}) - - -def test_list_local_directories(test_directory): - for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'zims', 'lost+found', 'Tags'): - try: - (test_directory / i).mkdir(parents=True) - except FileExistsError: - pass # 'Tags' is 'tags' on a filesystem that ignores case. - (test_directory / 'file.txt').write_text('a file') - (test_directory / 'link').symlink_to(test_directory / 'Movies') - - assert lib.list_local_directories('') == [ - dict(name='Movies', path='Movies'), dict(name='videos', path='videos'), dict(name='zims', path='zims')] - assert lib.list_local_directories('/videos/') == [dict(name='channel', path='videos/channel')] - for path in ('../', 'videos/../..', 'file.txt', 'missing', 'link'): - with pytest.raises(InvalidConfig): - lib.list_local_directories(path) - - -@pytest.mark.asyncio -async def test_list_primary_directories(primary, secondary_directory, make_secondary): - for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'): - (primary.directory / i).mkdir(parents=True) - (primary.directory / 'file.txt').write_text('a file') - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - assert await lib.list_primary_directories('') == [ - dict(name='Movies', path='Movies'), dict(name='videos', path='videos')] - assert await lib.list_primary_directories('videos') == [dict(name='channel', path='videos/channel')] - with pytest.raises(InvalidConfig): - await lib.list_primary_directories('../etc') - - -@pytest.mark.asyncio -async def test_run_sync_only_on_a_secondary(async_client, test_wrolpi_config, fake_primary_api): - status = await lib.run_sync() - assert status['state'] == 'idle' - assert status['last_skip_reason'] == 'This WROLPi is not a secondary' - assert status['started_at'] is None - - -@pytest.mark.asyncio -async def test_run_sync_refuses_a_secondary_primary(async_client, primary, secondary_directory, make_secondary, - fake_primary_api): - """Secondaries do not chain; a secondary's library may be stale.""" - fake_primary_api([manifest_page()], primary.directory, role='secondary') - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - status = await lib.run_sync() - assert status['state'] == 'error' - assert 'is not a primary' in status['last_error'] - - -@pytest.mark.asyncio -async def test_run_sync_cancel(async_client, primary, secondary_directory, make_secondary, fake_primary_api): - """A cancelled sync stops between pages; the files of finished pages stay.""" - first = write_primary_file(primary, 'first.txt', b'first') - second = write_primary_file(primary, 'second.txt', b'second') - pages = [manifest_page((first.path, first.size, first.mtime), next_after_id=1, max_id=2), - manifest_page((second.path, second.size, second.mtime), max_id=2)] - fake_primary_api(pages, primary.directory) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - original = lib._refresh - - async def refresh_then_cancel(paths): - await original(paths) - lib.request_sync_cancel() - - lib._refresh = refresh_then_cancel - try: - status = await lib.run_sync() - finally: - lib._refresh = original - - assert status['last_error'] == 'The sync was cancelled' - assert status['state'] == 'idle' - assert (secondary_directory / 'first.txt').exists() - assert not (secondary_directory / 'second.txt').exists() - - -@pytest.mark.asyncio -async def test_sync_configs_wrolpi_yaml(async_client, primary, secondary_directory, make_secondary): - """Only the library settings of the primary's wrolpi.yaml are applied; the secondary keeps its own role, hotspot, - and the like.""" - primary_config = dict( - videos_destination='my videos/%(channel_name)s', - ignored_directories=['config', 'tags', str(primary.directory / 'private'), '/elsewhere/directory'], - hotspot_ssid='primary hotspot', - role='primary', - nav_color='red', - version=12, - ) - (primary.directory / 'config').mkdir(exist_ok=True) - (primary.directory / 'config/wrolpi.yaml').write_text(yaml.dump(primary_config)) - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - async with lib.PrimaryClient(primary.url) as client: - await lib.sync_configs(client, str(primary.directory), reimport=False) - - config = get_wrolpi_config() - assert config.videos_destination == 'my videos/%(channel_name)s' - assert config.ignored_directories == ['config', 'tags', 'private'] - assert get_sync_config().role == 'secondary', 'the role is in sync.yaml, which is never synced' - assert config.hotspot_ssid == 'WROLPi' - assert config.nav_color == 'violet' - - -@pytest.mark.asyncio -async def test_sync_configs_inventories(async_client, primary, secondary_directory, make_secondary): - """Inventories are a directory of configs; each one is copied.""" - from modules.inventory.common import get_inventory_configs - - inventory_directory = primary.directory / 'config/inventory' - inventory_directory.mkdir(parents=True) - pantry = dict(version=3, slug='pantry', name='Pantry', type='food', fields=[], items=[]) - (inventory_directory / 'pantry.yaml').write_text(yaml.dump(pantry)) - (inventory_directory / '.hidden.yaml').write_text('ignored') - set_test_media_directory(secondary_directory) - make_secondary(primary.url) - - async with lib.PrimaryClient(primary.url) as client: - changed = await lib._sync_inventory_directory(client) - - assert changed - copy = get_inventory_configs().get_directory() / 'pantry.yaml' - assert yaml.safe_load(copy.read_text())['name'] == 'Pantry' - assert not (copy.parent / '.hidden.yaml').exists() - - -@pytest.mark.asyncio -async def test_sync_tick(async_client, test_wrolpi_config, make_secondary, monkeypatch): - """A requested sync runs; WROL Mode drops the request.""" - calls = list() - - async def fake_run_sync(force=False): - calls.append(force) - - monkeypatch.setattr(lib, 'run_sync', fake_run_sync) - - await lib.sync_tick() - assert calls == [], 'a primary never syncs' - - make_secondary() - await lib.sync_tick() - assert calls == [], 'nothing requested, no window' - - lib.request_sync(force=True) - await lib.sync_tick() - assert calls == [True] - assert get_sync_status()['requested'] is False - - get_wrolpi_config()._config['wrol_mode'] = True - lib.request_sync() - await lib.sync_tick() - assert calls == [True] - assert get_sync_status()['requested'] is False - assert get_sync_status()['last_skip_reason'] == 'WROL Mode is enabled' - - -def test_request_sync_cancel(async_client): - update_sync_status(state='running') - lib.request_sync_cancel() - assert get_sync_status()['cancel_requested'] is True - - update_sync_status(state='idle', cancel_requested=False, requested=True) - lib.request_sync_cancel() - assert get_sync_status()['requested'] is False - assert get_sync_status()['cancel_requested'] is False - - -def test_partial_path(): - assert lib.partial_path(pathlib.Path('/a/b/video.mp4')) == pathlib.Path('/a/b/.video.mp4.partial') diff --git a/wrolpi/sync/test/test_lib.py b/wrolpi/sync/test/test_lib.py new file mode 100644 index 00000000..7630070a --- /dev/null +++ b/wrolpi/sync/test/test_lib.py @@ -0,0 +1,116 @@ +"""What is left of the API's side of a secondary's sync: its settings, and the primary's folders it may exclude. +The sync itself is the Controller's (controller/test/test_sync.py).""" +from collections import namedtuple + +import pytest + +from wrolpi.common import set_test_media_directory +from wrolpi.errors import InvalidConfig +from wrolpi.sync import lib + +@pytest.mark.parametrize('path, allowed', [ + ('videos/channel/video.mp4', True), + ('a.txt', True), + ('../escape.txt', False), + ('videos/../../escape.txt', False), + ('/etc/passwd', False), + ('videos/.hidden/video.mp4', False), + ('.hidden.txt', False), + ('config/wrolpi.db', False), + ('config/tags.yaml', False), + ('tags/one/video.mp4', False), + # A drive that ignores case (exFAT, NTFS, macOS) would write these into the real config and tags folders. + ('Config/wrolpi.db', False), + ('CONFIG/tags.yaml', False), + ('Tags/one/video.mp4', False), + ('videos\\video.mp4', False), + ('', False), +]) +def test_resolve_destination(test_directory, path, allowed): + destination = lib.resolve_destination(test_directory, path) + if allowed: + assert destination == test_directory / path + else: + assert destination is None + + +def test_resolve_destination_refuses_symlinks(test_directory): + """A sync never writes through a symlink; it could point outside the media directory.""" + outside = test_directory / 'outside' + outside.mkdir() + (test_directory / 'videos').symlink_to(outside) + assert lib.resolve_destination(test_directory, 'videos/video.mp4') is None + + (test_directory / 'docs').mkdir() + (test_directory / 'docs/link.txt').symlink_to(outside / 'target.txt') + assert lib.resolve_destination(test_directory, 'docs/link.txt') is None + + +Primary = namedtuple('Primary', 'directory url state') + + +@pytest.fixture +def primary(test_directory, secondary_directory, primary_media_server) -> Primary: + """A primary whose media is `test_directory`, served like Caddy.""" + url, state = primary_media_server(test_directory) + return Primary(test_directory, url, state) + + +@pytest.mark.parametrize('values, expected', [ + ([], []), + (['Movies'], ['Movies']), + (['/Movies/', ' Movies ', 'Movies//'], ['Movies']), + (['videos/Movies', 'videos'], ['videos']), + (['Movies', 'Movies2', 'Movies/deep'], ['Movies', 'Movies2']), + (['a//b'], ['a/b']), + (['Movies [HD]'], ['Movies [HD]']), +]) +def test_normalize_excluded_directories(values, expected): + assert lib.normalize_excluded_directories(values) == expected + + +@pytest.mark.parametrize('value', ['', '/', '..', 'a/../b', '../a', 'a/./b', '.hidden', 'a/.hidden', 'config', + 'config/x', 'tags', 'Config', 'TAGS/x', 'a\\b', None]) +def test_normalize_excluded_directories_rejects(value): + with pytest.raises(InvalidConfig): + lib.normalize_excluded_directories([value]) + + +def test_is_excluded(): + assert lib.is_excluded('Movies/a.mp4', {'Movies'}) + assert lib.is_excluded('Movies', {'Movies'}) + assert not lib.is_excluded('Movies2/a.mp4', {'Movies'}) + assert not lib.is_excluded('docs/Movies/a.mp4', {'Movies'}) + assert lib.is_excluded('docs/Movies/a.mp4', {'docs/Movies'}) + + +def test_list_local_directories(test_directory): + for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'zims', 'lost+found', 'Tags'): + try: + (test_directory / i).mkdir(parents=True) + except FileExistsError: + pass # 'Tags' is 'tags' on a filesystem that ignores case. + (test_directory / 'file.txt').write_text('a file') + (test_directory / 'link').symlink_to(test_directory / 'Movies') + + assert lib.list_local_directories('') == [ + dict(name='Movies', path='Movies'), dict(name='videos', path='videos'), dict(name='zims', path='zims')] + assert lib.list_local_directories('/videos/') == [dict(name='channel', path='videos/channel')] + for path in ('../', 'videos/../..', 'file.txt', 'missing', 'link'): + with pytest.raises(InvalidConfig): + lib.list_local_directories(path) + + +@pytest.mark.asyncio +async def test_list_primary_directories(primary, secondary_directory, make_secondary): + for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'): + (primary.directory / i).mkdir(parents=True) + (primary.directory / 'file.txt').write_text('a file') + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + assert await lib.list_primary_directories('') == [ + dict(name='Movies', path='Movies'), dict(name='videos', path='videos')] + assert await lib.list_primary_directories('videos') == [dict(name='channel', path='videos/channel')] + with pytest.raises(InvalidConfig): + await lib.list_primary_directories('../etc') diff --git a/wrolpi/sync/test/test_manifest.py b/wrolpi/sync/test/test_manifest.py deleted file mode 100644 index 7416dadc..00000000 --- a/wrolpi/sync/test/test_manifest.py +++ /dev/null @@ -1,67 +0,0 @@ -import os - -import pytest - -from wrolpi.files.models import FileGroup -from wrolpi.sync import lib - - -@pytest.mark.asyncio -async def test_manifest_pages(async_client, test_session, test_directory, make_files_structure, refresh_files): - """The manifest lists every file of every FileGroup, a page of FileGroups at a time, in id order.""" - make_files_structure({f'docs/{i}.txt': f'file {i}' for i in range(7)}) - await refresh_files() - - pages = list() - after_id = 0 - while after_id is not None: - page = lib.build_manifest_page(test_session, after_id, limit=3) - pages.append(page) - after_id = page['next_after_id'] - - assert [len(i['groups']) for i in pages] == [3, 3, 1] - ids = [g['id'] for page in pages for g in page['groups']] - assert ids == sorted(ids) - assert pages[0]['max_id'] == max(ids) - - files = {p['path']: p for page in pages for g in page['groups'] for p in g['paths']} - assert set(files) == {f'docs/{i}.txt' for i in range(7)} - stat = (test_directory / 'docs/3.txt').stat() - assert files['docs/3.txt']['size'] == stat.st_size - assert files['docs/3.txt']['mtime'] == stat.st_mtime - - -def test_manifest_skips(test_session, test_directory): - """Files that are still downloading, hidden, missing, or outside the library are not listed.""" - directory = test_directory / 'videos' - directory.mkdir() - for name in ('video.mp4', 'video.mp4.part', 'video.f137.mp4', 'video.mp4.aria2', '.video.mp4.partial', - 'video.en.vtt'): - (directory / name).write_text(name) - outside = test_directory.parent / f'{test_directory.name}-outside.txt' - outside.write_text('outside') - try: - group = FileGroup.from_paths(test_session, directory / 'video.mp4', directory / 'video.en.vtt') - group.files = [ - *group.files, - dict(path='video.mp4.part'), dict(path='video.f137.mp4'), dict(path='video.mp4.aria2'), - dict(path='.video.mp4.partial'), dict(path='deleted.mp4'), - # Older rows stored absolute paths. - dict(path=str(directory / 'video.en.vtt')), - dict(path=str(outside)), - {}, - ] - test_session.commit() - - page = lib.build_manifest_page(test_session) - finally: - os.remove(outside) - - paths = [p['path'] for g in page['groups'] for p in g['paths']] - assert sorted(set(paths)) == ['videos/video.en.vtt', 'videos/video.mp4'] - assert page['next_after_id'] is None - - -def test_manifest_empty(test_session, test_directory): - assert lib.build_manifest_page(test_session) == dict(groups=[], next_after_id=None, max_id=0, - limit=lib.MANIFEST_PAGE_LIMIT) diff --git a/wrolpi/sync/test/test_notify.py b/wrolpi/sync/test/test_notify.py new file mode 100644 index 00000000..8da7e81e --- /dev/null +++ b/wrolpi/sync/test/test_notify.py @@ -0,0 +1,72 @@ +import pytest +import yaml + +from wrolpi.sync import notify + + +@pytest.fixture +def state(test_directory): + """Write the Controller's record of the last sync.""" + + def write(**fields): + record = dict(state='idle', finished_at='2026-11-19T03:00:00+00:00', last_error=None, files_transferred=0, + files_deleted=0, warnings=[]) + record.update(fields) + (test_directory / notify.STATE_FILE).write_text(yaml.safe_dump(record)) + + return write + + +@pytest.mark.parametrize('fields, expected', [ + (dict(files_transferred=12, files_deleted=3), + ('completed', 'Synced 12 files from the primary; 3 moved to the trash')), + (dict(files_transferred=1), ('completed', 'Synced 1 files from the primary')), + (dict(warnings=['Some files changed on the primary']), + ('completed', 'Synced 0 files from the primary; Some files changed on the primary')), + (dict(state='error', last_error='The primary did not answer'), + ('failed', 'Sync from the primary failed: The primary did not answer')), + # Nothing changed: within the window that is every 15 minutes. + (dict(), None), + (dict(last_error='The sync was cancelled', files_transferred=4), None), + (dict(last_error='WROL Mode was enabled'), None), +]) +def test_sync_message(fields, expected): + record = dict(state='idle', last_error=None, files_transferred=0, files_deleted=0, warnings=[]) + record.update(fields) + assert notify.sync_message(record) == expected + + +def test_each_sync_is_announced_once(async_client, test_directory, test_wrolpi_config, make_secondary, state, + events_fixture): + """The API is restarted by every sync, so what it announced is kept in a file of its own.""" + make_secondary('https://primary.example') + state(files_transferred=5) + notify.notify_tick() + events_fixture.assert_has_event('sync_completed') + assert yaml.safe_load((test_directory / notify.ANNOUNCED_FILE).read_text()) == dict( + finished_at='2026-11-19T03:00:00+00:00') + + # Again, and after a restart: the same sync is not announced twice. + notify.notify_tick() + assert len(events_fixture.get_by_name('sync_completed')) == 1 + + state(state='error', last_error='The primary did not answer', finished_at='2026-11-19T03:20:00+00:00') + notify.notify_tick() + events_fixture.assert_has_event('sync_failed', 'Sync from the primary failed: The primary did not answer') + + # A sync that changed nothing is recorded as seen, and not announced. + state(finished_at='2026-11-19T03:40:00+00:00') + notify.notify_tick() + assert len(events_fixture.get_by_name('sync_completed')) == 1 + assert yaml.safe_load((test_directory / notify.ANNOUNCED_FILE).read_text())['finished_at'] == \ + '2026-11-19T03:40:00+00:00' + + +def test_not_while_running_and_not_on_a_primary(async_client, test_directory, test_wrolpi_config, make_secondary, + state): + state(state='running', finished_at=None) + notify.notify_tick() + assert not (test_directory / notify.ANNOUNCED_FILE).exists(), 'on a primary nothing is read' + make_secondary('https://primary.example') + notify.notify_tick() + assert not (test_directory / notify.ANNOUNCED_FILE).exists(), 'a running sync has nothing to announce' diff --git a/wrolpi/sync/test/test_role.py b/wrolpi/sync/test/test_role.py index 2fc36922..0e731168 100644 --- a/wrolpi/sync/test/test_role.py +++ b/wrolpi/sync/test/test_role.py @@ -1,16 +1,14 @@ """A secondary WROLPi never saves the configs it copies from its primary, and never downloads.""" -from datetime import datetime, time, timezone +from datetime import time import pytest import yaml from wrolpi.common import get_wrolpi_config, SYNCED_CONFIG_FILE_NAMES, get_all_configs -from wrolpi.config_version import IMPORT, SAVE, get_config_version, set_config_version +from wrolpi.config_version import IMPORT, SAVE, set_config_version from wrolpi.dates import is_within_daily_window from wrolpi.db import get_db_session from wrolpi.downloader import download_manager -from wrolpi.sync import lib -from wrolpi.sync.config import get_sync_config from wrolpi.tags import get_tags_config @@ -87,19 +85,3 @@ def test_secondary_never_downloads(async_client, test_wrolpi_config, make_second ]) def test_is_within_daily_window(start, end, current, expected): assert is_within_daily_window(start, end, current) is expected - - -def test_sync_is_due(test_wrolpi_config): - config = get_sync_config() - local_now = datetime(2026, 11, 5, 2, 0, tzinfo=timezone.utc) - never = dict(finished_at=None) - assert not lib.sync_is_due(never, local_now), 'no window is "Sync now" only' - - config._config.update(window_start='01:00', window_end='05:00') - assert lib.sync_is_due(never, local_now) - assert not lib.sync_is_due(never, local_now.replace(hour=6)), 'outside the window' - - recent = dict(finished_at=local_now.replace(hour=1, minute=50).isoformat()) - assert not lib.sync_is_due(recent, local_now), 'synced 10 minutes ago' - earlier = dict(finished_at=local_now.replace(hour=1, minute=40).isoformat()) - assert lib.sync_is_due(earlier, local_now), 'synced 20 minutes ago' diff --git a/wrolpi/test/test_api_fixtures.py b/wrolpi/test/test_api_fixtures.py index 4111b6c7..98bbe3dd 100644 --- a/wrolpi/test/test_api_fixtures.py +++ b/wrolpi/test/test_api_fixtures.py @@ -219,12 +219,11 @@ async def test_downloads_summary_fixture_matches_the_download_manager(async_clie @pytest.mark.parametrize('fixture_name, seed_name, response_name', [ - ('syncStatusFixture', 'default_sync_status', 'SyncStatusResponse'), ('backupStatusFixture', 'default_backup_status', 'BackupStatusResponse'), ]) def test_sync_status_fixture_matches_the_seed(fixture_name, seed_name, response_name): - """statusFixture's `sync` and `backup` are seeded by wrolpi.sync.status and declared in wrolpi.schema; the - fixture, the seed and the declaration must name the same keys.""" + """statusFixture's `backup` is seeded by wrolpi.sync.status and declared in wrolpi.schema; the fixture, the + seed and the declaration must name the same keys.""" from dataclasses import fields from wrolpi import schema from wrolpi.sync import status @@ -271,11 +270,11 @@ async def test_which_fixtures_the_spec_can_vouch_for(async_client): source = TEST_FIXTURES_JS.read_text() api_fixtures = set(re.findall(r'^export const ([a-zA-Z]+Fixture) =', source, re.M)) - # domainsFixture is a list built from domainFixture, syncStatusFixture and backupStatusFixture are - # statusFixture's `sync` and `backup` (test_sync_status_fixture_matches_the_seed holds them), and the *Context* fixtures are held to - # their React contexts by the frontend's own test-fixtures.test.js. controllerSyncStatusFixture is the - # Controller's, not this API's: controller/test/test_sync.py holds it to the Controller's sync status. - api_fixtures -= {'domainsFixture', 'syncStatusFixture', 'backupStatusFixture', 'controllerSyncStatusFixture'} + # domainsFixture is a list built from domainFixture, backupStatusFixture is statusFixture's `backup` + # (test_sync_status_fixture_matches_the_seed holds it), and the *Context* fixtures are held to their React + # contexts by the frontend's own test-fixtures.test.js. controllerSyncStatusFixture is the Controller's, not + # this API's: controller/test/test_sync.py holds it to the Controller's sync status. + api_fixtures -= {'domainsFixture', 'backupStatusFixture', 'controllerSyncStatusFixture'} api_fixtures = {n for n in api_fixtures if 'Context' not in n} assert api_fixtures == set(checked) | set(unchecked), (