From ea7cc4dec2ac1b48145dc6c2ddf163fc35257b62 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Wed, 18 Nov 2026 20:12:20 -0700 Subject: [PATCH] The Controller can make a secondary a copy of its primary with rsync (#992) * The Controller can make a secondary a copy of its primary with rsync The second part of replacing the sync. The Controller runs it, because the secondary's API is stopped while its library is replaced: checks, a database snapshot on the primary, stop the API, rsync every file from the primary's daemon (ignored directories included), fetch the snapshot and put it in the database's place, start the API, purge. - Files the primary deleted or changed are moved to a dated folder in .sync-trash, and removed after delete_after_days (a new sync.yaml setting, 30 by default; none never deletes), or sooner when the drive is nearly full. - A primary of another version or database migration, or with its media unmounted, is refused before anything is touched. The database is replaced last, so a sync that stops earlier leaves one that works. - On a Raspberry Pi and Debian rsync runs as the wrolpi user; in Docker in a helper container made from the api container. A Controller that starts after an interrupted sync starts the API. - When a secondary's API starts it takes the library settings of the primary's wrolpi.yaml, which the sync left. Reached only through the Controller's /api/sync/run for now; the existing sync and its schedule are unchanged. Co-Authored-By: Claude Fable 5.2 * The sync copies no link that leaves the library, and root follows none in the media directory Security review of the Controller's sync: - An exclusion that ends in a slash matches only a directory. A file or link named config/ssl, tags or config/backup on the primary would have replaced this machine's directory of that name. The exclusions now match whatever has the name. - Links were copied as they are. One that points outside the primary's library would, on the secondary, point at the secondary's own files, which the web server then serves. Only links that stay inside the library are copied, and no device files. - The Controller is root, in a media directory that belongs to another user. It no longer writes, reads, replaces or purges through a symbolic link there. Found on the way: the directory the Controller made for the incoming database belonged to root, so rsync, which runs as the library's owner, could not have written into it. It now belongs to the media directory's owner. Co-Authored-By: Claude Fable 5.2 * A sync ends when its primary stops answering, and makes room before it copies Review found: - rsync waits forever by default. A primary that lost power or its link during a copy left the secondary's API stopped until someone cancelled. rsync now gives up after 30 seconds without a connection and 5 minutes without data, which ends the run and starts the API. - The trash was only purged after a run that succeeded. The trash is what fills the drive, and a full drive fails every run, so it never got its turn. It is now also purged before the copy. - A value in sync.yaml that could not be used raised after the run lock was taken and before the handlers that release it, leaving "A sync is already running" until the Controller restarted. - The playlists directory left out of the copy was this machine's, not the primary's, whose hard links would have arrived as full copies. The primary's wrolpi.yaml is now fetched before the copy, and both are left out. - A primary whose media directory is elsewhere is refused: its database names every file by its full path. - In Docker rsync is its own container, which outlives a Controller that stops. A Controller that starts after an interrupted sync now removes it before it starts the API. Co-Authored-By: Claude Fable 5.2 --------- Co-authored-by: Claude Fable 5.2 --- app/src/api.js | 1 + app/src/test-fixtures.js | 1 + controller/api/sync.py | 36 ++ controller/lib/sync.py | 697 ++++++++++++++++++++++++++++++ controller/main.py | 6 + controller/test/test_sync.py | 618 ++++++++++++++++++++++++++ main.py | 6 + wrolpi/root_api.py | 1 + wrolpi/schema.py | 2 + wrolpi/sync/api.py | 3 + wrolpi/sync/config.py | 14 + wrolpi/sync/lib.py | 25 ++ wrolpi/sync/schema.py | 4 + wrolpi/sync/test/test_snapshot.py | 63 ++- 14 files changed, 1475 insertions(+), 2 deletions(-) create mode 100644 controller/api/sync.py create mode 100644 controller/lib/sync.py create mode 100644 controller/test/test_sync.py diff --git a/app/src/api.js b/app/src/api.js index 6130bfbb..b69edded 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -428,6 +428,7 @@ export async function getSettings() { require_media_mounted: content.require_media_mounted, role: content.role, save_ffprobe_json: content.save_ffprobe_json, + sync_delete_after_days: content.sync_delete_after_days, sync_excluded_directories: content.sync_excluded_directories, sync_primary_url: content.sync_primary_url, sync_window_end: content.sync_window_end, diff --git a/app/src/test-fixtures.js b/app/src/test-fixtures.js index d988878d..0e71bb23 100644 --- a/app/src/test-fixtures.js +++ b/app/src/test-fixtures.js @@ -81,6 +81,7 @@ export const settingsFixture = (overrides = {}) => ({ require_media_mounted: false, role: 'primary', save_ffprobe_json: true, + sync_delete_after_days: 30, sync_excluded_directories: [], sync_primary_url: null, sync_window_end: null, diff --git a/controller/api/sync.py b/controller/api/sync.py new file mode 100644 index 00000000..56bc5ebd --- /dev/null +++ b/controller/api/sync.py @@ -0,0 +1,36 @@ +""" +Sync API endpoints for WROLPi Controller: a secondary WROLPi copying its primary (see controller.lib.sync). + +The Controller answers these because the API is stopped while a sync runs. +""" + +import asyncio + +from fastapi import APIRouter, HTTPException + +from controller.lib import sync + +router = APIRouter(prefix="/api/sync", tags=["sync"]) + + +@router.get("/status") +async def sync_status() -> dict: + """The progress and outcome of this secondary's sync.""" + return sync.get_status() + + +@router.post("/run", status_code=202) +async def sync_run() -> dict: + """Copy the primary now.""" + if reason := await asyncio.to_thread(sync.skip_reason): + raise HTTPException(status_code=409, detail=reason) + # Kept referenced, or the task could be collected while the sync runs. + router.sync_task = asyncio.create_task(asyncio.to_thread(sync.run_sync)) + return {"success": True} + + +@router.post("/cancel", status_code=202) +async def sync_cancel() -> dict: + """Stop the running sync; the API is started again.""" + sync.request_cancel() + return {"success": True} diff --git a/controller/lib/sync.py b/controller/lib/sync.py new file mode 100644 index 00000000..d30080a0 --- /dev/null +++ b/controller/lib/sync.py @@ -0,0 +1,697 @@ +""" +A secondary WROLPi copies its primary: the whole media directory, and the database. + +The Controller runs it, because the secondary's API is stopped while its library is replaced: + +1. Checks: this is a secondary, WROL Mode is off, the primary answers, runs the same WROLPi and database migration, + and has its media directory mounted. A failure here changes nothing. +2. The primary makes a snapshot of its database (see wrolpi.sync.snapshot). +3. The API is stopped. +4. rsync copies every file from the primary's daemon (controller/rsyncd.conf), ignored directories included. Files + the primary no longer has, and the old versions of files it changed, are moved to a dated folder in + `.sync-trash` rather than deleted. +5. The database snapshot is copied beside the database, then takes its place. Last, so that a run which stops + earlier leaves a database that works. +6. The API is started. +7. Trash folders older than the delete delay are removed. + +The settings are the API's `sync.yaml`; this module only reads them. What a run did is kept in `sync_state.yaml`. +""" + +import contextlib +import datetime +import logging +import os +import re +import shutil +import sqlite3 +import subprocess +import threading +import time +import uuid +from pathlib import Path +from typing import Callable, Optional +from urllib.parse import urlparse + +import httpx +import yaml + +from controller.lib.config import get_media_directory, is_docker_mode +from controller.lib.docker_services import _get_client, _get_container_name, can_manage_containers, \ + start_container, stop_container +from controller.lib.rsyncd import get_rsyncd_port +from controller.lib.systemd import start_service, stop_service +from controller.lib.wrol_mode import is_wrol_mode + +logger = logging.getLogger(__name__) + +TRASH_DIRECTORY = ".sync-trash" +PARTIAL_DIRECTORY = ".rsync-partial" +# Where the primary keeps its database snapshot, and where this secondary keeps what it fetched from its primary. +SYNC_DIRECTORY = "config/.sync" +SNAPSHOT_PATH = f"{SYNC_DIRECTORY}/config/wrolpi.db" +INCOMING_DATABASE = f"{SYNC_DIRECTORY}/incoming/wrolpi.db" +# The primary's wrolpi.yaml and /api/sync/info, from which the API takes the library settings when it starts. +PRIMARY_WROLPI_CONFIG = f"{SYNC_DIRECTORY}/primary-wrolpi.yaml" +PRIMARY_INFO = f"{SYNC_DIRECTORY}/primary-info.yaml" +STATE_FILE = "config/sync_state.yaml" +DEFAULT_DELETE_AFTER_DAYS = 30 +# Never fill the secondary's drive; leave this much free. +MIN_FREE_BYTES = 1024 ** 3 +SNAPSHOT_TIMEOUT_SECONDS = 60 * 60 +API_START_TIMEOUT_SECONDS = 10 * 60 +# A sync runs unattended with the API stopped: a primary that loses power or its link must end the run, not hold it. +# rsync's own default is to wait forever. +RSYNC_CONNECT_TIMEOUT_SECONDS = 30 +RSYNC_IO_TIMEOUT_SECONDS = 300 +# rsync: 23 = some files could not be transferred, 24 = some files vanished while copying (the primary is in use). +RSYNC_PARTIAL, RSYNC_VANISHED = 23, 24 + +# Never copied from the primary: what belongs to this machine, and what WROLPi rebuilds. No trailing slash: a +# pattern that ends in one matches only a directory, and a file or link of that name on the primary would then +# replace this machine's directory. +LOCAL_PATHS = ( + # Hard links, rebuilt from the database when the API starts. + "/tags", + # This machine's settings and state. + "/config/wrolpi.yaml", "/config/sync.yaml", "/config/controller.yaml", "/config/fstab.yaml", + "/config/.fstab-migrated", "/config/smb.conf", f"/{STATE_FILE}", "/config/.wrol_mode", "/config/ssl", + "/config/cookies.txt", "/config/cookies.txt.enc", "/config/backup", + # The database arrives as a snapshot, in its own step. + "/config/wrolpi.db", "/config/wrolpi.db-wal", "/config/wrolpi.db-shm", "/config/wrolpi.db.lock", + f"/{SYNC_DIRECTORY}", + f"/{TRASH_DIRECTORY}", "/lost+found", +) +# Anywhere in the library: files still being written on the primary. +UNFINISHED_PATTERNS = (PARTIAL_DIRECTORY, ".partial", "*.part", "*.aria2", "*.ytdl", "*.db-wal", "*.db-shm") + + +class SyncError(Exception): + pass + + +class SyncCancelled(SyncError): + pass + + +def default_status() -> dict: + return dict( + state="idle", # 'idle', 'running', or 'error' + phase=None, # 'checks', 'snapshot', 'files', 'database', 'starting', 'purge' + started_at=None, + finished_at=None, + last_success_at=None, + last_error=None, + primary_url=None, + primary_version=None, + percent=0, + bytes_transferred=0, + files_transferred=0, + files_deleted=0, + trash_bytes=0, + warnings=[], + ) + + +_status = None +_status_lock = threading.Lock() +_run_lock = threading.Lock() +_cancel = threading.Event() + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _state_file() -> Path: + return get_media_directory() / STATE_FILE + + +def get_status() -> dict: + """The progress and outcome of the sync. Read from the state file the first time, so a restarted Controller + still knows when the last sync succeeded.""" + global _status + with _status_lock: + if _status is None: + _status = default_status() + try: + saved = yaml.safe_load(checked_path(_state_file()).read_text()) + if isinstance(saved, dict): + _status.update({k: v for k, v in saved.items() if k in _status}) + except (OSError, yaml.YAMLError, SyncError): + pass + return dict(_status) + + +def update_status(save: bool = False, **changes) -> dict: + """Change the status. `save` also writes it to the state file (not done for every line of progress).""" + get_status() + with _status_lock: + _status.update(changes) + status = dict(_status) + if save: + try: + write_file(_state_file(), yaml.safe_dump(status)) + except (OSError, SyncError) as e: + logger.warning("Could not save the sync state: %s", e) + return status + + +def checked_path(path: Path) -> Path: + """`path`, which is inside the media directory, refused if it or a directory on the way to it is a symbolic + link. The Controller runs as root, and the media directory belongs to another user: a link there would have + root read, write or delete somewhere else.""" + current = get_media_directory() + for part in path.relative_to(current).parts: + current = current / part + if current.is_symlink(): + raise SyncError(f"{current} is a symbolic link") + return path + + +def write_file(file: Path, text: str): + """Write `file` in the media directory without following a link: to a new file of its own, renamed into + place.""" + checked_path(file) + temporary = file.with_name(f".{file.name}.{uuid.uuid4().hex}.tmp") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644) + with os.fdopen(descriptor, "w") as fh: + fh.write(text) + os.replace(temporary, file) + + +def make_directory(directory: Path) -> Path: + """Make a directory in the media directory that belongs to the media directory's owner, as rsync (which does + not run as root) must write into it.""" + media_directory = get_media_directory() + checked_path(directory) + owner = media_directory.stat() + missing = [i for i in (directory, *directory.parents) if media_directory in i.parents and not i.exists()] + directory.mkdir(parents=True, exist_ok=True) + for path in missing: + with contextlib.suppress(PermissionError): + os.chown(path, owner.st_uid, owner.st_gid, follow_symlinks=False) + return directory + + +def read_sync_config() -> dict: + try: + return yaml.safe_load((get_media_directory() / "config/sync.yaml").read_text()) or {} + except (OSError, yaml.YAMLError): + return {} + + +def get_delete_after_days(config: dict) -> Optional[int]: + """Days a file the primary deleted is kept; None means such files are never deleted.""" + if "delete_after_days" not in config: + return DEFAULT_DELETE_AFTER_DAYS + days = config["delete_after_days"] + return None if days is None else max(0, int(days)) + + +def rsync_url(primary_url: str, path: str = "") -> str: + """The primary's rsync daemon address for a path in its media directory.""" + host = urlparse(primary_url).hostname + if not host: + raise SyncError(f"Invalid primary address: {primary_url!r}") + if ":" in host: + host = f"[{host}]" + return f"rsync://{host}:{get_rsyncd_port()}/media/{path}" + + +def timeout_arguments() -> list: + return [f"--contimeout={RSYNC_CONNECT_TIMEOUT_SECONDS}", f"--timeout={RSYNC_IO_TIMEOUT_SECONDS}"] + + +def rsync_literal(path: str) -> str: + """`path` as an rsync pattern that matches only itself (rsync reads `* ? [ \\` as wildcards).""" + return re.sub(r"([*?\[\\])", r"\\\1", path) + + +def build_files_command(primary_url: str, media_directory: Path, trash: Optional[Path], + excluded_directories: list = None, playlists_destinations: list = ("playlists",)) -> list: + """The rsync arguments (without `rsync`) that make `media_directory` a copy of the primary's. + + `trash` is where deleted and replaced files are moved; None never deletes. `playlists_destinations` are the + playlists directories of the primary (hard links there, which would arrive as full copies) and of this machine + (its own links, which must not be deleted).""" + arguments = [ + # Not --archive: owner and group are this machine's, and a library holds no devices. + "--recursive", "--perms", "--times", + # A link is copied only if it points inside the library. One that points elsewhere would, on this machine, + # point at this machine's files, which the web server would then serve. + "--links", "--safe-links", + "--partial", f"--partial-dir={PARTIAL_DIRECTORY}", + "--info=progress2,stats2", "--no-human-readable", + *timeout_arguments(), + ] + if trash is not None: + # An excluded path is never deleted, so the trash and this machine's own files are safe from --delete. + arguments.extend(["--delete", "--backup", f"--backup-dir={trash}"]) + arguments.extend(f"--exclude={i}" for i in LOCAL_PATHS) + for playlists in dict.fromkeys(str(i or "").strip("/") for i in playlists_destinations): + if playlists: + arguments.append(f"--exclude=/{rsync_literal(playlists)}") + arguments.extend(f"--exclude={i}" for i in UNFINISHED_PATTERNS) + # Folders the owner chose not to copy. + for directory in excluded_directories or []: + if directory := str(directory).strip("/"): + arguments.append(f"--exclude=/{rsync_literal(directory)}") + arguments.extend([rsync_url(primary_url), f"{media_directory}/"]) + return arguments + + +def build_database_command(primary_url: str, media_directory: Path) -> list: + """The rsync arguments that fetch the primary's database snapshot to `INCOMING_DATABASE`.""" + return [ + "--times", "--partial", "--info=progress2,stats2", "--no-human-readable", + *timeout_arguments(), + # The database in use is what the snapshot is built from, so only what differs is sent. + f"--copy-dest={media_directory / 'config'}", + # Never judged the same by size and time: two databases of one size, written in the same second, are common + # (a database grows a page at a time), and the secondary would keep its own. + "--ignore-times", + rsync_url(primary_url, SNAPSHOT_PATH), str(media_directory / INCOMING_DATABASE), + ] + + +PROGRESS = re.compile(r"^\s*(\d+)\s+(\d+)%") +STATS = { + "files_transferred": re.compile(r"^Number of regular files transferred: (\d+)"), + "files_deleted": re.compile(r"^Number of deleted files: (\d+)"), +} + + +def parse_rsync_line(line: str) -> dict: + """What a line of rsync's output says about its progress, as status changes.""" + if match := PROGRESS.match(line): + return dict(bytes_transferred=int(match.group(1)), percent=int(match.group(2))) + for key, pattern in STATS.items(): + if match := pattern.match(line.replace(",", "")): + return {key: int(match.group(1))} + return {} + + +# --------------------------------------------------------------------------------------------------------------------- +# What differs between a Raspberry Pi or Debian (systemd) and Docker. + + +class SystemdRunner: + """rsync runs here as the wrolpi user, who owns the library; the API is a systemd unit.""" + + def check(self): + if not shutil.which("rsync"): + raise SyncError("rsync is not installed") + + def stop_api(self): + result = stop_service("wrolpi-api") + if not result.get("success"): + raise SyncError(f"Could not stop the API: {result.get('error')}") + + def start_api(self): + start_service("wrolpi-api") + + def stop_rsync(self): + """Nothing to do: rsync is the Controller's child, and systemd stops it with the Controller.""" + + def rsync(self, arguments: list, on_line: Callable[[str], None]) -> int: + command = ["runuser", "-u", "wrolpi", "--", "rsync", *arguments] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return _follow(process.stdout.read1, on_line, process.terminate, process.wait) + + +class DockerRunner: + """rsync runs in a helper container made from the api container (its image, user and volumes), so it writes + where the API reads. The Controller does the rest in its own media directory, which must be the same one.""" + HELPER_NAME = _get_container_name("sync") + + def _api(self): + return _get_client().containers.get(_get_container_name("api")) + + def check(self): + if not can_manage_containers(): + raise SyncError("Docker management is not available") + # Asked of the containers themselves: the paths Docker reports for one folder can differ. + probe = make_directory(get_media_directory() / SYNC_DIRECTORY) / f".probe-{uuid.uuid4().hex}" + write_file(probe, "") + try: + seen = self._api().exec_run(["test", "-e", str(probe)]).exit_code == 0 + finally: + probe.unlink(missing_ok=True) + if not seen: + raise SyncError("The controller and api containers have different media directories; " + "give the controller service the api service's media volume") + + def stop_api(self): + result = stop_container("api") + if not result.get("success"): + raise SyncError(f"Could not stop the API: {result.get('error')}") + + def start_api(self): + start_container("api") + + def stop_rsync(self): + """The helper is its own container: it outlives a Controller that stops, and goes on replacing files.""" + with contextlib.suppress(Exception): + _get_client().containers.get(self.HELPER_NAME).remove(force=True) + + def rsync(self, arguments: list, on_line: Callable[[str], None]) -> int: + client, api = _get_client(), self._api() + self.stop_rsync() + helper = client.containers.run( + api.image.id, arguments, entrypoint="rsync", name=self.HELPER_NAME, detach=True, + user=api.attrs["Config"].get("User") or None, volumes_from=[api.name]) + try: + stream = helper.logs(stream=True, follow=True) + + def read(_: int = 0) -> bytes: + return next(stream, b"") + + return _follow(read, on_line, helper.kill, lambda: helper.wait()["StatusCode"]) + finally: + with contextlib.suppress(Exception): + helper.remove(force=True) + + +def _follow(read: Callable, on_line: Callable[[str], None], stop: Callable, wait: Callable) -> int: + """Hand rsync's output to `on_line` a line at a time (progress lines end in a carriage return). Stops rsync + as soon as the sync is cancelled, even while rsync is silent. Returns rsync's exit code.""" + finished = threading.Event() + + def stop_when_cancelled(): + while not finished.is_set(): + if _cancel.wait(0.2): + with contextlib.suppress(Exception): + stop() + return + + watcher = threading.Thread(target=stop_when_cancelled, daemon=True) + watcher.start() + try: + buffer = b"" + while chunk := read(65536): + buffer += chunk + *lines, buffer = re.split(rb"[\r\n]", buffer) + for line in lines: + if line.strip(): + on_line(line.decode(errors="replace")) + if buffer.strip(): + on_line(buffer.decode(errors="replace")) + code = wait() + finally: + finished.set() + if _cancel.is_set(): + raise SyncCancelled("The sync was cancelled") + return code + + +def get_runner(): + return DockerRunner() if is_docker_mode() else SystemdRunner() + + +# --------------------------------------------------------------------------------------------------------------------- +# The primary's and this machine's API. + + +def _local_api_url() -> str: + return "http://api:8081" if is_docker_mode() else "http://127.0.0.1:8081" + + +def _get_json(url: str, timeout: float = 30) -> dict: + # WROLPi serves HTTPS with its own certificate authority; on a trusted LAN the primary is not verified. + response = httpx.get(url, timeout=timeout, verify=False) + response.raise_for_status() + return response.json() + + +def get_primary_info(primary_url: str) -> dict: + return _get_json(f"{primary_url}/api/sync/info") + + +def get_local_info() -> dict: + return _get_json(f"{_local_api_url()}/api/sync/info") + + +def request_snapshot(primary_url: str) -> dict: + """Ask the primary for a database snapshot and wait for it. Returns the snapshot's status.""" + before = _get_json(f"{primary_url}/api/sync/status").get("snapshot") or {} + response = httpx.post(f"{primary_url}/api/sync/snapshot", timeout=30, verify=False) + if response.status_code >= 400: + with contextlib.suppress(Exception): + raise SyncError(f"The primary made no snapshot: {response.json()['error']}") + raise SyncError(f"The primary made no snapshot: HTTP {response.status_code}") + deadline = time.monotonic() + SNAPSHOT_TIMEOUT_SECONDS + while time.monotonic() < deadline: + _check_cancel() + snapshot = _get_json(f"{primary_url}/api/sync/status").get("snapshot") or {} + finished = snapshot.get("finished_at") + if not snapshot.get("requested") and snapshot.get("state") != "running" \ + and finished and finished != before.get("finished_at"): + if snapshot.get("state") == "error": + raise SyncError(f"The primary made no snapshot: {snapshot.get('last_error')}") + return snapshot + time.sleep(2) + raise SyncError("The primary did not finish its snapshot in time") + + +def wait_for_api(timeout: float = API_START_TIMEOUT_SECONDS) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with contextlib.suppress(Exception): + if httpx.get(f"{_local_api_url()}/api/echo", timeout=5).status_code == 200: + return True + time.sleep(2) + return False + + +# --------------------------------------------------------------------------------------------------------------------- +# The steps. + + +def _check_cancel(): + if _cancel.is_set(): + raise SyncCancelled("The sync was cancelled") + if is_wrol_mode(): + raise SyncCancelled("WROL Mode was enabled") + + +def skip_reason(config: dict = None) -> Optional[str]: + """Why a sync cannot start now, or None.""" + config = read_sync_config() if config is None else config + if config.get("role") != "secondary": + return "This WROLPi is not a secondary" + if not config.get("primary_url"): + return "No primary is configured" + if is_wrol_mode(): + return "WROL Mode is enabled" + if not (get_media_directory() / "config/controller.yaml").exists(): + return "The media directory is not mounted" + if _run_lock.locked(): + return "A sync is already running" + return None + + +def check_primary(primary: dict, local: dict): + """Refuse a primary whose library or database this secondary must not take.""" + if primary.get("role") != "primary": + raise SyncError("The primary address is not a primary WROLPi") + if not primary.get("media_mounted"): + # Its media directory would look like an empty library. + raise SyncError("The primary's media directory is not mounted") + for key, name in (("version", "WROLPi version"), ("database_revision", "database migration")): + if not primary.get(key) or primary.get(key) != local.get(key): + raise SyncError(f"The primary's {name} ({primary.get(key)}) is not this WROLPi's ({local.get(key)}); " + "upgrade both to the same version") + if not primary.get("media_directory") or primary.get("media_directory") != local.get("media_directory"): + # The database names every file by its full path. + raise SyncError(f"The primary's media directory ({primary.get('media_directory')}) is not this WROLPi's " + f"({local.get('media_directory')}); its database would find no file here") + + +def replace_database(media_directory: Path): + """Put the fetched snapshot in the database's place. + + The old write-ahead log must be gone first: left beside the new database it would be applied to it. It is + emptied by a checkpoint rather than only deleted, so a run that stops between the two steps leaves the old + database complete.""" + database = checked_path(media_directory / "config/wrolpi.db") + incoming = checked_path(media_directory / INCOMING_DATABASE) + with contextlib.closing(sqlite3.connect(f"file:{incoming}?mode=ro", uri=True)) as connection: + if connection.execute("PRAGMA schema_version").fetchone()[0] == 0: + raise SyncError("The snapshot from the primary is not a database") + if database.is_file(): + with contextlib.closing(sqlite3.connect(database, timeout=60)) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + for suffix in ("-wal", "-shm"): + database.with_name(f"wrolpi.db{suffix}").unlink(missing_ok=True) + os.replace(incoming, database) + + +def directory_size(directory: Path) -> int: + total = 0 + for root, _, names in os.walk(directory): + for name in names: + with contextlib.suppress(OSError): + total += os.lstat(os.path.join(root, name)).st_size + return total + + +def purge_trash(media_directory: Path, delete_after_days: Optional[int], today: datetime.date = None, + min_free_bytes: int = MIN_FREE_BYTES) -> list: + """Remove the trash folders that have waited `delete_after_days` and, while the drive is nearly full, the + oldest of the rest. A folder is as old as its name says, whatever its files' times are. Returns their names.""" + trash = media_directory / TRASH_DIRECTORY + if trash.is_symlink() or not trash.is_dir(): + return [] + today = today or datetime.date.today() + dated = [] + for folder in trash.iterdir(): + if folder.is_symlink() or not folder.is_dir(): + continue + with contextlib.suppress(ValueError): + dated.append((datetime.date.fromisoformat(folder.name), folder)) + purged = [] + for day, folder in sorted(dated): + expired = delete_after_days is not None and (today - day).days >= delete_after_days + nearly_full = shutil.disk_usage(media_directory).free < min_free_bytes + if not expired and not nearly_full: + continue + logger.info("Removing the sync trash of %s (%s)", folder.name, "expired" if expired else "drive nearly full") + shutil.rmtree(folder, ignore_errors=True) + purged.append(folder.name) + return purged + + +def run_sync(runner=None) -> dict: + """Make this secondary a copy of its primary. Returns the final status.""" + config = read_sync_config() + if reason := skip_reason(config): + logger.info("Not syncing: %s", reason) + return update_status(last_error=reason) + if not _run_lock.acquire(blocking=False): + return update_status(last_error="A sync is already running") + + runner = runner or get_runner() + media_directory = get_media_directory() + api_stopped = False + warnings = [] + _cancel.clear() + # Everything after the lock is taken is inside the `try`: whatever goes wrong, the lock is released. + try: + primary_url = str(config["primary_url"]).rstrip("/") + update_status(save=True, state="running", phase="checks", started_at=_now(), finished_at=None, + last_error=None, primary_url=primary_url, primary_version=None, percent=0, + bytes_transferred=0, files_transferred=0, files_deleted=0, warnings=[]) + logger.warning("Syncing from the primary %s", primary_url) + delete_after_days = get_delete_after_days(config) + runner.check() + primary, local = get_primary_info(primary_url), get_local_info() + update_status(primary_version=primary.get("version")) + check_primary(primary, local) + _check_cancel() + + update_status(phase="snapshot") + snapshot = request_snapshot(primary_url) + # The trash is what fills the drive, and a full drive fails every run: make room before the copy, not + # only after a run that succeeded. + purge_trash(media_directory, delete_after_days) + if shutil.disk_usage(media_directory).free - int(snapshot.get("size") or 0) < MIN_FREE_BYTES: + raise SyncError("Not enough free space for the primary's database") + _check_cancel() + + make_directory(media_directory / SYNC_DIRECTORY / "incoming") + # The primary's wrolpi.yaml names its playlists directory, which must be left out of the copy. + code = runner.rsync(["--times", *timeout_arguments(), rsync_url(primary_url, "config/wrolpi.yaml"), + str(media_directory / PRIMARY_WROLPI_CONFIG)], lambda _: None) + if code != 0: + raise SyncError(f"rsync of the primary's wrolpi.yaml failed with exit code {code}") + write_file(media_directory / PRIMARY_INFO, yaml.safe_dump(primary)) + playlists = [_read_yaml(media_directory / i).get("playlists_destination") or "playlists" + for i in (PRIMARY_WROLPI_CONFIG, "config/wrolpi.yaml")] + + update_status(save=True, phase="files") + runner.stop_api() + api_stopped = True + + trash = None + if delete_after_days is not None: + trash = media_directory / TRASH_DIRECTORY / datetime.date.today().isoformat() + command = build_files_command(primary_url, media_directory, trash, + config.get("sync_excluded_directories"), playlists) + code = runner.rsync(command, lambda line: update_status(**parse_rsync_line(line))) + if code in (RSYNC_PARTIAL, RSYNC_VANISHED): + warnings.append("Some files changed on the primary during the sync, or could not be copied; " + "the next sync copies them") + elif code != 0: + raise SyncError(f"rsync failed with exit code {code}") + _check_cancel() + + update_status(save=True, phase="database", percent=0) + code = runner.rsync( + build_database_command(primary_url, media_directory), + lambda line: update_status(**{k: v for k, v in parse_rsync_line(line).items() if k == "percent"})) + if code != 0: + raise SyncError(f"rsync of the database failed with exit code {code}") + # After this the run cannot be undone by stopping it. + replace_database(media_directory) + + update_status(save=True, phase="starting", percent=100) + runner.start_api() + api_stopped = False + if not wait_for_api(): + warnings.append("The API did not answer after the sync") + + update_status(phase="purge") + purge_trash(media_directory, delete_after_days) + except SyncCancelled as e: + logger.warning("Sync stopped: %s", e) + return _finish(runner, api_stopped, state="idle", last_error=str(e), warnings=warnings) + except Exception as e: + logger.error("Sync failed", exc_info=e) + return _finish(runner, api_stopped, state="error", last_error=str(e) or type(e).__name__, warnings=warnings) + + logger.warning("Synced from the primary %s", config["primary_url"]) + return _finish(runner, api_stopped, state="idle", last_error=None, last_success_at=_now(), warnings=warnings) + + +def _finish(runner, api_stopped: bool, **changes) -> dict: + """Whatever happened, the API runs again and the lock is free.""" + try: + if api_stopped: + runner.start_api() + finally: + _run_lock.release() + trash = get_media_directory() / TRASH_DIRECTORY + return update_status(save=True, phase=None, finished_at=_now(), + trash_bytes=directory_size(trash) if trash.is_dir() and not trash.is_symlink() else 0, + **changes) + + +def _read_yaml(file: Path) -> dict: + try: + return yaml.safe_load(file.read_text()) or {} + except (OSError, yaml.YAMLError): + return {} + + +def request_cancel(): + """Stop the running sync at its next step, or its rsync at once.""" + if _run_lock.locked(): + _cancel.set() + + +def recover_interrupted_sync(runner=None): + """Called when the Controller starts. A sync that was running when the Controller stopped left the API stopped; + start it, and say the sync was interrupted.""" + status = get_status() + if status["state"] != "running": + return + logger.warning("A sync was interrupted; starting the API") + runner = runner or get_runner() + # Before the API starts: nothing may still be replacing the library under it. + with contextlib.suppress(Exception): + runner.stop_rsync() + with contextlib.suppress(Exception): + runner.start_api() + update_status(save=True, state="error", phase=None, finished_at=_now(), + last_error="The sync was interrupted") diff --git a/controller/main.py b/controller/main.py index c4241a78..7f8a4230 100644 --- a/controller/main.py +++ b/controller/main.py @@ -33,6 +33,7 @@ from controller.api.schemas import HealthResponse from controller.api.scripts import router as scripts_router from controller.api.services import router as services_router from controller.api.status import router as status_router +from controller.api.sync import router as sync_router from controller.lib.config import ( get_ca_certificate_path, get_config_value, @@ -58,6 +59,7 @@ from controller.lib.status import ( get_uptime_status, ) from controller.lib.status_worker import status_worker_loop +from controller.lib.sync import recover_interrupted_sync from controller.lib.systemd import get_all_services_status # Templates directory @@ -124,6 +126,9 @@ async def lifespan(app: FastAPI): else: logger.info("Primary drive not mounted - using default configuration") + # A sync that was running when the Controller stopped left the API stopped. + app.state.sync_recovery_task = asyncio.create_task(asyncio.to_thread(recover_interrupted_sync)) + # Docker: the library is served to secondary WROLPis from a helper container, kept matching the api container. if is_docker_mode(): app.state.rsync_daemon_task = asyncio.create_task(rsync_daemon_loop()) @@ -176,6 +181,7 @@ app.include_router(ready_router) app.include_router(scripts_router) app.include_router(services_router) app.include_router(status_router) +app.include_router(sync_router) def get_host(request: Request) -> str: diff --git a/controller/test/test_sync.py b/controller/test/test_sync.py new file mode 100644 index 00000000..0761b8f1 --- /dev/null +++ b/controller/test/test_sync.py @@ -0,0 +1,618 @@ +""" +Unit tests for controller.lib.sync: a secondary WROLPi copying its primary with rsync. +""" + +import datetime +import os +import shutil +import socket +import sqlite3 +import subprocess +import threading +import time +from collections import namedtuple +from pathlib import Path + +import pytest +import yaml +from fastapi.testclient import TestClient + +from controller.lib import rsyncd, sync +from controller.main import app + +PRIMARY = "https://primary.example:8443" +INFO = dict(role="primary", version="1.0", database_revision="abc", media_mounted=True, media_directory="/media/wrolpi") +RSYNC_3 = bool(shutil.which("rsync")) and b"rsync version 3" in subprocess.run( + ["rsync", "--version"], capture_output=True).stdout + + +@pytest.fixture +def media(tmp_path, monkeypatch) -> Path: + """This machine's media directory: a mounted secondary of PRIMARY.""" + directory = tmp_path / "media" + (directory / "config").mkdir(parents=True) + (directory / "config/controller.yaml").write_text("{}") + (directory / "config/sync.yaml").write_text(yaml.safe_dump(dict(role="secondary", primary_url=PRIMARY))) + monkeypatch.setenv("MEDIA_DIRECTORY", str(directory)) + monkeypatch.setattr(sync, "_status", None) + sync._cancel.clear() + yield directory + assert not sync._run_lock.locked(), "the test left a sync running" + + +def make_database(file: Path, value: str): + file.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(file) as connection: + connection.execute("CREATE TABLE t (name TEXT)") + connection.execute("INSERT INTO t VALUES (?)", (value,)) + connection.close() + + +def read_database(file: Path) -> str: + connection = sqlite3.connect(f"file:{file}?mode=ro", uri=True) + try: + return connection.execute("SELECT name FROM t").fetchone()[0] + finally: + connection.close() + + +class FakeRunner: + """Records what a sync does to the API, and plays rsync.""" + + def __init__(self, media: Path, code: int = 0, on_files=None): + self.media, self.code, self.on_files = media, code, on_files + self.events = [] + + def check(self): + self.events.append("check") + + def stop_api(self): + self.events.append("stop_api") + + def start_api(self): + self.events.append("start_api") + + def stop_rsync(self): + self.events.append("stop_rsync") + + def rsync(self, arguments, on_line): + if arguments[-1].endswith("/"): + self.files_command = arguments + self.events.append("files") + on_line(" 1234567 42% 10.00MB/s 0:00:01") + on_line("Number of regular files transferred: 7") + if self.on_files: + self.on_files() + return self.code + if arguments[-1].endswith("primary-wrolpi.yaml"): + Path(arguments[-1]).write_text("videos_destination: videos\nplaylists_destination: lists\n") + return 0 + self.events.append("database") + make_database(Path(arguments[-1]), "from the primary") + return 0 + + +@pytest.fixture +def primary_api(monkeypatch): + """The primary's and this machine's APIs answer; both run the same WROLPi.""" + monkeypatch.setattr(sync, "get_primary_info", lambda url: dict(INFO)) + monkeypatch.setattr(sync, "get_local_info", lambda: dict(INFO, role="secondary")) + monkeypatch.setattr(sync, "request_snapshot", lambda url: dict(size=1000, written=True)) + monkeypatch.setattr(sync, "wait_for_api", lambda: True) + + +class TestCommand: + + def test_copies_everything_but_what_is_local(self, media): + command = sync.build_files_command(PRIMARY, media, media / ".sync-trash/2026-11-18", ["old [stuff]", "a/b/"], + ["my playlists", "lists", "my playlists"]) + assert command[-2:] == ["rsync://primary.example:8873/media/", f"{media}/"] + assert "--delete" in command and "--backup" in command + assert f"--backup-dir={media}/.sync-trash/2026-11-18" in command + for excluded in ("/tags", "/my playlists", "/lists", "/config/wrolpi.yaml", "/config/sync.yaml", "/config/ssl", + "/config/wrolpi.db", "/config/wrolpi.db-wal", "/config/.sync", "/.sync-trash", + "/config/sync_state.yaml", "/config/.wrol_mode", "*.part", ".partial", "*.db-wal", + "/old \\[stuff]", "/a/b"): + assert f"--exclude={excluded}" in command, excluded + # A pattern ending in a slash matches only a directory; a file or link of that name would get through. + assert not [i for i in command if i.startswith("--exclude=") and i.endswith("/")] + assert command.count("--exclude=/my playlists") == 1 + # A primary that stops answering ends the run; rsync alone would wait forever with the API stopped. + assert "--contimeout=30" in command and "--timeout=300" in command + # Only links that stay inside the library, and no device files. + assert "--safe-links" in command and "--devices" not in command and "--specials" not in command + # An ignored directory is still part of the library: nothing leaves out the map or the repos. + assert not [i for i in command if "map" in i or "repos" in i] + # Owner and group are this machine's. + assert "--archive" not in command and "--owner" not in command + + def test_database_is_never_judged_the_same_by_size_and_time(self, media): + command = sync.build_database_command(PRIMARY, media) + assert "--ignore-times" in command and f"--copy-dest={media}/config" in command + assert "--contimeout=30" in command and "--timeout=300" in command + assert command[-2:] == ["rsync://primary.example:8873/media/config/.sync/config/wrolpi.db", + f"{media}/config/.sync/incoming/wrolpi.db"] + + def test_never_deletes_without_a_trash(self, media): + command = sync.build_files_command(PRIMARY, media, None) + assert "--delete" not in command and "--backup" not in command + + @pytest.mark.parametrize("url, expected", [ + ("https://10.0.0.5", "rsync://10.0.0.5:8873/media/"), + ("https://wrolpi.local:8443", "rsync://wrolpi.local:8873/media/"), + ("https://[fd00::5]:8443", "rsync://[fd00::5]:8873/media/"), + ]) + def test_rsync_url(self, url, expected): + assert sync.rsync_url(url) == expected + assert sync.rsync_url(url, "config/wrolpi.yaml") == expected + "config/wrolpi.yaml" + + def test_rsync_url_invalid(self): + with pytest.raises(sync.SyncError): + sync.rsync_url("not an address") + + @pytest.mark.parametrize("line, expected", [ + (" 32768000 12% 31.25MB/s 0:00:07 (xfr#3, to-chk=10/20)", + dict(bytes_transferred=32768000, percent=12)), + ("Number of regular files transferred: 1,234", dict(files_transferred=1234)), + ("Number of deleted files: 3 (reg: 2, dir: 1)", dict(files_deleted=3)), + ("sending incremental file list", {}), + ]) + def test_parse_rsync_line(self, line, expected): + assert sync.parse_rsync_line(line) == expected + + @pytest.mark.parametrize("config, expected", [ + ({}, 30), (dict(delete_after_days=7), 7), (dict(delete_after_days=0), 0), (dict(delete_after_days=None), None), + ]) + def test_delete_after_days(self, config, expected): + assert sync.get_delete_after_days(config) == expected + + +class TestRootInAnotherUsersDirectory: + """The Controller runs as root; the media directory belongs to the wrolpi user, who must not be able to point + root somewhere else with a symbolic link.""" + + def test_state_is_not_written_through_a_link(self, media, tmp_path): + target = tmp_path / "somewhere else.txt" + target.write_text("untouched") + (media / "config/sync_state.yaml").symlink_to(target) + sync.update_status(save=True, last_error="x") + assert target.read_text() == "untouched" + + def test_write_file_replaces_never_follows(self, media, tmp_path): + target = tmp_path / "somewhere else.txt" + target.write_text("untouched") + (media / "config/.sync").symlink_to(tmp_path) + with pytest.raises(sync.SyncError, match="symbolic link"): + sync.write_file(media / sync.PRIMARY_INFO, "x") + with pytest.raises(sync.SyncError, match="symbolic link"): + sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming") + assert not (tmp_path / "incoming").exists() and not (tmp_path / "primary-info.yaml").exists() + + def test_directories_belong_to_the_library_owner(self, media, monkeypatch): + """rsync does not run as root, and must write into what the Controller made.""" + chowned = [] + monkeypatch.setattr(sync.os, "chown", lambda path, uid, gid, follow_symlinks=True: chowned.append( + (Path(path).relative_to(media).as_posix(), uid, gid))) + owner = media.stat() + sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming") + assert sorted(chowned) == [("config/.sync", owner.st_uid, owner.st_gid), + ("config/.sync/incoming", owner.st_uid, owner.st_gid)] + chowned.clear() + sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming") + assert chowned == [], "only what it made" + + def test_trash_that_is_a_link_is_not_purged(self, media, tmp_path): + elsewhere = tmp_path / "elsewhere" + (elsewhere / "2020-01-01").mkdir(parents=True) + (media / ".sync-trash").symlink_to(elsewhere) + assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=0) == [] + assert (elsewhere / "2020-01-01").is_dir() + + (media / ".sync-trash").unlink() + (media / ".sync-trash").mkdir() + (media / ".sync-trash/2020-01-02").symlink_to(elsewhere / "2020-01-01") + assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=0) == [] + assert (elsewhere / "2020-01-01").is_dir() + + def test_database_that_is_a_link_is_not_replaced(self, media, tmp_path): + make_database(tmp_path / "elsewhere.db", "somewhere else") + (media / "config/wrolpi.db").symlink_to(tmp_path / "elsewhere.db") + make_database(media / sync.INCOMING_DATABASE, "from the primary") + with pytest.raises(sync.SyncError, match="symbolic link"): + sync.replace_database(media) + assert read_database(tmp_path / "elsewhere.db") == "somewhere else" + + +class TestChecks: + + def test_skip_reason(self, media): + assert sync.skip_reason() is None + assert sync.skip_reason(dict(role="primary", primary_url=PRIMARY)) == "This WROLPi is not a secondary" + assert sync.skip_reason(dict(role="secondary")) == "No primary is configured" + (media / "config/.wrol_mode").touch() + assert sync.skip_reason() == "WROL Mode is enabled" + (media / "config/.wrol_mode").unlink() + (media / "config/controller.yaml").unlink() + assert sync.skip_reason() == "The media directory is not mounted" + + @pytest.mark.parametrize("change, message", [ + (dict(role="secondary"), "not a primary"), + (dict(media_mounted=False), "not mounted"), + (dict(version="2.0"), "WROLPi version"), + (dict(database_revision="def"), "database migration"), + (dict(database_revision=None), "database migration"), + # The database names every file by its full path. + (dict(media_directory="/data/library"), "media directory"), + (dict(media_directory=None), "media directory"), + ]) + def test_check_primary_refuses(self, change, message): + with pytest.raises(sync.SyncError, match=message): + sync.check_primary(dict(INFO, **change), dict(INFO)) + + def test_check_primary(self): + sync.check_primary(dict(INFO), dict(INFO, role="secondary")) + + +class TestPurge: + + def test_by_the_name_of_the_folder(self, media): + """A folder is as old as its name says; files keep the modification times they had in the library.""" + today = datetime.date(2026, 11, 18) + trash = media / ".sync-trash" + for name in ("2026-10-18", "2026-10-19", "2026-10-20", "2026-11-18", "not a date"): + (trash / name).mkdir(parents=True) + (trash / name / "file.txt").write_text("deleted on the primary") + assert sync.purge_trash(media, 30, today, min_free_bytes=0) == ["2026-10-18", "2026-10-19"] + assert sorted(i.name for i in trash.iterdir()) == ["2026-10-20", "2026-11-18", "not a date"] + + def test_never(self, media): + (media / ".sync-trash/2020-01-01").mkdir(parents=True) + assert sync.purge_trash(media, None, datetime.date(2026, 11, 18), min_free_bytes=0) == [] + + def test_oldest_first_when_the_drive_is_nearly_full(self, media, monkeypatch): + """A full drive is worse than a shorter wait: the oldest trash goes until there is room again.""" + trash = media / ".sync-trash" + for name in ("2026-11-16", "2026-11-17", "2026-11-18"): + (trash / name).mkdir(parents=True) + usage = namedtuple("usage", "total used free") + monkeypatch.setattr(sync.shutil, "disk_usage", + lambda _: usage(100, 0, 5 if (trash / "2026-11-16").exists() else 50)) + assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=10) == ["2026-11-16"] + + +class TestReplaceDatabase: + + def test_replaces_and_removes_the_old_log(self, media): + """The old write-ahead log left beside the new database would be applied to it.""" + database = media / "config/wrolpi.db" + make_database(database, "this secondary's") + live = sqlite3.connect(database) + live.execute("PRAGMA journal_mode=WAL") + live.execute("UPDATE t SET name = 'written to the log'") + live.commit() + assert database.with_name("wrolpi.db-wal").stat().st_size > 0 + make_database(media / sync.INCOMING_DATABASE, "from the primary") + + sync.replace_database(media) + live.close() + + assert read_database(database) == "from the primary" + assert not database.with_name("wrolpi.db-wal").exists() and not database.with_name("wrolpi.db-shm").exists() + assert not (media / sync.INCOMING_DATABASE).exists() + + def test_refuses_what_is_not_a_database(self, media): + make_database(media / "config/wrolpi.db", "this secondary's") + incoming = media / sync.INCOMING_DATABASE + incoming.parent.mkdir(parents=True) + incoming.write_bytes(b"") + with pytest.raises(sync.SyncError, match="not a database"): + sync.replace_database(media) + assert read_database(media / "config/wrolpi.db") == "this secondary's" + + +class TestRunSync: + + def test_order_and_outcome(self, media, primary_api): + """The API is down only while the library is replaced, and what the run did survives a restart.""" + make_database(media / "config/wrolpi.db", "this secondary's") + runner = FakeRunner(media) + + status = sync.run_sync(runner) + + assert runner.events == ["check", "stop_api", "files", "database", "start_api"] + assert status["state"] == "idle" and status["last_error"] is None and status["last_success_at"] + assert status["files_transferred"] == 7 and status["primary_version"] == "1.0" + assert read_database(media / "config/wrolpi.db") == "from the primary" + assert yaml.safe_load((media / sync.PRIMARY_INFO).read_text())["media_directory"] == "/media/wrolpi" + assert (media / sync.PRIMARY_WROLPI_CONFIG).is_file() + # A restarted Controller reads the state file. + sync._status = None + assert sync.get_status()["last_success_at"] == status["last_success_at"] + + def test_leaves_out_the_playlists_of_both_machines(self, media, primary_api): + """The primary names its playlists directory in its own wrolpi.yaml, which is fetched before the copy: its + hard links would arrive as full copies. This machine's own playlists directory must not be deleted.""" + (media / "config/wrolpi.yaml").write_text("playlists_destination: mine\n") + runner = FakeRunner(media) + sync.run_sync(runner) + assert "--exclude=/lists" in runner.files_command and "--exclude=/mine" in runner.files_command + + def test_trash_is_purged_even_when_the_copy_fails(self, media, primary_api): + """A full drive fails every copy, and the trash is what fills it: it must be emptied before the copy.""" + (media / ".sync-trash/2020-01-01").mkdir(parents=True) + status = sync.run_sync(FakeRunner(media, code=12)) + assert status["state"] == "error" + assert not (media / ".sync-trash/2020-01-01").exists() + + @pytest.mark.parametrize("config", [ + dict(delete_after_days="soon"), dict(primary_url=["not", "text"]), dict(sync_excluded_directories=5), + ]) + def test_a_bad_setting_ends_the_run_and_frees_it(self, media, primary_api, config): + """sync.yaml can be edited by hand. A value that cannot be used must not leave "A sync is already running" + behind until the Controller restarts.""" + (media / "config/sync.yaml").write_text(yaml.safe_dump(dict(dict(role="secondary", primary_url=PRIMARY), + **config))) + runner = FakeRunner(media) + status = sync.run_sync(runner) + assert status["state"] == "error" and status["last_error"] + assert not sync._run_lock.locked() + assert runner.events.count("stop_api") == runner.events.count("start_api") + assert sync.skip_reason(dict(role="secondary", primary_url=PRIMARY)) is None + + def test_a_refused_primary_changes_nothing(self, media, primary_api, monkeypatch): + monkeypatch.setattr(sync, "get_primary_info", lambda url: dict(INFO, version="2.0")) + runner = FakeRunner(media) + status = sync.run_sync(runner) + assert status["state"] == "error" and "upgrade both" in status["last_error"] + assert runner.events == ["check"], "the API was never stopped" + + def test_failed_files_leave_the_database_and_start_the_api(self, media, primary_api): + make_database(media / "config/wrolpi.db", "this secondary's") + runner = FakeRunner(media, code=12) + status = sync.run_sync(runner) + assert status["state"] == "error" and "exit code 12" in status["last_error"] + assert runner.events == ["check", "stop_api", "files", "start_api"] + assert read_database(media / "config/wrolpi.db") == "this secondary's" + assert status["last_success_at"] is None + + def test_files_that_vanished_are_only_a_warning(self, media, primary_api): + """The primary is in use while it is copied.""" + status = sync.run_sync(FakeRunner(media, code=sync.RSYNC_VANISHED)) + assert status["state"] == "idle" and status["last_success_at"] and status["warnings"] + + def test_cancelled(self, media, primary_api): + make_database(media / "config/wrolpi.db", "this secondary's") + runner = FakeRunner(media, on_files=sync.request_cancel) + status = sync.run_sync(runner) + assert status["state"] == "idle" and status["last_error"] == "The sync was cancelled" + assert runner.events[-1] == "start_api" and "database" not in runner.events + assert read_database(media / "config/wrolpi.db") == "this secondary's" + + def test_wrol_mode_stops_it(self, media, primary_api): + runner = FakeRunner(media, on_files=lambda: (media / "config/.wrol_mode").touch()) + status = sync.run_sync(runner) + assert status["last_error"] == "WROL Mode was enabled" and runner.events[-1] == "start_api" + + def test_not_a_secondary(self, media, primary_api): + (media / "config/sync.yaml").write_text("role: primary\n") + runner = FakeRunner(media) + assert sync.run_sync(runner)["last_error"] == "This WROLPi is not a secondary" + assert runner.events == [] + + def test_interrupted_sync_is_recovered(self, media): + """A Controller that stopped during a sync left the API stopped.""" + sync.update_status(save=True, state="running", phase="files") + sync._status = None + runner = FakeRunner(media) + sync.recover_interrupted_sync(runner) + # In Docker rsync is its own container and is still replacing files: it is stopped before the API starts. + assert runner.events == ["stop_rsync", "start_api"] + assert sync.get_status()["state"] == "error" and sync.get_status()["last_error"] == "The sync was interrupted" + + runner = FakeRunner(media) + sync.recover_interrupted_sync(runner) + assert runner.events == [], "nothing to recover the second time" + + +class TestApi: + + def test_status_run_cancel(self, media, monkeypatch): + client = TestClient(app) + assert client.get("/api/sync/status").json()["state"] == "idle" + + started = threading.Event() + monkeypatch.setattr(sync, "run_sync", started.set) + assert client.post("/api/sync/run").status_code == 202 + assert started.wait(5) + assert client.post("/api/sync/cancel").status_code == 202 + + (media / "config/sync.yaml").write_text("role: primary\n") + response = client.post("/api/sync/run") + assert response.status_code == 409 and response.json()["detail"] == "This WROLPi is not a secondary" + + +class LocalRunner(sync.SystemdRunner): + """Real rsync, as whoever runs the tests; the API is only recorded.""" + + def __init__(self): + self.events = [] + + def stop_api(self): + self.events.append("stop_api") + + def start_api(self): + self.events.append("start_api") + + def rsync(self, arguments, on_line): + process = subprocess.Popen(["rsync", *arguments], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return sync._follow(process.stdout.read1, on_line, process.terminate, process.wait) + + +def write(root: Path, files: dict): + for path, content in files.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text(content) + + +@pytest.mark.skipif(not RSYNC_3, reason="needs rsync 3") +def test_sync_with_a_real_daemon(media, tmp_path, primary_api, monkeypatch): + """A whole sync against a real rsync daemon run with the real config: the secondary becomes a copy of its + primary, ignored directories included; what the primary deleted or changed waits in the trash; what belongs to + this machine is left alone; and a second sync finds nothing to do.""" + primary = tmp_path / "primary" + write(primary, { + "videos/a.mp4": "a video", "videos/changed.txt": "the primary's newer version", + # Ignored on the primary, so in no FileGroup: the old sync never copied these. + "map/region.pmtiles": "a map", "repos/tool/README.md": "a repo", + "map/.partial/unfinished.pmtiles": "half a map", "map/region.search.db-wal": "in use", + "excluded/big.bin": "not wanted here", + "tags/one/a.mp4": "a hard link on the primary", "playlists/Watch/0001_a.mp4": "a hard link on the primary", + "config/tags.yaml": "the primary's tags", "config/wrolpi.yaml": "videos_destination: videos\n", + "config/sync.yaml": "role: primary\n", "config/controller.yaml": "the primary's", + "config/ssl/ca.key": "the primary's key", "config/wrolpi.db": "the primary's database, in use", + }) + make_database(primary / sync.SNAPSHOT_PATH, "from the primary") + # A link inside the library is part of it; one that points outside would point at this machine's files. + (primary / "videos/inside").symlink_to("a.mp4") + (primary / "videos/outside").symlink_to("/etc/hostname") + (primary / "videos/climbs out").symlink_to("../../outside.txt") + # Where this machine has a directory of its own, the primary has a file of that name. + (primary / "config/backup").write_text("not a directory on the primary") + write(media, {"config/backup/tags-20261118.yaml": "this machine's config backup"}) + write(media, { + "videos/changed.txt": "older", "videos/deleted on the primary.txt": "gone there", + "tags/mine/x.txt": "this machine's links", "config/wrolpi.yaml": "hotspot_ssid: mine\n", + "config/ssl/ca.key": "this machine's key", + }) + (media / "config/sync.yaml").write_text(yaml.safe_dump(dict( + role="secondary", primary_url="https://127.0.0.1:8443", sync_excluded_directories=["excluded"]))) + make_database(media / "config/wrolpi.db", "this secondary's") + # The same size, written in the same second: the two databases look the same to a quick comparison. + snapshot_stat = (primary / sync.SNAPSHOT_PATH).stat() + assert (media / "config/wrolpi.db").stat().st_size == snapshot_stat.st_size + os.utime(media / "config/wrolpi.db", ns=(snapshot_stat.st_atime_ns, snapshot_stat.st_mtime_ns)) + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + config = rsyncd.read_rsyncd_config().replace("port = 8873", f"port = {port}\naddress = 127.0.0.1") \ + .replace("path = /media/wrolpi", f"path = {primary}").replace("log file = /dev/stderr", "") + (tmp_path / "rsyncd.conf").write_text(config) + monkeypatch.setattr(sync, "get_rsyncd_port", lambda: port) + daemon = subprocess.Popen(["rsync", "--daemon", "--no-detach", f"--config={tmp_path / 'rsyncd.conf'}"]) + try: + for _ in range(50): + if subprocess.run(["rsync", f"rsync://127.0.0.1:{port}/"], capture_output=True).returncode == 0: + break + time.sleep(0.1) + runner = LocalRunner() + + status = sync.run_sync(runner) + + assert status["state"] == "idle" and status["last_error"] is None, status + assert runner.events == ["stop_api", "start_api"] + for path in ("videos/a.mp4", "map/region.pmtiles", "repos/tool/README.md", "config/tags.yaml"): + assert (media / path).read_text() == (primary / path).read_text(), path + assert (media / "videos/changed.txt").read_text() == "the primary's newer version" + assert (media / "videos/a.mp4").stat().st_mtime == (primary / "videos/a.mp4").stat().st_mtime + # Deleted and replaced files wait in today's trash. + trash = media / ".sync-trash" / datetime.date.today().isoformat() + assert (trash / "videos/deleted on the primary.txt").read_text() == "gone there" + assert (trash / "videos/changed.txt").read_text() == "older" + assert not (media / "videos/deleted on the primary.txt").exists() + # This machine's own, and what WROLPi rebuilds, are neither copied nor deleted. + assert (media / "config/wrolpi.yaml").read_text() == "hotspot_ssid: mine\n" + assert yaml.safe_load((media / "config/sync.yaml").read_text())["role"] == "secondary" + assert (media / "config/ssl/ca.key").read_text() == "this machine's key" + assert (media / "config/controller.yaml").read_text() == "{}" + assert (media / "tags/mine/x.txt").exists() and not (media / "tags/one").exists() + assert not (media / "playlists").exists() + assert (media / "config/backup/tags-20261118.yaml").read_text() == "this machine's config backup" + assert (media / "videos/inside").is_symlink() and (media / "videos/inside").read_text() == "a video" + assert not (media / "videos/outside").is_symlink() and not (media / "videos/climbs out").is_symlink() + # Unfinished files and the excluded folder stay on the primary. + assert not (media / "map/.partial").exists() and not (media / "map/region.search.db-wal").exists() + assert not (media / "excluded").exists() + # The database is the primary's snapshot, never its database in use; the library settings come along. + assert read_database(media / "config/wrolpi.db") == "from the primary" + assert (media / sync.PRIMARY_WROLPI_CONFIG).read_text() == "videos_destination: videos\n" + assert not (media / sync.SNAPSHOT_PATH).exists() + assert status["files_transferred"] >= 4 and status["files_deleted"] >= 1 and status["trash_bytes"] > 0 + + # Nothing changed: nothing is copied, and nothing more goes to the trash. + before = sorted(i for i in trash.rglob("*")) + status = sync.run_sync(LocalRunner()) + assert status["state"] == "idle" and status["files_transferred"] == 0 and status["files_deleted"] == 0 + assert sorted(i for i in trash.rglob("*")) == before + finally: + daemon.terminate() + daemon.wait(timeout=10) + + +class TestDockerRunner: + + def make(self, monkeypatch, media, exit_code): + from unittest import mock + api = mock.Mock() + seen = [] + + def exec_run(command): + seen.append(Path(command[-1]).exists()) + return mock.Mock(exit_code=exit_code) + + api.exec_run.side_effect = exec_run + client = mock.Mock() + client.containers.get.return_value = api + monkeypatch.setattr(sync, "can_manage_containers", lambda: True) + monkeypatch.setattr(sync, "_get_client", lambda: client) + return sync.DockerRunner(), seen + + def test_refuses_when_the_api_container_has_another_media_directory(self, media, monkeypatch): + """The Controller replaces the database in its own media directory; rsync writes to the api container's. + A probe file the api container cannot see means they are two directories.""" + runner, seen = self.make(monkeypatch, media, exit_code=1) + with pytest.raises(sync.SyncError, match="different media directories"): + runner.check() + assert seen == [True], "the probe existed while the api container was asked" + assert not list((media / sync.SYNC_DIRECTORY).glob(".probe-*")), "and is removed" + + def test_stop_rsync_removes_the_helper(self, media, monkeypatch): + from unittest import mock + helper = mock.Mock() + client = mock.Mock() + client.containers.get.return_value = helper + monkeypatch.setattr(sync, "_get_client", lambda: client) + sync.DockerRunner().stop_rsync() + client.containers.get.assert_called_once_with("wrolpi-sync-1") + helper.remove.assert_called_once_with(force=True) + + client.containers.get.side_effect = RuntimeError("no such container") + sync.DockerRunner().stop_rsync() + + def test_accepts_the_same_media_directory(self, media, monkeypatch): + runner, _ = self.make(monkeypatch, media, exit_code=0) + runner.check() + assert not list((media / sync.SYNC_DIRECTORY).glob(".probe-*")) + + +@pytest.mark.skipif(not RSYNC_3, reason="needs rsync 3") +def test_a_primary_that_stops_answering_ends_rsync(media, monkeypatch): + """A primary that accepts the connection and then says nothing (it lost power mid-copy, as far as TCP can tell): + rsync gives up, so the run ends and the API is started again.""" + monkeypatch.setattr(sync, "RSYNC_IO_TIMEOUT_SECONDS", 2) + monkeypatch.setattr(sync, "RSYNC_CONNECT_TIMEOUT_SECONDS", 2) + with socket.socket() as silent: + silent.bind(("127.0.0.1", 0)) + silent.listen() + monkeypatch.setattr(sync, "get_rsyncd_port", lambda: silent.getsockname()[1]) + # Without its timeouts rsync never returns; stop it, so that is a failure here and not a test that hangs. + gave_up = threading.Timer(30, sync._cancel.set) + gave_up.start() + try: + code = LocalRunner().rsync(sync.build_files_command("https://127.0.0.1", media, None), lambda _: None) + except sync.SyncCancelled: + pytest.fail("rsync waited on a silent primary until it was stopped") + finally: + gave_up.cancel() + sync._cancel.clear() + assert code != 0 diff --git a/main.py b/main.py index 1778af70..14b47959 100755 --- a/main.py +++ b/main.py @@ -22,6 +22,7 @@ from wrolpi.downloader import download_manager, get_download_manager_config from wrolpi.errors import WROLModeEnabled from wrolpi.files.worker import file_worker from wrolpi.perpetual import register_perpetual_process, perpetual_process_health_check +from wrolpi.sync.lib import apply_primary_wrolpi_config from wrolpi.vars import PROJECT_DIR, DOCKERIZED from wrolpi.version import get_version_string @@ -275,6 +276,11 @@ async def start_single_tasks(app: Sanic): # Import configs that require the database. if wrolpi_config.successful_import and not wrolpi_config.wrol_mode: + try: + # A secondary's library settings are its primary's; before the configs that depend on them. + apply_primary_wrolpi_config() + except Exception as e: + logger.error("Failed to apply the primary's library settings", exc_info=e) from wrolpi.common import import_all_db_configs await import_all_db_configs() diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index c778e8f5..ad436d28 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -217,6 +217,7 @@ def get_settings(_: Request): 'role': sync_config.role, 'save_ffprobe_json': wrolpi_config.save_ffprobe_json, 'sync_primary_url': sync_config.primary_url, + 'sync_delete_after_days': sync_config.delete_after_days, 'sync_excluded_directories': sync_config.sync_excluded_directories, 'sync_window_end': sync_config.window_end, 'sync_window_start': sync_config.window_start, diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 03b74d0f..161e86a1 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -45,6 +45,8 @@ class SettingsResponse: require_cookies_unlocked: bool role: str sync_primary_url: Optional[str] + # None never deletes. + sync_delete_after_days: Optional[int] sync_excluded_directories: List[str] sync_window_end: Optional[str] sync_window_start: Optional[str] diff --git a/wrolpi/sync/api.py b/wrolpi/sync/api.py index 0d8fc858..eee301f7 100644 --- a/wrolpi/sync/api.py +++ b/wrolpi/sync/api.py @@ -171,6 +171,9 @@ async def put_settings(_: Request, body: schema.SyncSettingsRequest): if value and not re.match(r'^([01]\d|2[0-3]):[0-5]\d$', value): raise InvalidConfig(f'{request_key} must be in HH:MM format') changes[config_key] = value or None + if body.sync_delete_after_days is not None: + days = int(body.sync_delete_after_days) + changes['delete_after_days'] = None if days < 0 else days for key in ('sync_excluded_directories', 'backup_excluded_directories'): if (value := getattr(body, key)) is not None: changes[key] = lib.normalize_excluded_directories(value) diff --git a/wrolpi/sync/config.py b/wrolpi/sync/config.py index da2c98f3..cdeab874 100644 --- a/wrolpi/sync/config.py +++ b/wrolpi/sync/config.py @@ -21,6 +21,8 @@ class SyncConfigValidator: window_end: str = None # Folders (relative to the media directory) a secondary never downloads from its primary. sync_excluded_directories: list[str] = field(default_factory=list) + # Days a secondary keeps a file its primary deleted or replaced (in `.sync-trash`); None never deletes. + delete_after_days: int = None # A second drive this WROLPi copies its media directory to (see wrolpi.sync.backup), and the UUID of the drive # that must be mounted there. backup_directory: str = None @@ -34,6 +36,10 @@ class SyncConfigValidator: backup_last_success_at: str = None +# Long enough to notice a file that should not have been deleted on the primary. +DEFAULT_DELETE_AFTER_DAYS = 30 + + class SyncConfig(ConfigFile): file_name = 'sync.yaml' default_config = dict( @@ -43,6 +49,7 @@ class SyncConfig(ConfigFile): window_start=None, window_end=None, sync_excluded_directories=[], + delete_after_days=DEFAULT_DELETE_AFTER_DAYS, backup_directory=None, backup_device_uuid=None, backup_window_start=None, @@ -78,6 +85,13 @@ class SyncConfig(ConfigFile): def sync_excluded_directories(self) -> List[str]: return list(self._config.get('sync_excluded_directories') or []) + @property + def delete_after_days(self) -> Optional[int]: + """None never deletes. A sync.yaml written before this setting has no key, and gets the default.""" + if 'delete_after_days' not in self._config: + return DEFAULT_DELETE_AFTER_DAYS + return self._config['delete_after_days'] + @property def backup_directory(self) -> Optional[str]: return self._config.get('backup_directory') diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py index 7388f23e..0c843bff 100644 --- a/wrolpi/sync/lib.py +++ b/wrolpi/sync/lib.py @@ -704,6 +704,31 @@ def merge_wrolpi_config(primary_config: dict, primary_media_directory: str) -> d return changes +# Where the Controller's sync (controller/lib/sync.py) leaves the primary's wrolpi.yaml and /api/sync/info. +PRIMARY_WROLPI_CONFIG = 'config/.sync/primary-wrolpi.yaml' +PRIMARY_INFO = 'config/.sync/primary-info.yaml' + + +def apply_primary_wrolpi_config() -> List[str]: + """A secondary's wrolpi.yaml is its own (hotspot, timezone), except for the settings that describe the library: + those are taken from the copy of the primary's wrolpi.yaml that the last sync left. Returns the changed keys.""" + if not is_secondary(): + return [] + media_directory = get_media_directory() + try: + primary_config = _parse_config((media_directory / PRIMARY_WROLPI_CONFIG).read_bytes(), 'wrolpi.yaml') + info = yaml.safe_load((media_directory / PRIMARY_INFO).read_text()) or {} + except (OSError, yaml.YAMLError, SyncError) as e: + if not isinstance(e, FileNotFoundError): + logger.error(f"Could not read the primary's wrolpi.yaml: {e}") + return [] + changes = merge_wrolpi_config(primary_config, str(info.get('media_directory') or media_directory)) + if changes: + logger.info(f'Applying library settings from the primary: {sorted(changes)}') + get_wrolpi_config().update(changes) + return sorted(changes) + + def _relative_directories(directories: List[str], primary_media_directory: str) -> List[str]: relative = list() for directory in directories: diff --git a/wrolpi/sync/schema.py b/wrolpi/sync/schema.py index 1332bc0a..813dda2e 100644 --- a/wrolpi/sync/schema.py +++ b/wrolpi/sync/schema.py @@ -47,6 +47,8 @@ class SyncSettingsRequest: backup_device_uuid: Optional[str] = None backup_window_start: Optional[str] = None backup_window_end: Optional[str] = None + # Days a secondary keeps a file its primary deleted or replaced; -1 never deletes. + sync_delete_after_days: Optional[int] = None # Folders, relative to the media directory, never copied ([] clears). sync_excluded_directories: Optional[List[str]] = None backup_excluded_directories: Optional[List[str]] = None @@ -54,3 +56,5 @@ class SyncSettingsRequest: def __post_init__(self): if self.role is not None and self.role not in ('primary', 'secondary'): raise ValidationError('role must be "primary" or "secondary"') + if self.sync_delete_after_days is not None and not -1 <= int(self.sync_delete_after_days) <= 3650: + raise ValidationError('sync_delete_after_days must be between 0 and 3650, or -1 to never delete') diff --git a/wrolpi/sync/test/test_snapshot.py b/wrolpi/sync/test/test_snapshot.py index 6f5727b4..4b818f90 100644 --- a/wrolpi/sync/test/test_snapshot.py +++ b/wrolpi/sync/test/test_snapshot.py @@ -1,14 +1,17 @@ import contextlib +import json import shutil import sqlite3 from collections import namedtuple from http import HTTPStatus import pytest +import yaml -from wrolpi.common import enable_wrol_mode +from wrolpi.common import enable_wrol_mode, get_wrolpi_config from wrolpi.db import get_db_file -from wrolpi.sync import backup, snapshot +from wrolpi.sync import backup, lib, snapshot +from wrolpi.sync.config import get_sync_config from wrolpi.sync.status import get_snapshot_status, update_snapshot_status @@ -128,3 +131,59 @@ def test_backup_drive_does_not_copy_the_snapshot(test_directory, tmp_path): """The backup drive gets its own snapshot; the one kept for secondaries is not copied to it as well.""" command = backup.build_rsync_command(test_directory, tmp_path) assert f'--exclude=/{snapshot.SNAPSHOT_DIRECTORY}/' in command + + +@pytest.mark.asyncio +async def test_delete_after_days_setting(async_client, test_session, test_directory, test_wrolpi_config, + await_switches): + """How long a secondary keeps what its primary deleted: 30 days unless set, and -1 asks to never delete.""" + request, response = await async_client.get('/api/settings') + assert response.json['sync_delete_after_days'] == 30 + + for sent, kept in ((7, 7), (0, 0), (-1, None)): + request, response = await async_client.put('/api/sync/settings', + content=json.dumps(dict(sync_delete_after_days=sent))) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json + assert get_sync_config().delete_after_days == kept + request, response = await async_client.get('/api/settings') + assert response.json['sync_delete_after_days'] == kept + + await await_switches() + saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text()) + assert 'delete_after_days' in saved and saved['delete_after_days'] is None, 'what the Controller reads' + + request, response = await async_client.put('/api/sync/settings', + content=json.dumps(dict(sync_delete_after_days=-2))) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +def test_delete_after_days_of_an_older_config(test_directory, test_wrolpi_config): + """A sync.yaml written before the setting existed has no key; it gets the default, not "never".""" + get_sync_config()._config.pop('delete_after_days', None) + assert get_sync_config().delete_after_days == 30 + + +def test_apply_primary_wrolpi_config(async_client, test_directory, test_wrolpi_config, make_secondary): + """When a secondary's API starts it takes the settings that describe the library from the primary's wrolpi.yaml, + which the Controller's sync left; the settings of this machine stay.""" + sync_directory = test_directory / 'config/.sync' + sync_directory.mkdir(parents=True) + (sync_directory / 'primary-wrolpi.yaml').write_text(yaml.safe_dump(dict( + videos_destination='videos/%(channel_name)s', ignored_directories=['/media/wrolpi/map', 'config', 'elsewhere'], + hotspot_ssid='the primary', timezone='Europe/Paris'))) + (sync_directory / 'primary-info.yaml').write_text(yaml.safe_dump(dict(media_directory='/media/wrolpi'))) + timezone = get_wrolpi_config().timezone + + assert lib.apply_primary_wrolpi_config() == [], 'only on a secondary' + + make_secondary('https://primary.example') + assert lib.apply_primary_wrolpi_config() == ['ignored_directories', 'videos_destination'] + assert get_wrolpi_config().videos_destination == 'videos/%(channel_name)s' + # The primary's absolute paths are this machine's relative ones. + assert get_wrolpi_config().ignored_directories == ['map', 'config', 'elsewhere'] + assert get_wrolpi_config().timezone == timezone + + assert lib.apply_primary_wrolpi_config() == [], 'nothing changed since' + + (sync_directory / 'primary-wrolpi.yaml').write_text('not: [a config') + assert lib.apply_primary_wrolpi_config() == []