diff --git a/app/src/api.js b/app/src/api.js index 6130bfbb..b69edded 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -428,6 +428,7 @@ export async function getSettings() { require_media_mounted: content.require_media_mounted, role: content.role, save_ffprobe_json: content.save_ffprobe_json, + sync_delete_after_days: content.sync_delete_after_days, sync_excluded_directories: content.sync_excluded_directories, sync_primary_url: content.sync_primary_url, sync_window_end: content.sync_window_end, diff --git a/app/src/test-fixtures.js b/app/src/test-fixtures.js index d988878d..0e71bb23 100644 --- a/app/src/test-fixtures.js +++ b/app/src/test-fixtures.js @@ -81,6 +81,7 @@ export const settingsFixture = (overrides = {}) => ({ require_media_mounted: false, role: 'primary', save_ffprobe_json: true, + sync_delete_after_days: 30, sync_excluded_directories: [], sync_primary_url: null, sync_window_end: null, diff --git a/controller/api/sync.py b/controller/api/sync.py new file mode 100644 index 00000000..56bc5ebd --- /dev/null +++ b/controller/api/sync.py @@ -0,0 +1,36 @@ +""" +Sync API endpoints for WROLPi Controller: a secondary WROLPi copying its primary (see controller.lib.sync). + +The Controller answers these because the API is stopped while a sync runs. +""" + +import asyncio + +from fastapi import APIRouter, HTTPException + +from controller.lib import sync + +router = APIRouter(prefix="/api/sync", tags=["sync"]) + + +@router.get("/status") +async def sync_status() -> dict: + """The progress and outcome of this secondary's sync.""" + return sync.get_status() + + +@router.post("/run", status_code=202) +async def sync_run() -> dict: + """Copy the primary now.""" + if reason := await asyncio.to_thread(sync.skip_reason): + raise HTTPException(status_code=409, detail=reason) + # Kept referenced, or the task could be collected while the sync runs. + router.sync_task = asyncio.create_task(asyncio.to_thread(sync.run_sync)) + return {"success": True} + + +@router.post("/cancel", status_code=202) +async def sync_cancel() -> dict: + """Stop the running sync; the API is started again.""" + sync.request_cancel() + return {"success": True} diff --git a/controller/lib/sync.py b/controller/lib/sync.py new file mode 100644 index 00000000..d30080a0 --- /dev/null +++ b/controller/lib/sync.py @@ -0,0 +1,697 @@ +""" +A secondary WROLPi copies its primary: the whole media directory, and the database. + +The Controller runs it, because the secondary's API is stopped while its library is replaced: + +1. Checks: this is a secondary, WROL Mode is off, the primary answers, runs the same WROLPi and database migration, + and has its media directory mounted. A failure here changes nothing. +2. The primary makes a snapshot of its database (see wrolpi.sync.snapshot). +3. The API is stopped. +4. rsync copies every file from the primary's daemon (controller/rsyncd.conf), ignored directories included. Files + the primary no longer has, and the old versions of files it changed, are moved to a dated folder in + `.sync-trash` rather than deleted. +5. The database snapshot is copied beside the database, then takes its place. Last, so that a run which stops + earlier leaves a database that works. +6. The API is started. +7. Trash folders older than the delete delay are removed. + +The settings are the API's `sync.yaml`; this module only reads them. What a run did is kept in `sync_state.yaml`. +""" + +import contextlib +import datetime +import logging +import os +import re +import shutil +import sqlite3 +import subprocess +import threading +import time +import uuid +from pathlib import Path +from typing import Callable, Optional +from urllib.parse import urlparse + +import httpx +import yaml + +from controller.lib.config import get_media_directory, is_docker_mode +from controller.lib.docker_services import _get_client, _get_container_name, can_manage_containers, \ + start_container, stop_container +from controller.lib.rsyncd import get_rsyncd_port +from controller.lib.systemd import start_service, stop_service +from controller.lib.wrol_mode import is_wrol_mode + +logger = logging.getLogger(__name__) + +TRASH_DIRECTORY = ".sync-trash" +PARTIAL_DIRECTORY = ".rsync-partial" +# Where the primary keeps its database snapshot, and where this secondary keeps what it fetched from its primary. +SYNC_DIRECTORY = "config/.sync" +SNAPSHOT_PATH = f"{SYNC_DIRECTORY}/config/wrolpi.db" +INCOMING_DATABASE = f"{SYNC_DIRECTORY}/incoming/wrolpi.db" +# The primary's wrolpi.yaml and /api/sync/info, from which the API takes the library settings when it starts. +PRIMARY_WROLPI_CONFIG = f"{SYNC_DIRECTORY}/primary-wrolpi.yaml" +PRIMARY_INFO = f"{SYNC_DIRECTORY}/primary-info.yaml" +STATE_FILE = "config/sync_state.yaml" +DEFAULT_DELETE_AFTER_DAYS = 30 +# Never fill the secondary's drive; leave this much free. +MIN_FREE_BYTES = 1024 ** 3 +SNAPSHOT_TIMEOUT_SECONDS = 60 * 60 +API_START_TIMEOUT_SECONDS = 10 * 60 +# A sync runs unattended with the API stopped: a primary that loses power or its link must end the run, not hold it. +# rsync's own default is to wait forever. +RSYNC_CONNECT_TIMEOUT_SECONDS = 30 +RSYNC_IO_TIMEOUT_SECONDS = 300 +# rsync: 23 = some files could not be transferred, 24 = some files vanished while copying (the primary is in use). +RSYNC_PARTIAL, RSYNC_VANISHED = 23, 24 + +# Never copied from the primary: what belongs to this machine, and what WROLPi rebuilds. No trailing slash: a +# pattern that ends in one matches only a directory, and a file or link of that name on the primary would then +# replace this machine's directory. +LOCAL_PATHS = ( + # Hard links, rebuilt from the database when the API starts. + "/tags", + # This machine's settings and state. + "/config/wrolpi.yaml", "/config/sync.yaml", "/config/controller.yaml", "/config/fstab.yaml", + "/config/.fstab-migrated", "/config/smb.conf", f"/{STATE_FILE}", "/config/.wrol_mode", "/config/ssl", + "/config/cookies.txt", "/config/cookies.txt.enc", "/config/backup", + # The database arrives as a snapshot, in its own step. + "/config/wrolpi.db", "/config/wrolpi.db-wal", "/config/wrolpi.db-shm", "/config/wrolpi.db.lock", + f"/{SYNC_DIRECTORY}", + f"/{TRASH_DIRECTORY}", "/lost+found", +) +# Anywhere in the library: files still being written on the primary. +UNFINISHED_PATTERNS = (PARTIAL_DIRECTORY, ".partial", "*.part", "*.aria2", "*.ytdl", "*.db-wal", "*.db-shm") + + +class SyncError(Exception): + pass + + +class SyncCancelled(SyncError): + pass + + +def default_status() -> dict: + return dict( + state="idle", # 'idle', 'running', or 'error' + phase=None, # 'checks', 'snapshot', 'files', 'database', 'starting', 'purge' + started_at=None, + finished_at=None, + last_success_at=None, + last_error=None, + primary_url=None, + primary_version=None, + percent=0, + bytes_transferred=0, + files_transferred=0, + files_deleted=0, + trash_bytes=0, + warnings=[], + ) + + +_status = None +_status_lock = threading.Lock() +_run_lock = threading.Lock() +_cancel = threading.Event() + + +def _now() -> str: + return datetime.datetime.now(datetime.timezone.utc).isoformat() + + +def _state_file() -> Path: + return get_media_directory() / STATE_FILE + + +def get_status() -> dict: + """The progress and outcome of the sync. Read from the state file the first time, so a restarted Controller + still knows when the last sync succeeded.""" + global _status + with _status_lock: + if _status is None: + _status = default_status() + try: + saved = yaml.safe_load(checked_path(_state_file()).read_text()) + if isinstance(saved, dict): + _status.update({k: v for k, v in saved.items() if k in _status}) + except (OSError, yaml.YAMLError, SyncError): + pass + return dict(_status) + + +def update_status(save: bool = False, **changes) -> dict: + """Change the status. `save` also writes it to the state file (not done for every line of progress).""" + get_status() + with _status_lock: + _status.update(changes) + status = dict(_status) + if save: + try: + write_file(_state_file(), yaml.safe_dump(status)) + except (OSError, SyncError) as e: + logger.warning("Could not save the sync state: %s", e) + return status + + +def checked_path(path: Path) -> Path: + """`path`, which is inside the media directory, refused if it or a directory on the way to it is a symbolic + link. The Controller runs as root, and the media directory belongs to another user: a link there would have + root read, write or delete somewhere else.""" + current = get_media_directory() + for part in path.relative_to(current).parts: + current = current / part + if current.is_symlink(): + raise SyncError(f"{current} is a symbolic link") + return path + + +def write_file(file: Path, text: str): + """Write `file` in the media directory without following a link: to a new file of its own, renamed into + place.""" + checked_path(file) + temporary = file.with_name(f".{file.name}.{uuid.uuid4().hex}.tmp") + descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644) + with os.fdopen(descriptor, "w") as fh: + fh.write(text) + os.replace(temporary, file) + + +def make_directory(directory: Path) -> Path: + """Make a directory in the media directory that belongs to the media directory's owner, as rsync (which does + not run as root) must write into it.""" + media_directory = get_media_directory() + checked_path(directory) + owner = media_directory.stat() + missing = [i for i in (directory, *directory.parents) if media_directory in i.parents and not i.exists()] + directory.mkdir(parents=True, exist_ok=True) + for path in missing: + with contextlib.suppress(PermissionError): + os.chown(path, owner.st_uid, owner.st_gid, follow_symlinks=False) + return directory + + +def read_sync_config() -> dict: + try: + return yaml.safe_load((get_media_directory() / "config/sync.yaml").read_text()) or {} + except (OSError, yaml.YAMLError): + return {} + + +def get_delete_after_days(config: dict) -> Optional[int]: + """Days a file the primary deleted is kept; None means such files are never deleted.""" + if "delete_after_days" not in config: + return DEFAULT_DELETE_AFTER_DAYS + days = config["delete_after_days"] + return None if days is None else max(0, int(days)) + + +def rsync_url(primary_url: str, path: str = "") -> str: + """The primary's rsync daemon address for a path in its media directory.""" + host = urlparse(primary_url).hostname + if not host: + raise SyncError(f"Invalid primary address: {primary_url!r}") + if ":" in host: + host = f"[{host}]" + return f"rsync://{host}:{get_rsyncd_port()}/media/{path}" + + +def timeout_arguments() -> list: + return [f"--contimeout={RSYNC_CONNECT_TIMEOUT_SECONDS}", f"--timeout={RSYNC_IO_TIMEOUT_SECONDS}"] + + +def rsync_literal(path: str) -> str: + """`path` as an rsync pattern that matches only itself (rsync reads `* ? [ \\` as wildcards).""" + return re.sub(r"([*?\[\\])", r"\\\1", path) + + +def build_files_command(primary_url: str, media_directory: Path, trash: Optional[Path], + excluded_directories: list = None, playlists_destinations: list = ("playlists",)) -> list: + """The rsync arguments (without `rsync`) that make `media_directory` a copy of the primary's. + + `trash` is where deleted and replaced files are moved; None never deletes. `playlists_destinations` are the + playlists directories of the primary (hard links there, which would arrive as full copies) and of this machine + (its own links, which must not be deleted).""" + arguments = [ + # Not --archive: owner and group are this machine's, and a library holds no devices. + "--recursive", "--perms", "--times", + # A link is copied only if it points inside the library. One that points elsewhere would, on this machine, + # point at this machine's files, which the web server would then serve. + "--links", "--safe-links", + "--partial", f"--partial-dir={PARTIAL_DIRECTORY}", + "--info=progress2,stats2", "--no-human-readable", + *timeout_arguments(), + ] + if trash is not None: + # An excluded path is never deleted, so the trash and this machine's own files are safe from --delete. + arguments.extend(["--delete", "--backup", f"--backup-dir={trash}"]) + arguments.extend(f"--exclude={i}" for i in LOCAL_PATHS) + for playlists in dict.fromkeys(str(i or "").strip("/") for i in playlists_destinations): + if playlists: + arguments.append(f"--exclude=/{rsync_literal(playlists)}") + arguments.extend(f"--exclude={i}" for i in UNFINISHED_PATTERNS) + # Folders the owner chose not to copy. + for directory in excluded_directories or []: + if directory := str(directory).strip("/"): + arguments.append(f"--exclude=/{rsync_literal(directory)}") + arguments.extend([rsync_url(primary_url), f"{media_directory}/"]) + return arguments + + +def build_database_command(primary_url: str, media_directory: Path) -> list: + """The rsync arguments that fetch the primary's database snapshot to `INCOMING_DATABASE`.""" + return [ + "--times", "--partial", "--info=progress2,stats2", "--no-human-readable", + *timeout_arguments(), + # The database in use is what the snapshot is built from, so only what differs is sent. + f"--copy-dest={media_directory / 'config'}", + # Never judged the same by size and time: two databases of one size, written in the same second, are common + # (a database grows a page at a time), and the secondary would keep its own. + "--ignore-times", + rsync_url(primary_url, SNAPSHOT_PATH), str(media_directory / INCOMING_DATABASE), + ] + + +PROGRESS = re.compile(r"^\s*(\d+)\s+(\d+)%") +STATS = { + "files_transferred": re.compile(r"^Number of regular files transferred: (\d+)"), + "files_deleted": re.compile(r"^Number of deleted files: (\d+)"), +} + + +def parse_rsync_line(line: str) -> dict: + """What a line of rsync's output says about its progress, as status changes.""" + if match := PROGRESS.match(line): + return dict(bytes_transferred=int(match.group(1)), percent=int(match.group(2))) + for key, pattern in STATS.items(): + if match := pattern.match(line.replace(",", "")): + return {key: int(match.group(1))} + return {} + + +# --------------------------------------------------------------------------------------------------------------------- +# What differs between a Raspberry Pi or Debian (systemd) and Docker. + + +class SystemdRunner: + """rsync runs here as the wrolpi user, who owns the library; the API is a systemd unit.""" + + def check(self): + if not shutil.which("rsync"): + raise SyncError("rsync is not installed") + + def stop_api(self): + result = stop_service("wrolpi-api") + if not result.get("success"): + raise SyncError(f"Could not stop the API: {result.get('error')}") + + def start_api(self): + start_service("wrolpi-api") + + def stop_rsync(self): + """Nothing to do: rsync is the Controller's child, and systemd stops it with the Controller.""" + + def rsync(self, arguments: list, on_line: Callable[[str], None]) -> int: + command = ["runuser", "-u", "wrolpi", "--", "rsync", *arguments] + process = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return _follow(process.stdout.read1, on_line, process.terminate, process.wait) + + +class DockerRunner: + """rsync runs in a helper container made from the api container (its image, user and volumes), so it writes + where the API reads. The Controller does the rest in its own media directory, which must be the same one.""" + HELPER_NAME = _get_container_name("sync") + + def _api(self): + return _get_client().containers.get(_get_container_name("api")) + + def check(self): + if not can_manage_containers(): + raise SyncError("Docker management is not available") + # Asked of the containers themselves: the paths Docker reports for one folder can differ. + probe = make_directory(get_media_directory() / SYNC_DIRECTORY) / f".probe-{uuid.uuid4().hex}" + write_file(probe, "") + try: + seen = self._api().exec_run(["test", "-e", str(probe)]).exit_code == 0 + finally: + probe.unlink(missing_ok=True) + if not seen: + raise SyncError("The controller and api containers have different media directories; " + "give the controller service the api service's media volume") + + def stop_api(self): + result = stop_container("api") + if not result.get("success"): + raise SyncError(f"Could not stop the API: {result.get('error')}") + + def start_api(self): + start_container("api") + + def stop_rsync(self): + """The helper is its own container: it outlives a Controller that stops, and goes on replacing files.""" + with contextlib.suppress(Exception): + _get_client().containers.get(self.HELPER_NAME).remove(force=True) + + def rsync(self, arguments: list, on_line: Callable[[str], None]) -> int: + client, api = _get_client(), self._api() + self.stop_rsync() + helper = client.containers.run( + api.image.id, arguments, entrypoint="rsync", name=self.HELPER_NAME, detach=True, + user=api.attrs["Config"].get("User") or None, volumes_from=[api.name]) + try: + stream = helper.logs(stream=True, follow=True) + + def read(_: int = 0) -> bytes: + return next(stream, b"") + + return _follow(read, on_line, helper.kill, lambda: helper.wait()["StatusCode"]) + finally: + with contextlib.suppress(Exception): + helper.remove(force=True) + + +def _follow(read: Callable, on_line: Callable[[str], None], stop: Callable, wait: Callable) -> int: + """Hand rsync's output to `on_line` a line at a time (progress lines end in a carriage return). Stops rsync + as soon as the sync is cancelled, even while rsync is silent. Returns rsync's exit code.""" + finished = threading.Event() + + def stop_when_cancelled(): + while not finished.is_set(): + if _cancel.wait(0.2): + with contextlib.suppress(Exception): + stop() + return + + watcher = threading.Thread(target=stop_when_cancelled, daemon=True) + watcher.start() + try: + buffer = b"" + while chunk := read(65536): + buffer += chunk + *lines, buffer = re.split(rb"[\r\n]", buffer) + for line in lines: + if line.strip(): + on_line(line.decode(errors="replace")) + if buffer.strip(): + on_line(buffer.decode(errors="replace")) + code = wait() + finally: + finished.set() + if _cancel.is_set(): + raise SyncCancelled("The sync was cancelled") + return code + + +def get_runner(): + return DockerRunner() if is_docker_mode() else SystemdRunner() + + +# --------------------------------------------------------------------------------------------------------------------- +# The primary's and this machine's API. + + +def _local_api_url() -> str: + return "http://api:8081" if is_docker_mode() else "http://127.0.0.1:8081" + + +def _get_json(url: str, timeout: float = 30) -> dict: + # WROLPi serves HTTPS with its own certificate authority; on a trusted LAN the primary is not verified. + response = httpx.get(url, timeout=timeout, verify=False) + response.raise_for_status() + return response.json() + + +def get_primary_info(primary_url: str) -> dict: + return _get_json(f"{primary_url}/api/sync/info") + + +def get_local_info() -> dict: + return _get_json(f"{_local_api_url()}/api/sync/info") + + +def request_snapshot(primary_url: str) -> dict: + """Ask the primary for a database snapshot and wait for it. Returns the snapshot's status.""" + before = _get_json(f"{primary_url}/api/sync/status").get("snapshot") or {} + response = httpx.post(f"{primary_url}/api/sync/snapshot", timeout=30, verify=False) + if response.status_code >= 400: + with contextlib.suppress(Exception): + raise SyncError(f"The primary made no snapshot: {response.json()['error']}") + raise SyncError(f"The primary made no snapshot: HTTP {response.status_code}") + deadline = time.monotonic() + SNAPSHOT_TIMEOUT_SECONDS + while time.monotonic() < deadline: + _check_cancel() + snapshot = _get_json(f"{primary_url}/api/sync/status").get("snapshot") or {} + finished = snapshot.get("finished_at") + if not snapshot.get("requested") and snapshot.get("state") != "running" \ + and finished and finished != before.get("finished_at"): + if snapshot.get("state") == "error": + raise SyncError(f"The primary made no snapshot: {snapshot.get('last_error')}") + return snapshot + time.sleep(2) + raise SyncError("The primary did not finish its snapshot in time") + + +def wait_for_api(timeout: float = API_START_TIMEOUT_SECONDS) -> bool: + deadline = time.monotonic() + timeout + while time.monotonic() < deadline: + with contextlib.suppress(Exception): + if httpx.get(f"{_local_api_url()}/api/echo", timeout=5).status_code == 200: + return True + time.sleep(2) + return False + + +# --------------------------------------------------------------------------------------------------------------------- +# The steps. + + +def _check_cancel(): + if _cancel.is_set(): + raise SyncCancelled("The sync was cancelled") + if is_wrol_mode(): + raise SyncCancelled("WROL Mode was enabled") + + +def skip_reason(config: dict = None) -> Optional[str]: + """Why a sync cannot start now, or None.""" + config = read_sync_config() if config is None else config + if config.get("role") != "secondary": + return "This WROLPi is not a secondary" + if not config.get("primary_url"): + return "No primary is configured" + if is_wrol_mode(): + return "WROL Mode is enabled" + if not (get_media_directory() / "config/controller.yaml").exists(): + return "The media directory is not mounted" + if _run_lock.locked(): + return "A sync is already running" + return None + + +def check_primary(primary: dict, local: dict): + """Refuse a primary whose library or database this secondary must not take.""" + if primary.get("role") != "primary": + raise SyncError("The primary address is not a primary WROLPi") + if not primary.get("media_mounted"): + # Its media directory would look like an empty library. + raise SyncError("The primary's media directory is not mounted") + for key, name in (("version", "WROLPi version"), ("database_revision", "database migration")): + if not primary.get(key) or primary.get(key) != local.get(key): + raise SyncError(f"The primary's {name} ({primary.get(key)}) is not this WROLPi's ({local.get(key)}); " + "upgrade both to the same version") + if not primary.get("media_directory") or primary.get("media_directory") != local.get("media_directory"): + # The database names every file by its full path. + raise SyncError(f"The primary's media directory ({primary.get('media_directory')}) is not this WROLPi's " + f"({local.get('media_directory')}); its database would find no file here") + + +def replace_database(media_directory: Path): + """Put the fetched snapshot in the database's place. + + The old write-ahead log must be gone first: left beside the new database it would be applied to it. It is + emptied by a checkpoint rather than only deleted, so a run that stops between the two steps leaves the old + database complete.""" + database = checked_path(media_directory / "config/wrolpi.db") + incoming = checked_path(media_directory / INCOMING_DATABASE) + with contextlib.closing(sqlite3.connect(f"file:{incoming}?mode=ro", uri=True)) as connection: + if connection.execute("PRAGMA schema_version").fetchone()[0] == 0: + raise SyncError("The snapshot from the primary is not a database") + if database.is_file(): + with contextlib.closing(sqlite3.connect(database, timeout=60)) as connection: + connection.execute("PRAGMA wal_checkpoint(TRUNCATE)") + for suffix in ("-wal", "-shm"): + database.with_name(f"wrolpi.db{suffix}").unlink(missing_ok=True) + os.replace(incoming, database) + + +def directory_size(directory: Path) -> int: + total = 0 + for root, _, names in os.walk(directory): + for name in names: + with contextlib.suppress(OSError): + total += os.lstat(os.path.join(root, name)).st_size + return total + + +def purge_trash(media_directory: Path, delete_after_days: Optional[int], today: datetime.date = None, + min_free_bytes: int = MIN_FREE_BYTES) -> list: + """Remove the trash folders that have waited `delete_after_days` and, while the drive is nearly full, the + oldest of the rest. A folder is as old as its name says, whatever its files' times are. Returns their names.""" + trash = media_directory / TRASH_DIRECTORY + if trash.is_symlink() or not trash.is_dir(): + return [] + today = today or datetime.date.today() + dated = [] + for folder in trash.iterdir(): + if folder.is_symlink() or not folder.is_dir(): + continue + with contextlib.suppress(ValueError): + dated.append((datetime.date.fromisoformat(folder.name), folder)) + purged = [] + for day, folder in sorted(dated): + expired = delete_after_days is not None and (today - day).days >= delete_after_days + nearly_full = shutil.disk_usage(media_directory).free < min_free_bytes + if not expired and not nearly_full: + continue + logger.info("Removing the sync trash of %s (%s)", folder.name, "expired" if expired else "drive nearly full") + shutil.rmtree(folder, ignore_errors=True) + purged.append(folder.name) + return purged + + +def run_sync(runner=None) -> dict: + """Make this secondary a copy of its primary. Returns the final status.""" + config = read_sync_config() + if reason := skip_reason(config): + logger.info("Not syncing: %s", reason) + return update_status(last_error=reason) + if not _run_lock.acquire(blocking=False): + return update_status(last_error="A sync is already running") + + runner = runner or get_runner() + media_directory = get_media_directory() + api_stopped = False + warnings = [] + _cancel.clear() + # Everything after the lock is taken is inside the `try`: whatever goes wrong, the lock is released. + try: + primary_url = str(config["primary_url"]).rstrip("/") + update_status(save=True, state="running", phase="checks", started_at=_now(), finished_at=None, + last_error=None, primary_url=primary_url, primary_version=None, percent=0, + bytes_transferred=0, files_transferred=0, files_deleted=0, warnings=[]) + logger.warning("Syncing from the primary %s", primary_url) + delete_after_days = get_delete_after_days(config) + runner.check() + primary, local = get_primary_info(primary_url), get_local_info() + update_status(primary_version=primary.get("version")) + check_primary(primary, local) + _check_cancel() + + update_status(phase="snapshot") + snapshot = request_snapshot(primary_url) + # The trash is what fills the drive, and a full drive fails every run: make room before the copy, not + # only after a run that succeeded. + purge_trash(media_directory, delete_after_days) + if shutil.disk_usage(media_directory).free - int(snapshot.get("size") or 0) < MIN_FREE_BYTES: + raise SyncError("Not enough free space for the primary's database") + _check_cancel() + + make_directory(media_directory / SYNC_DIRECTORY / "incoming") + # The primary's wrolpi.yaml names its playlists directory, which must be left out of the copy. + code = runner.rsync(["--times", *timeout_arguments(), rsync_url(primary_url, "config/wrolpi.yaml"), + str(media_directory / PRIMARY_WROLPI_CONFIG)], lambda _: None) + if code != 0: + raise SyncError(f"rsync of the primary's wrolpi.yaml failed with exit code {code}") + write_file(media_directory / PRIMARY_INFO, yaml.safe_dump(primary)) + playlists = [_read_yaml(media_directory / i).get("playlists_destination") or "playlists" + for i in (PRIMARY_WROLPI_CONFIG, "config/wrolpi.yaml")] + + update_status(save=True, phase="files") + runner.stop_api() + api_stopped = True + + trash = None + if delete_after_days is not None: + trash = media_directory / TRASH_DIRECTORY / datetime.date.today().isoformat() + command = build_files_command(primary_url, media_directory, trash, + config.get("sync_excluded_directories"), playlists) + code = runner.rsync(command, lambda line: update_status(**parse_rsync_line(line))) + if code in (RSYNC_PARTIAL, RSYNC_VANISHED): + warnings.append("Some files changed on the primary during the sync, or could not be copied; " + "the next sync copies them") + elif code != 0: + raise SyncError(f"rsync failed with exit code {code}") + _check_cancel() + + update_status(save=True, phase="database", percent=0) + code = runner.rsync( + build_database_command(primary_url, media_directory), + lambda line: update_status(**{k: v for k, v in parse_rsync_line(line).items() if k == "percent"})) + if code != 0: + raise SyncError(f"rsync of the database failed with exit code {code}") + # After this the run cannot be undone by stopping it. + replace_database(media_directory) + + update_status(save=True, phase="starting", percent=100) + runner.start_api() + api_stopped = False + if not wait_for_api(): + warnings.append("The API did not answer after the sync") + + update_status(phase="purge") + purge_trash(media_directory, delete_after_days) + except SyncCancelled as e: + logger.warning("Sync stopped: %s", e) + return _finish(runner, api_stopped, state="idle", last_error=str(e), warnings=warnings) + except Exception as e: + logger.error("Sync failed", exc_info=e) + return _finish(runner, api_stopped, state="error", last_error=str(e) or type(e).__name__, warnings=warnings) + + logger.warning("Synced from the primary %s", config["primary_url"]) + return _finish(runner, api_stopped, state="idle", last_error=None, last_success_at=_now(), warnings=warnings) + + +def _finish(runner, api_stopped: bool, **changes) -> dict: + """Whatever happened, the API runs again and the lock is free.""" + try: + if api_stopped: + runner.start_api() + finally: + _run_lock.release() + trash = get_media_directory() / TRASH_DIRECTORY + return update_status(save=True, phase=None, finished_at=_now(), + trash_bytes=directory_size(trash) if trash.is_dir() and not trash.is_symlink() else 0, + **changes) + + +def _read_yaml(file: Path) -> dict: + try: + return yaml.safe_load(file.read_text()) or {} + except (OSError, yaml.YAMLError): + return {} + + +def request_cancel(): + """Stop the running sync at its next step, or its rsync at once.""" + if _run_lock.locked(): + _cancel.set() + + +def recover_interrupted_sync(runner=None): + """Called when the Controller starts. A sync that was running when the Controller stopped left the API stopped; + start it, and say the sync was interrupted.""" + status = get_status() + if status["state"] != "running": + return + logger.warning("A sync was interrupted; starting the API") + runner = runner or get_runner() + # Before the API starts: nothing may still be replacing the library under it. + with contextlib.suppress(Exception): + runner.stop_rsync() + with contextlib.suppress(Exception): + runner.start_api() + update_status(save=True, state="error", phase=None, finished_at=_now(), + last_error="The sync was interrupted") diff --git a/controller/main.py b/controller/main.py index c4241a78..7f8a4230 100644 --- a/controller/main.py +++ b/controller/main.py @@ -33,6 +33,7 @@ from controller.api.schemas import HealthResponse from controller.api.scripts import router as scripts_router from controller.api.services import router as services_router from controller.api.status import router as status_router +from controller.api.sync import router as sync_router from controller.lib.config import ( get_ca_certificate_path, get_config_value, @@ -58,6 +59,7 @@ from controller.lib.status import ( get_uptime_status, ) from controller.lib.status_worker import status_worker_loop +from controller.lib.sync import recover_interrupted_sync from controller.lib.systemd import get_all_services_status # Templates directory @@ -124,6 +126,9 @@ async def lifespan(app: FastAPI): else: logger.info("Primary drive not mounted - using default configuration") + # A sync that was running when the Controller stopped left the API stopped. + app.state.sync_recovery_task = asyncio.create_task(asyncio.to_thread(recover_interrupted_sync)) + # Docker: the library is served to secondary WROLPis from a helper container, kept matching the api container. if is_docker_mode(): app.state.rsync_daemon_task = asyncio.create_task(rsync_daemon_loop()) @@ -176,6 +181,7 @@ app.include_router(ready_router) app.include_router(scripts_router) app.include_router(services_router) app.include_router(status_router) +app.include_router(sync_router) def get_host(request: Request) -> str: diff --git a/controller/test/test_sync.py b/controller/test/test_sync.py new file mode 100644 index 00000000..0761b8f1 --- /dev/null +++ b/controller/test/test_sync.py @@ -0,0 +1,618 @@ +""" +Unit tests for controller.lib.sync: a secondary WROLPi copying its primary with rsync. +""" + +import datetime +import os +import shutil +import socket +import sqlite3 +import subprocess +import threading +import time +from collections import namedtuple +from pathlib import Path + +import pytest +import yaml +from fastapi.testclient import TestClient + +from controller.lib import rsyncd, sync +from controller.main import app + +PRIMARY = "https://primary.example:8443" +INFO = dict(role="primary", version="1.0", database_revision="abc", media_mounted=True, media_directory="/media/wrolpi") +RSYNC_3 = bool(shutil.which("rsync")) and b"rsync version 3" in subprocess.run( + ["rsync", "--version"], capture_output=True).stdout + + +@pytest.fixture +def media(tmp_path, monkeypatch) -> Path: + """This machine's media directory: a mounted secondary of PRIMARY.""" + directory = tmp_path / "media" + (directory / "config").mkdir(parents=True) + (directory / "config/controller.yaml").write_text("{}") + (directory / "config/sync.yaml").write_text(yaml.safe_dump(dict(role="secondary", primary_url=PRIMARY))) + monkeypatch.setenv("MEDIA_DIRECTORY", str(directory)) + monkeypatch.setattr(sync, "_status", None) + sync._cancel.clear() + yield directory + assert not sync._run_lock.locked(), "the test left a sync running" + + +def make_database(file: Path, value: str): + file.parent.mkdir(parents=True, exist_ok=True) + with sqlite3.connect(file) as connection: + connection.execute("CREATE TABLE t (name TEXT)") + connection.execute("INSERT INTO t VALUES (?)", (value,)) + connection.close() + + +def read_database(file: Path) -> str: + connection = sqlite3.connect(f"file:{file}?mode=ro", uri=True) + try: + return connection.execute("SELECT name FROM t").fetchone()[0] + finally: + connection.close() + + +class FakeRunner: + """Records what a sync does to the API, and plays rsync.""" + + def __init__(self, media: Path, code: int = 0, on_files=None): + self.media, self.code, self.on_files = media, code, on_files + self.events = [] + + def check(self): + self.events.append("check") + + def stop_api(self): + self.events.append("stop_api") + + def start_api(self): + self.events.append("start_api") + + def stop_rsync(self): + self.events.append("stop_rsync") + + def rsync(self, arguments, on_line): + if arguments[-1].endswith("/"): + self.files_command = arguments + self.events.append("files") + on_line(" 1234567 42% 10.00MB/s 0:00:01") + on_line("Number of regular files transferred: 7") + if self.on_files: + self.on_files() + return self.code + if arguments[-1].endswith("primary-wrolpi.yaml"): + Path(arguments[-1]).write_text("videos_destination: videos\nplaylists_destination: lists\n") + return 0 + self.events.append("database") + make_database(Path(arguments[-1]), "from the primary") + return 0 + + +@pytest.fixture +def primary_api(monkeypatch): + """The primary's and this machine's APIs answer; both run the same WROLPi.""" + monkeypatch.setattr(sync, "get_primary_info", lambda url: dict(INFO)) + monkeypatch.setattr(sync, "get_local_info", lambda: dict(INFO, role="secondary")) + monkeypatch.setattr(sync, "request_snapshot", lambda url: dict(size=1000, written=True)) + monkeypatch.setattr(sync, "wait_for_api", lambda: True) + + +class TestCommand: + + def test_copies_everything_but_what_is_local(self, media): + command = sync.build_files_command(PRIMARY, media, media / ".sync-trash/2026-11-18", ["old [stuff]", "a/b/"], + ["my playlists", "lists", "my playlists"]) + assert command[-2:] == ["rsync://primary.example:8873/media/", f"{media}/"] + assert "--delete" in command and "--backup" in command + assert f"--backup-dir={media}/.sync-trash/2026-11-18" in command + for excluded in ("/tags", "/my playlists", "/lists", "/config/wrolpi.yaml", "/config/sync.yaml", "/config/ssl", + "/config/wrolpi.db", "/config/wrolpi.db-wal", "/config/.sync", "/.sync-trash", + "/config/sync_state.yaml", "/config/.wrol_mode", "*.part", ".partial", "*.db-wal", + "/old \\[stuff]", "/a/b"): + assert f"--exclude={excluded}" in command, excluded + # A pattern ending in a slash matches only a directory; a file or link of that name would get through. + assert not [i for i in command if i.startswith("--exclude=") and i.endswith("/")] + assert command.count("--exclude=/my playlists") == 1 + # A primary that stops answering ends the run; rsync alone would wait forever with the API stopped. + assert "--contimeout=30" in command and "--timeout=300" in command + # Only links that stay inside the library, and no device files. + assert "--safe-links" in command and "--devices" not in command and "--specials" not in command + # An ignored directory is still part of the library: nothing leaves out the map or the repos. + assert not [i for i in command if "map" in i or "repos" in i] + # Owner and group are this machine's. + assert "--archive" not in command and "--owner" not in command + + def test_database_is_never_judged_the_same_by_size_and_time(self, media): + command = sync.build_database_command(PRIMARY, media) + assert "--ignore-times" in command and f"--copy-dest={media}/config" in command + assert "--contimeout=30" in command and "--timeout=300" in command + assert command[-2:] == ["rsync://primary.example:8873/media/config/.sync/config/wrolpi.db", + f"{media}/config/.sync/incoming/wrolpi.db"] + + def test_never_deletes_without_a_trash(self, media): + command = sync.build_files_command(PRIMARY, media, None) + assert "--delete" not in command and "--backup" not in command + + @pytest.mark.parametrize("url, expected", [ + ("https://10.0.0.5", "rsync://10.0.0.5:8873/media/"), + ("https://wrolpi.local:8443", "rsync://wrolpi.local:8873/media/"), + ("https://[fd00::5]:8443", "rsync://[fd00::5]:8873/media/"), + ]) + def test_rsync_url(self, url, expected): + assert sync.rsync_url(url) == expected + assert sync.rsync_url(url, "config/wrolpi.yaml") == expected + "config/wrolpi.yaml" + + def test_rsync_url_invalid(self): + with pytest.raises(sync.SyncError): + sync.rsync_url("not an address") + + @pytest.mark.parametrize("line, expected", [ + (" 32768000 12% 31.25MB/s 0:00:07 (xfr#3, to-chk=10/20)", + dict(bytes_transferred=32768000, percent=12)), + ("Number of regular files transferred: 1,234", dict(files_transferred=1234)), + ("Number of deleted files: 3 (reg: 2, dir: 1)", dict(files_deleted=3)), + ("sending incremental file list", {}), + ]) + def test_parse_rsync_line(self, line, expected): + assert sync.parse_rsync_line(line) == expected + + @pytest.mark.parametrize("config, expected", [ + ({}, 30), (dict(delete_after_days=7), 7), (dict(delete_after_days=0), 0), (dict(delete_after_days=None), None), + ]) + def test_delete_after_days(self, config, expected): + assert sync.get_delete_after_days(config) == expected + + +class TestRootInAnotherUsersDirectory: + """The Controller runs as root; the media directory belongs to the wrolpi user, who must not be able to point + root somewhere else with a symbolic link.""" + + def test_state_is_not_written_through_a_link(self, media, tmp_path): + target = tmp_path / "somewhere else.txt" + target.write_text("untouched") + (media / "config/sync_state.yaml").symlink_to(target) + sync.update_status(save=True, last_error="x") + assert target.read_text() == "untouched" + + def test_write_file_replaces_never_follows(self, media, tmp_path): + target = tmp_path / "somewhere else.txt" + target.write_text("untouched") + (media / "config/.sync").symlink_to(tmp_path) + with pytest.raises(sync.SyncError, match="symbolic link"): + sync.write_file(media / sync.PRIMARY_INFO, "x") + with pytest.raises(sync.SyncError, match="symbolic link"): + sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming") + assert not (tmp_path / "incoming").exists() and not (tmp_path / "primary-info.yaml").exists() + + def test_directories_belong_to_the_library_owner(self, media, monkeypatch): + """rsync does not run as root, and must write into what the Controller made.""" + chowned = [] + monkeypatch.setattr(sync.os, "chown", lambda path, uid, gid, follow_symlinks=True: chowned.append( + (Path(path).relative_to(media).as_posix(), uid, gid))) + owner = media.stat() + sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming") + assert sorted(chowned) == [("config/.sync", owner.st_uid, owner.st_gid), + ("config/.sync/incoming", owner.st_uid, owner.st_gid)] + chowned.clear() + sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming") + assert chowned == [], "only what it made" + + def test_trash_that_is_a_link_is_not_purged(self, media, tmp_path): + elsewhere = tmp_path / "elsewhere" + (elsewhere / "2020-01-01").mkdir(parents=True) + (media / ".sync-trash").symlink_to(elsewhere) + assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=0) == [] + assert (elsewhere / "2020-01-01").is_dir() + + (media / ".sync-trash").unlink() + (media / ".sync-trash").mkdir() + (media / ".sync-trash/2020-01-02").symlink_to(elsewhere / "2020-01-01") + assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=0) == [] + assert (elsewhere / "2020-01-01").is_dir() + + def test_database_that_is_a_link_is_not_replaced(self, media, tmp_path): + make_database(tmp_path / "elsewhere.db", "somewhere else") + (media / "config/wrolpi.db").symlink_to(tmp_path / "elsewhere.db") + make_database(media / sync.INCOMING_DATABASE, "from the primary") + with pytest.raises(sync.SyncError, match="symbolic link"): + sync.replace_database(media) + assert read_database(tmp_path / "elsewhere.db") == "somewhere else" + + +class TestChecks: + + def test_skip_reason(self, media): + assert sync.skip_reason() is None + assert sync.skip_reason(dict(role="primary", primary_url=PRIMARY)) == "This WROLPi is not a secondary" + assert sync.skip_reason(dict(role="secondary")) == "No primary is configured" + (media / "config/.wrol_mode").touch() + assert sync.skip_reason() == "WROL Mode is enabled" + (media / "config/.wrol_mode").unlink() + (media / "config/controller.yaml").unlink() + assert sync.skip_reason() == "The media directory is not mounted" + + @pytest.mark.parametrize("change, message", [ + (dict(role="secondary"), "not a primary"), + (dict(media_mounted=False), "not mounted"), + (dict(version="2.0"), "WROLPi version"), + (dict(database_revision="def"), "database migration"), + (dict(database_revision=None), "database migration"), + # The database names every file by its full path. + (dict(media_directory="/data/library"), "media directory"), + (dict(media_directory=None), "media directory"), + ]) + def test_check_primary_refuses(self, change, message): + with pytest.raises(sync.SyncError, match=message): + sync.check_primary(dict(INFO, **change), dict(INFO)) + + def test_check_primary(self): + sync.check_primary(dict(INFO), dict(INFO, role="secondary")) + + +class TestPurge: + + def test_by_the_name_of_the_folder(self, media): + """A folder is as old as its name says; files keep the modification times they had in the library.""" + today = datetime.date(2026, 11, 18) + trash = media / ".sync-trash" + for name in ("2026-10-18", "2026-10-19", "2026-10-20", "2026-11-18", "not a date"): + (trash / name).mkdir(parents=True) + (trash / name / "file.txt").write_text("deleted on the primary") + assert sync.purge_trash(media, 30, today, min_free_bytes=0) == ["2026-10-18", "2026-10-19"] + assert sorted(i.name for i in trash.iterdir()) == ["2026-10-20", "2026-11-18", "not a date"] + + def test_never(self, media): + (media / ".sync-trash/2020-01-01").mkdir(parents=True) + assert sync.purge_trash(media, None, datetime.date(2026, 11, 18), min_free_bytes=0) == [] + + def test_oldest_first_when_the_drive_is_nearly_full(self, media, monkeypatch): + """A full drive is worse than a shorter wait: the oldest trash goes until there is room again.""" + trash = media / ".sync-trash" + for name in ("2026-11-16", "2026-11-17", "2026-11-18"): + (trash / name).mkdir(parents=True) + usage = namedtuple("usage", "total used free") + monkeypatch.setattr(sync.shutil, "disk_usage", + lambda _: usage(100, 0, 5 if (trash / "2026-11-16").exists() else 50)) + assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=10) == ["2026-11-16"] + + +class TestReplaceDatabase: + + def test_replaces_and_removes_the_old_log(self, media): + """The old write-ahead log left beside the new database would be applied to it.""" + database = media / "config/wrolpi.db" + make_database(database, "this secondary's") + live = sqlite3.connect(database) + live.execute("PRAGMA journal_mode=WAL") + live.execute("UPDATE t SET name = 'written to the log'") + live.commit() + assert database.with_name("wrolpi.db-wal").stat().st_size > 0 + make_database(media / sync.INCOMING_DATABASE, "from the primary") + + sync.replace_database(media) + live.close() + + assert read_database(database) == "from the primary" + assert not database.with_name("wrolpi.db-wal").exists() and not database.with_name("wrolpi.db-shm").exists() + assert not (media / sync.INCOMING_DATABASE).exists() + + def test_refuses_what_is_not_a_database(self, media): + make_database(media / "config/wrolpi.db", "this secondary's") + incoming = media / sync.INCOMING_DATABASE + incoming.parent.mkdir(parents=True) + incoming.write_bytes(b"") + with pytest.raises(sync.SyncError, match="not a database"): + sync.replace_database(media) + assert read_database(media / "config/wrolpi.db") == "this secondary's" + + +class TestRunSync: + + def test_order_and_outcome(self, media, primary_api): + """The API is down only while the library is replaced, and what the run did survives a restart.""" + make_database(media / "config/wrolpi.db", "this secondary's") + runner = FakeRunner(media) + + status = sync.run_sync(runner) + + assert runner.events == ["check", "stop_api", "files", "database", "start_api"] + assert status["state"] == "idle" and status["last_error"] is None and status["last_success_at"] + assert status["files_transferred"] == 7 and status["primary_version"] == "1.0" + assert read_database(media / "config/wrolpi.db") == "from the primary" + assert yaml.safe_load((media / sync.PRIMARY_INFO).read_text())["media_directory"] == "/media/wrolpi" + assert (media / sync.PRIMARY_WROLPI_CONFIG).is_file() + # A restarted Controller reads the state file. + sync._status = None + assert sync.get_status()["last_success_at"] == status["last_success_at"] + + def test_leaves_out_the_playlists_of_both_machines(self, media, primary_api): + """The primary names its playlists directory in its own wrolpi.yaml, which is fetched before the copy: its + hard links would arrive as full copies. This machine's own playlists directory must not be deleted.""" + (media / "config/wrolpi.yaml").write_text("playlists_destination: mine\n") + runner = FakeRunner(media) + sync.run_sync(runner) + assert "--exclude=/lists" in runner.files_command and "--exclude=/mine" in runner.files_command + + def test_trash_is_purged_even_when_the_copy_fails(self, media, primary_api): + """A full drive fails every copy, and the trash is what fills it: it must be emptied before the copy.""" + (media / ".sync-trash/2020-01-01").mkdir(parents=True) + status = sync.run_sync(FakeRunner(media, code=12)) + assert status["state"] == "error" + assert not (media / ".sync-trash/2020-01-01").exists() + + @pytest.mark.parametrize("config", [ + dict(delete_after_days="soon"), dict(primary_url=["not", "text"]), dict(sync_excluded_directories=5), + ]) + def test_a_bad_setting_ends_the_run_and_frees_it(self, media, primary_api, config): + """sync.yaml can be edited by hand. A value that cannot be used must not leave "A sync is already running" + behind until the Controller restarts.""" + (media / "config/sync.yaml").write_text(yaml.safe_dump(dict(dict(role="secondary", primary_url=PRIMARY), + **config))) + runner = FakeRunner(media) + status = sync.run_sync(runner) + assert status["state"] == "error" and status["last_error"] + assert not sync._run_lock.locked() + assert runner.events.count("stop_api") == runner.events.count("start_api") + assert sync.skip_reason(dict(role="secondary", primary_url=PRIMARY)) is None + + def test_a_refused_primary_changes_nothing(self, media, primary_api, monkeypatch): + monkeypatch.setattr(sync, "get_primary_info", lambda url: dict(INFO, version="2.0")) + runner = FakeRunner(media) + status = sync.run_sync(runner) + assert status["state"] == "error" and "upgrade both" in status["last_error"] + assert runner.events == ["check"], "the API was never stopped" + + def test_failed_files_leave_the_database_and_start_the_api(self, media, primary_api): + make_database(media / "config/wrolpi.db", "this secondary's") + runner = FakeRunner(media, code=12) + status = sync.run_sync(runner) + assert status["state"] == "error" and "exit code 12" in status["last_error"] + assert runner.events == ["check", "stop_api", "files", "start_api"] + assert read_database(media / "config/wrolpi.db") == "this secondary's" + assert status["last_success_at"] is None + + def test_files_that_vanished_are_only_a_warning(self, media, primary_api): + """The primary is in use while it is copied.""" + status = sync.run_sync(FakeRunner(media, code=sync.RSYNC_VANISHED)) + assert status["state"] == "idle" and status["last_success_at"] and status["warnings"] + + def test_cancelled(self, media, primary_api): + make_database(media / "config/wrolpi.db", "this secondary's") + runner = FakeRunner(media, on_files=sync.request_cancel) + status = sync.run_sync(runner) + assert status["state"] == "idle" and status["last_error"] == "The sync was cancelled" + assert runner.events[-1] == "start_api" and "database" not in runner.events + assert read_database(media / "config/wrolpi.db") == "this secondary's" + + def test_wrol_mode_stops_it(self, media, primary_api): + runner = FakeRunner(media, on_files=lambda: (media / "config/.wrol_mode").touch()) + status = sync.run_sync(runner) + assert status["last_error"] == "WROL Mode was enabled" and runner.events[-1] == "start_api" + + def test_not_a_secondary(self, media, primary_api): + (media / "config/sync.yaml").write_text("role: primary\n") + runner = FakeRunner(media) + assert sync.run_sync(runner)["last_error"] == "This WROLPi is not a secondary" + assert runner.events == [] + + def test_interrupted_sync_is_recovered(self, media): + """A Controller that stopped during a sync left the API stopped.""" + sync.update_status(save=True, state="running", phase="files") + sync._status = None + runner = FakeRunner(media) + sync.recover_interrupted_sync(runner) + # In Docker rsync is its own container and is still replacing files: it is stopped before the API starts. + assert runner.events == ["stop_rsync", "start_api"] + assert sync.get_status()["state"] == "error" and sync.get_status()["last_error"] == "The sync was interrupted" + + runner = FakeRunner(media) + sync.recover_interrupted_sync(runner) + assert runner.events == [], "nothing to recover the second time" + + +class TestApi: + + def test_status_run_cancel(self, media, monkeypatch): + client = TestClient(app) + assert client.get("/api/sync/status").json()["state"] == "idle" + + started = threading.Event() + monkeypatch.setattr(sync, "run_sync", started.set) + assert client.post("/api/sync/run").status_code == 202 + assert started.wait(5) + assert client.post("/api/sync/cancel").status_code == 202 + + (media / "config/sync.yaml").write_text("role: primary\n") + response = client.post("/api/sync/run") + assert response.status_code == 409 and response.json()["detail"] == "This WROLPi is not a secondary" + + +class LocalRunner(sync.SystemdRunner): + """Real rsync, as whoever runs the tests; the API is only recorded.""" + + def __init__(self): + self.events = [] + + def stop_api(self): + self.events.append("stop_api") + + def start_api(self): + self.events.append("start_api") + + def rsync(self, arguments, on_line): + process = subprocess.Popen(["rsync", *arguments], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) + return sync._follow(process.stdout.read1, on_line, process.terminate, process.wait) + + +def write(root: Path, files: dict): + for path, content in files.items(): + (root / path).parent.mkdir(parents=True, exist_ok=True) + (root / path).write_text(content) + + +@pytest.mark.skipif(not RSYNC_3, reason="needs rsync 3") +def test_sync_with_a_real_daemon(media, tmp_path, primary_api, monkeypatch): + """A whole sync against a real rsync daemon run with the real config: the secondary becomes a copy of its + primary, ignored directories included; what the primary deleted or changed waits in the trash; what belongs to + this machine is left alone; and a second sync finds nothing to do.""" + primary = tmp_path / "primary" + write(primary, { + "videos/a.mp4": "a video", "videos/changed.txt": "the primary's newer version", + # Ignored on the primary, so in no FileGroup: the old sync never copied these. + "map/region.pmtiles": "a map", "repos/tool/README.md": "a repo", + "map/.partial/unfinished.pmtiles": "half a map", "map/region.search.db-wal": "in use", + "excluded/big.bin": "not wanted here", + "tags/one/a.mp4": "a hard link on the primary", "playlists/Watch/0001_a.mp4": "a hard link on the primary", + "config/tags.yaml": "the primary's tags", "config/wrolpi.yaml": "videos_destination: videos\n", + "config/sync.yaml": "role: primary\n", "config/controller.yaml": "the primary's", + "config/ssl/ca.key": "the primary's key", "config/wrolpi.db": "the primary's database, in use", + }) + make_database(primary / sync.SNAPSHOT_PATH, "from the primary") + # A link inside the library is part of it; one that points outside would point at this machine's files. + (primary / "videos/inside").symlink_to("a.mp4") + (primary / "videos/outside").symlink_to("/etc/hostname") + (primary / "videos/climbs out").symlink_to("../../outside.txt") + # Where this machine has a directory of its own, the primary has a file of that name. + (primary / "config/backup").write_text("not a directory on the primary") + write(media, {"config/backup/tags-20261118.yaml": "this machine's config backup"}) + write(media, { + "videos/changed.txt": "older", "videos/deleted on the primary.txt": "gone there", + "tags/mine/x.txt": "this machine's links", "config/wrolpi.yaml": "hotspot_ssid: mine\n", + "config/ssl/ca.key": "this machine's key", + }) + (media / "config/sync.yaml").write_text(yaml.safe_dump(dict( + role="secondary", primary_url="https://127.0.0.1:8443", sync_excluded_directories=["excluded"]))) + make_database(media / "config/wrolpi.db", "this secondary's") + # The same size, written in the same second: the two databases look the same to a quick comparison. + snapshot_stat = (primary / sync.SNAPSHOT_PATH).stat() + assert (media / "config/wrolpi.db").stat().st_size == snapshot_stat.st_size + os.utime(media / "config/wrolpi.db", ns=(snapshot_stat.st_atime_ns, snapshot_stat.st_mtime_ns)) + + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + config = rsyncd.read_rsyncd_config().replace("port = 8873", f"port = {port}\naddress = 127.0.0.1") \ + .replace("path = /media/wrolpi", f"path = {primary}").replace("log file = /dev/stderr", "") + (tmp_path / "rsyncd.conf").write_text(config) + monkeypatch.setattr(sync, "get_rsyncd_port", lambda: port) + daemon = subprocess.Popen(["rsync", "--daemon", "--no-detach", f"--config={tmp_path / 'rsyncd.conf'}"]) + try: + for _ in range(50): + if subprocess.run(["rsync", f"rsync://127.0.0.1:{port}/"], capture_output=True).returncode == 0: + break + time.sleep(0.1) + runner = LocalRunner() + + status = sync.run_sync(runner) + + assert status["state"] == "idle" and status["last_error"] is None, status + assert runner.events == ["stop_api", "start_api"] + for path in ("videos/a.mp4", "map/region.pmtiles", "repos/tool/README.md", "config/tags.yaml"): + assert (media / path).read_text() == (primary / path).read_text(), path + assert (media / "videos/changed.txt").read_text() == "the primary's newer version" + assert (media / "videos/a.mp4").stat().st_mtime == (primary / "videos/a.mp4").stat().st_mtime + # Deleted and replaced files wait in today's trash. + trash = media / ".sync-trash" / datetime.date.today().isoformat() + assert (trash / "videos/deleted on the primary.txt").read_text() == "gone there" + assert (trash / "videos/changed.txt").read_text() == "older" + assert not (media / "videos/deleted on the primary.txt").exists() + # This machine's own, and what WROLPi rebuilds, are neither copied nor deleted. + assert (media / "config/wrolpi.yaml").read_text() == "hotspot_ssid: mine\n" + assert yaml.safe_load((media / "config/sync.yaml").read_text())["role"] == "secondary" + assert (media / "config/ssl/ca.key").read_text() == "this machine's key" + assert (media / "config/controller.yaml").read_text() == "{}" + assert (media / "tags/mine/x.txt").exists() and not (media / "tags/one").exists() + assert not (media / "playlists").exists() + assert (media / "config/backup/tags-20261118.yaml").read_text() == "this machine's config backup" + assert (media / "videos/inside").is_symlink() and (media / "videos/inside").read_text() == "a video" + assert not (media / "videos/outside").is_symlink() and not (media / "videos/climbs out").is_symlink() + # Unfinished files and the excluded folder stay on the primary. + assert not (media / "map/.partial").exists() and not (media / "map/region.search.db-wal").exists() + assert not (media / "excluded").exists() + # The database is the primary's snapshot, never its database in use; the library settings come along. + assert read_database(media / "config/wrolpi.db") == "from the primary" + assert (media / sync.PRIMARY_WROLPI_CONFIG).read_text() == "videos_destination: videos\n" + assert not (media / sync.SNAPSHOT_PATH).exists() + assert status["files_transferred"] >= 4 and status["files_deleted"] >= 1 and status["trash_bytes"] > 0 + + # Nothing changed: nothing is copied, and nothing more goes to the trash. + before = sorted(i for i in trash.rglob("*")) + status = sync.run_sync(LocalRunner()) + assert status["state"] == "idle" and status["files_transferred"] == 0 and status["files_deleted"] == 0 + assert sorted(i for i in trash.rglob("*")) == before + finally: + daemon.terminate() + daemon.wait(timeout=10) + + +class TestDockerRunner: + + def make(self, monkeypatch, media, exit_code): + from unittest import mock + api = mock.Mock() + seen = [] + + def exec_run(command): + seen.append(Path(command[-1]).exists()) + return mock.Mock(exit_code=exit_code) + + api.exec_run.side_effect = exec_run + client = mock.Mock() + client.containers.get.return_value = api + monkeypatch.setattr(sync, "can_manage_containers", lambda: True) + monkeypatch.setattr(sync, "_get_client", lambda: client) + return sync.DockerRunner(), seen + + def test_refuses_when_the_api_container_has_another_media_directory(self, media, monkeypatch): + """The Controller replaces the database in its own media directory; rsync writes to the api container's. + A probe file the api container cannot see means they are two directories.""" + runner, seen = self.make(monkeypatch, media, exit_code=1) + with pytest.raises(sync.SyncError, match="different media directories"): + runner.check() + assert seen == [True], "the probe existed while the api container was asked" + assert not list((media / sync.SYNC_DIRECTORY).glob(".probe-*")), "and is removed" + + def test_stop_rsync_removes_the_helper(self, media, monkeypatch): + from unittest import mock + helper = mock.Mock() + client = mock.Mock() + client.containers.get.return_value = helper + monkeypatch.setattr(sync, "_get_client", lambda: client) + sync.DockerRunner().stop_rsync() + client.containers.get.assert_called_once_with("wrolpi-sync-1") + helper.remove.assert_called_once_with(force=True) + + client.containers.get.side_effect = RuntimeError("no such container") + sync.DockerRunner().stop_rsync() + + def test_accepts_the_same_media_directory(self, media, monkeypatch): + runner, _ = self.make(monkeypatch, media, exit_code=0) + runner.check() + assert not list((media / sync.SYNC_DIRECTORY).glob(".probe-*")) + + +@pytest.mark.skipif(not RSYNC_3, reason="needs rsync 3") +def test_a_primary_that_stops_answering_ends_rsync(media, monkeypatch): + """A primary that accepts the connection and then says nothing (it lost power mid-copy, as far as TCP can tell): + rsync gives up, so the run ends and the API is started again.""" + monkeypatch.setattr(sync, "RSYNC_IO_TIMEOUT_SECONDS", 2) + monkeypatch.setattr(sync, "RSYNC_CONNECT_TIMEOUT_SECONDS", 2) + with socket.socket() as silent: + silent.bind(("127.0.0.1", 0)) + silent.listen() + monkeypatch.setattr(sync, "get_rsyncd_port", lambda: silent.getsockname()[1]) + # Without its timeouts rsync never returns; stop it, so that is a failure here and not a test that hangs. + gave_up = threading.Timer(30, sync._cancel.set) + gave_up.start() + try: + code = LocalRunner().rsync(sync.build_files_command("https://127.0.0.1", media, None), lambda _: None) + except sync.SyncCancelled: + pytest.fail("rsync waited on a silent primary until it was stopped") + finally: + gave_up.cancel() + sync._cancel.clear() + assert code != 0 diff --git a/main.py b/main.py index 1778af70..14b47959 100755 --- a/main.py +++ b/main.py @@ -22,6 +22,7 @@ from wrolpi.downloader import download_manager, get_download_manager_config from wrolpi.errors import WROLModeEnabled from wrolpi.files.worker import file_worker from wrolpi.perpetual import register_perpetual_process, perpetual_process_health_check +from wrolpi.sync.lib import apply_primary_wrolpi_config from wrolpi.vars import PROJECT_DIR, DOCKERIZED from wrolpi.version import get_version_string @@ -275,6 +276,11 @@ async def start_single_tasks(app: Sanic): # Import configs that require the database. if wrolpi_config.successful_import and not wrolpi_config.wrol_mode: + try: + # A secondary's library settings are its primary's; before the configs that depend on them. + apply_primary_wrolpi_config() + except Exception as e: + logger.error("Failed to apply the primary's library settings", exc_info=e) from wrolpi.common import import_all_db_configs await import_all_db_configs() diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index c778e8f5..ad436d28 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -217,6 +217,7 @@ def get_settings(_: Request): 'role': sync_config.role, 'save_ffprobe_json': wrolpi_config.save_ffprobe_json, 'sync_primary_url': sync_config.primary_url, + 'sync_delete_after_days': sync_config.delete_after_days, 'sync_excluded_directories': sync_config.sync_excluded_directories, 'sync_window_end': sync_config.window_end, 'sync_window_start': sync_config.window_start, diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 03b74d0f..161e86a1 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -45,6 +45,8 @@ class SettingsResponse: require_cookies_unlocked: bool role: str sync_primary_url: Optional[str] + # None never deletes. + sync_delete_after_days: Optional[int] sync_excluded_directories: List[str] sync_window_end: Optional[str] sync_window_start: Optional[str] diff --git a/wrolpi/sync/api.py b/wrolpi/sync/api.py index 0d8fc858..eee301f7 100644 --- a/wrolpi/sync/api.py +++ b/wrolpi/sync/api.py @@ -171,6 +171,9 @@ async def put_settings(_: Request, body: schema.SyncSettingsRequest): if value and not re.match(r'^([01]\d|2[0-3]):[0-5]\d$', value): raise InvalidConfig(f'{request_key} must be in HH:MM format') changes[config_key] = value or None + if body.sync_delete_after_days is not None: + days = int(body.sync_delete_after_days) + changes['delete_after_days'] = None if days < 0 else days for key in ('sync_excluded_directories', 'backup_excluded_directories'): if (value := getattr(body, key)) is not None: changes[key] = lib.normalize_excluded_directories(value) diff --git a/wrolpi/sync/config.py b/wrolpi/sync/config.py index da2c98f3..cdeab874 100644 --- a/wrolpi/sync/config.py +++ b/wrolpi/sync/config.py @@ -21,6 +21,8 @@ class SyncConfigValidator: window_end: str = None # Folders (relative to the media directory) a secondary never downloads from its primary. sync_excluded_directories: list[str] = field(default_factory=list) + # Days a secondary keeps a file its primary deleted or replaced (in `.sync-trash`); None never deletes. + delete_after_days: int = None # A second drive this WROLPi copies its media directory to (see wrolpi.sync.backup), and the UUID of the drive # that must be mounted there. backup_directory: str = None @@ -34,6 +36,10 @@ class SyncConfigValidator: backup_last_success_at: str = None +# Long enough to notice a file that should not have been deleted on the primary. +DEFAULT_DELETE_AFTER_DAYS = 30 + + class SyncConfig(ConfigFile): file_name = 'sync.yaml' default_config = dict( @@ -43,6 +49,7 @@ class SyncConfig(ConfigFile): window_start=None, window_end=None, sync_excluded_directories=[], + delete_after_days=DEFAULT_DELETE_AFTER_DAYS, backup_directory=None, backup_device_uuid=None, backup_window_start=None, @@ -78,6 +85,13 @@ class SyncConfig(ConfigFile): def sync_excluded_directories(self) -> List[str]: return list(self._config.get('sync_excluded_directories') or []) + @property + def delete_after_days(self) -> Optional[int]: + """None never deletes. A sync.yaml written before this setting has no key, and gets the default.""" + if 'delete_after_days' not in self._config: + return DEFAULT_DELETE_AFTER_DAYS + return self._config['delete_after_days'] + @property def backup_directory(self) -> Optional[str]: return self._config.get('backup_directory') diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py index 7388f23e..0c843bff 100644 --- a/wrolpi/sync/lib.py +++ b/wrolpi/sync/lib.py @@ -704,6 +704,31 @@ def merge_wrolpi_config(primary_config: dict, primary_media_directory: str) -> d return changes +# Where the Controller's sync (controller/lib/sync.py) leaves the primary's wrolpi.yaml and /api/sync/info. +PRIMARY_WROLPI_CONFIG = 'config/.sync/primary-wrolpi.yaml' +PRIMARY_INFO = 'config/.sync/primary-info.yaml' + + +def apply_primary_wrolpi_config() -> List[str]: + """A secondary's wrolpi.yaml is its own (hotspot, timezone), except for the settings that describe the library: + those are taken from the copy of the primary's wrolpi.yaml that the last sync left. Returns the changed keys.""" + if not is_secondary(): + return [] + media_directory = get_media_directory() + try: + primary_config = _parse_config((media_directory / PRIMARY_WROLPI_CONFIG).read_bytes(), 'wrolpi.yaml') + info = yaml.safe_load((media_directory / PRIMARY_INFO).read_text()) or {} + except (OSError, yaml.YAMLError, SyncError) as e: + if not isinstance(e, FileNotFoundError): + logger.error(f"Could not read the primary's wrolpi.yaml: {e}") + return [] + changes = merge_wrolpi_config(primary_config, str(info.get('media_directory') or media_directory)) + if changes: + logger.info(f'Applying library settings from the primary: {sorted(changes)}') + get_wrolpi_config().update(changes) + return sorted(changes) + + def _relative_directories(directories: List[str], primary_media_directory: str) -> List[str]: relative = list() for directory in directories: diff --git a/wrolpi/sync/schema.py b/wrolpi/sync/schema.py index 1332bc0a..813dda2e 100644 --- a/wrolpi/sync/schema.py +++ b/wrolpi/sync/schema.py @@ -47,6 +47,8 @@ class SyncSettingsRequest: backup_device_uuid: Optional[str] = None backup_window_start: Optional[str] = None backup_window_end: Optional[str] = None + # Days a secondary keeps a file its primary deleted or replaced; -1 never deletes. + sync_delete_after_days: Optional[int] = None # Folders, relative to the media directory, never copied ([] clears). sync_excluded_directories: Optional[List[str]] = None backup_excluded_directories: Optional[List[str]] = None @@ -54,3 +56,5 @@ class SyncSettingsRequest: def __post_init__(self): if self.role is not None and self.role not in ('primary', 'secondary'): raise ValidationError('role must be "primary" or "secondary"') + if self.sync_delete_after_days is not None and not -1 <= int(self.sync_delete_after_days) <= 3650: + raise ValidationError('sync_delete_after_days must be between 0 and 3650, or -1 to never delete') diff --git a/wrolpi/sync/test/test_snapshot.py b/wrolpi/sync/test/test_snapshot.py index 6f5727b4..4b818f90 100644 --- a/wrolpi/sync/test/test_snapshot.py +++ b/wrolpi/sync/test/test_snapshot.py @@ -1,14 +1,17 @@ import contextlib +import json import shutil import sqlite3 from collections import namedtuple from http import HTTPStatus import pytest +import yaml -from wrolpi.common import enable_wrol_mode +from wrolpi.common import enable_wrol_mode, get_wrolpi_config from wrolpi.db import get_db_file -from wrolpi.sync import backup, snapshot +from wrolpi.sync import backup, lib, snapshot +from wrolpi.sync.config import get_sync_config from wrolpi.sync.status import get_snapshot_status, update_snapshot_status @@ -128,3 +131,59 @@ def test_backup_drive_does_not_copy_the_snapshot(test_directory, tmp_path): """The backup drive gets its own snapshot; the one kept for secondaries is not copied to it as well.""" command = backup.build_rsync_command(test_directory, tmp_path) assert f'--exclude=/{snapshot.SNAPSHOT_DIRECTORY}/' in command + + +@pytest.mark.asyncio +async def test_delete_after_days_setting(async_client, test_session, test_directory, test_wrolpi_config, + await_switches): + """How long a secondary keeps what its primary deleted: 30 days unless set, and -1 asks to never delete.""" + request, response = await async_client.get('/api/settings') + assert response.json['sync_delete_after_days'] == 30 + + for sent, kept in ((7, 7), (0, 0), (-1, None)): + request, response = await async_client.put('/api/sync/settings', + content=json.dumps(dict(sync_delete_after_days=sent))) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json + assert get_sync_config().delete_after_days == kept + request, response = await async_client.get('/api/settings') + assert response.json['sync_delete_after_days'] == kept + + await await_switches() + saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text()) + assert 'delete_after_days' in saved and saved['delete_after_days'] is None, 'what the Controller reads' + + request, response = await async_client.put('/api/sync/settings', + content=json.dumps(dict(sync_delete_after_days=-2))) + assert response.status_code == HTTPStatus.BAD_REQUEST + + +def test_delete_after_days_of_an_older_config(test_directory, test_wrolpi_config): + """A sync.yaml written before the setting existed has no key; it gets the default, not "never".""" + get_sync_config()._config.pop('delete_after_days', None) + assert get_sync_config().delete_after_days == 30 + + +def test_apply_primary_wrolpi_config(async_client, test_directory, test_wrolpi_config, make_secondary): + """When a secondary's API starts it takes the settings that describe the library from the primary's wrolpi.yaml, + which the Controller's sync left; the settings of this machine stay.""" + sync_directory = test_directory / 'config/.sync' + sync_directory.mkdir(parents=True) + (sync_directory / 'primary-wrolpi.yaml').write_text(yaml.safe_dump(dict( + videos_destination='videos/%(channel_name)s', ignored_directories=['/media/wrolpi/map', 'config', 'elsewhere'], + hotspot_ssid='the primary', timezone='Europe/Paris'))) + (sync_directory / 'primary-info.yaml').write_text(yaml.safe_dump(dict(media_directory='/media/wrolpi'))) + timezone = get_wrolpi_config().timezone + + assert lib.apply_primary_wrolpi_config() == [], 'only on a secondary' + + make_secondary('https://primary.example') + assert lib.apply_primary_wrolpi_config() == ['ignored_directories', 'videos_destination'] + assert get_wrolpi_config().videos_destination == 'videos/%(channel_name)s' + # The primary's absolute paths are this machine's relative ones. + assert get_wrolpi_config().ignored_directories == ['map', 'config', 'elsewhere'] + assert get_wrolpi_config().timezone == timezone + + assert lib.apply_primary_wrolpi_config() == [], 'nothing changed since' + + (sync_directory / 'primary-wrolpi.yaml').write_text('not: [a config') + assert lib.apply_primary_wrolpi_config() == []