diff --git a/controller/defaults.py b/controller/defaults.py index 530b8529..ffacffa9 100644 --- a/controller/defaults.py +++ b/controller/defaults.py @@ -81,6 +81,14 @@ DEFAULT_CONFIG = { "use_https": True, "description": "Kiwix/Zim server", }, + { + "name": "wrolpi-rsyncd", + "systemd_name": "wrolpi-rsyncd", + "port": 8873, + "viewable": False, + "use_https": False, + "description": "Library for secondary WROLPis (rsync)", + }, # System services { "name": "caddy", diff --git a/controller/lib/docker_services.py b/controller/lib/docker_services.py index 82459363..67cdafbc 100644 --- a/controller/lib/docker_services.py +++ b/controller/lib/docker_services.py @@ -27,7 +27,7 @@ except ImportError: CONTAINER_PREFIX = os.environ.get("COMPOSE_PROJECT_NAME", "wrolpi") # Containers that should not have a "View" button (non-HTTP services) -NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba"} +NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba", "rsyncd"} # Containers that should use HTTPS HTTPS_CONTAINERS = {"web"} diff --git a/controller/lib/rsyncd.py b/controller/lib/rsyncd.py new file mode 100644 index 00000000..b4f6e8ae --- /dev/null +++ b/controller/lib/rsyncd.py @@ -0,0 +1,133 @@ +""" +The rsync daemon that serves this WROLPi's library to secondary WROLPis. + +On a Raspberry Pi and on Debian it is the `wrolpi-rsyncd` systemd unit. In Docker the Controller runs it in a helper +container made from the api container: the same image, user and volumes (read-only), so the media directory is +configured in one place, the api service. Both read `controller/rsyncd.conf`. +""" + +import asyncio +import hashlib +import logging +import re +from pathlib import Path + +from controller.lib.docker_services import NotFound, _get_client, _get_container_name, can_manage_containers + +logger = logging.getLogger(__name__) + +RSYNCD_CONFIG = Path(__file__).parent.parent / "rsyncd.conf" +# Named like a compose service's container, so the Control page can start, stop and restart it. +HELPER_NAME = _get_container_name("rsyncd") +# What a helper was made from; a helper made from anything else is replaced. +SOURCE_LABEL = "wrolpi.rsyncd.source" +# The api image starts the API; the helper starts rsync instead. The config arrives in the environment because the +# helper shares no directory with the Controller. +HELPER_COMMAND = ('printf "%s" "$RSYNCD_CONF" > /tmp/rsyncd.conf' + ' && exec rsync --daemon --no-detach --config=/tmp/rsyncd.conf') + + +def read_rsyncd_config() -> str: + return RSYNCD_CONFIG.read_text() + + +def get_rsyncd_port(config: str = None) -> int: + """The port in rsyncd.conf, so the published port cannot differ from the one the daemon listens on.""" + match = re.search(r"^port\s*=\s*(\d+)\s*$", config or read_rsyncd_config(), re.MULTILINE) + if not match: + raise RuntimeError(f"No port in {RSYNCD_CONFIG}") + return int(match.group(1)) + + +# The api container can be recreated or rebuilt while the Controller runs. +CHECK_INTERVAL_SECONDS = 60 +# Images found to have no rsync (built before it was added), so they are not tried again at every check. +_images_without_rsync = set() +# The last failure logged, so a failure that lasts is logged once. +_last_error = None + + +def _image_has_rsync(client, image_id: str) -> bool: + """A helper made from an image without rsync exits at once and, restarted by Docker, never stops trying.""" + if image_id in _images_without_rsync: + return False + try: + client.containers.run(image_id, ["-c", "command -v rsync"], entrypoint="sh", remove=True) + return True + except Exception as e: + if getattr(e, "exit_status", None) is None: + # Docker failed, not the command; this says nothing about the image. + raise + _images_without_rsync.add(image_id) + return False + + +def _failed(error: str) -> dict: + global _last_error + if error != _last_error: + logger.warning("Could not start the rsync daemon container: %s", error) + _last_error = error + return {"success": False, "error": error} + + +async def rsync_daemon_loop(): + """Docker only: keep the helper matching the api container for as long as the Controller runs.""" + while True: + await asyncio.to_thread(ensure_rsync_daemon) + await asyncio.sleep(CHECK_INTERVAL_SECONDS) + + +def ensure_rsync_daemon() -> dict: + """Docker only: make sure the helper container serves the api container's volumes. + + A helper made from this api container, image and config is left alone (started if it stopped); any other is + replaced. Never raises: a WROLPi without the daemon still works, it only cannot be copied by a secondary.""" + global _last_error + if not can_manage_containers(): + return {"success": False, "error": "Docker management not available"} + + try: + client = _get_client() + config = read_rsyncd_config() + port = get_rsyncd_port(config) + api = client.containers.get(_get_container_name("api")) + source = hashlib.sha256(f"{api.id}:{api.image.id}:{config}".encode()).hexdigest() + + try: + helper = client.containers.get(HELPER_NAME) + except NotFound: + helper = None + if helper is not None: + if helper.labels.get(SOURCE_LABEL) == source: + if helper.status != "running": + logger.info("Starting the rsync daemon container %s", HELPER_NAME) + helper.start() + _last_error = None + return {"success": True, "action": "kept"} + + if not _image_has_rsync(client, api.image.id): + # A helper from an older api container is left as it is; it still serves the library. + return _failed("The api image has no rsync; rebuild it (docker compose build api)") + + if helper is not None: + logger.info("Replacing the rsync daemon container %s", HELPER_NAME) + helper.remove(force=True) + + client.containers.run( + api.image.id, + ["-c", HELPER_COMMAND], + entrypoint="sh", + name=HELPER_NAME, + detach=True, + user=api.attrs["Config"].get("User") or None, + volumes_from=[f"{api.name}:ro"], + ports={f"{port}/tcp": port}, + environment={"RSYNCD_CONF": config}, + labels={SOURCE_LABEL: source}, + restart_policy={"Name": "unless-stopped"}, + ) + logger.info("Started the rsync daemon container %s on port %s", HELPER_NAME, port) + _last_error = None + return {"success": True, "action": "created"} + except Exception as e: + return _failed(str(e)) diff --git a/controller/main.py b/controller/main.py index 84de6025..c4241a78 100644 --- a/controller/main.py +++ b/controller/main.py @@ -43,6 +43,7 @@ from controller.lib.config import ( save_config, update_config, ) +from controller.lib.rsyncd import rsync_daemon_loop from controller.lib.scripts import get_script_status from controller.lib.docker_services import ( can_manage_containers, @@ -123,6 +124,10 @@ async def lifespan(app: FastAPI): else: logger.info("Primary drive not mounted - using default configuration") + # Docker: the library is served to secondary WROLPis from a helper container, kept matching the api container. + if is_docker_mode(): + app.state.rsync_daemon_task = asyncio.create_task(rsync_daemon_loop()) + # After the config (and any migrated hotspot settings) are loaded. Starting can take a while; do not block. from controller.lib.admin import start_hotspot_on_boot app.state.hotspot_on_boot_task = asyncio.create_task(asyncio.to_thread(start_hotspot_on_boot)) @@ -142,6 +147,8 @@ async def lifespan(app: FastAPI): await status_task except asyncio.CancelledError: pass + if rsync_daemon_task := getattr(app.state, "rsync_daemon_task", None): + rsync_daemon_task.cancel() # Create FastAPI app diff --git a/controller/rsyncd.conf b/controller/rsyncd.conf new file mode 100644 index 00000000..53b6052a --- /dev/null +++ b/controller/rsyncd.conf @@ -0,0 +1,22 @@ +# How a secondary WROLPi copies this WROLPi's library: an rsync daemon that serves the media directory read-only. +# The library is no more private here than it is over HTTPS at /media/. +# +# One file for every deployment: the systemd unit reads it on a Raspberry Pi and on Debian, and the Controller hands +# it to the helper container in Docker (see controller/lib/rsyncd.py). +port = 8873 +# Run as the unprivileged wrolpi user: no chroot, and no connection limit (which needs a lock file under /var/run). +use chroot = no +log file = /dev/stderr + +[media] + path = /media/wrolpi + read only = yes + list = yes + # A symbolic link is sent as a link. Asked to follow links while it copies a directory, the daemon would send + # the files left out below through a link to them from elsewhere in the library. (A link that is itself part of + # the path a client asks for is still followed: the files below are left out by where they are, nothing more.) + refuse options = copy-links copy-unsafe-links copy-dirlinks + # Never served: this machine's TLS keys and cookies, and the database in use (a copy of a database that is + # being written can be torn; a secondary copies the snapshot under config/.sync instead), and a snapshot that + # is still being written. + exclude = /config/ssl/ /config/cookies.txt /config/cookies.txt.enc /config/wrolpi.db /config/wrolpi.db-wal /config/wrolpi.db-shm /config/wrolpi.db.lock .wrolpi.db.backup /lost+found/ diff --git a/controller/test/test_defaults.py b/controller/test/test_defaults.py index 960721d3..c4ba6ab3 100644 --- a/controller/test/test_defaults.py +++ b/controller/test/test_defaults.py @@ -57,7 +57,7 @@ class TestManagedServicesConfig: def test_managed_services_count(self): """Should have expected number of managed services.""" services = DEFAULT_CONFIG["managed_services"] - assert len(services) == 11 + assert len(services) == 12 @pytest.mark.parametrize("name,expected", [ ("wrolpi-api", diff --git a/controller/test/test_rsyncd.py b/controller/test/test_rsyncd.py new file mode 100644 index 00000000..46261dbc --- /dev/null +++ b/controller/test/test_rsyncd.py @@ -0,0 +1,236 @@ +""" +Unit tests for controller.lib.rsyncd: the rsync daemon that serves the library to secondary WROLPis. +""" + +import asyncio +import shutil +import socket +import subprocess +import time +from pathlib import Path +from unittest import mock + +import pytest + +from controller.lib import rsyncd +from controller.lib.docker_services import NON_VIEWABLE_CONTAINERS + +REPOSITORY = Path(__file__).parent.parent.parent + + +class FakeNotFound(Exception): + pass + + +def make_client(helper=None): + api = mock.Mock() + api.id, api.name, api.image.id = "api-id", "wrolpi-api-1", "sha256:image" + api.attrs = {"Config": {"User": "1000:1000"}} + client = mock.Mock() + + def get(name): + if name == "wrolpi-api-1": + return api + if helper is not None: + return helper + raise FakeNotFound(name) + + client.containers.get.side_effect = get + return client + + +@pytest.fixture(autouse=True) +def forget_images(): + rsyncd._images_without_rsync.clear() + rsyncd._last_error = None + + +class FakeContainerError(Exception): + exit_status = 127 + + +def helper_runs(client) -> list: + """The calls that made a helper (the others only look for rsync in the image).""" + return [i for i in client.containers.run.call_args_list if i.kwargs.get("detach")] + + +def ensure(client): + with mock.patch.object(rsyncd, "can_manage_containers", return_value=True), \ + mock.patch.object(rsyncd, "_get_client", return_value=client), \ + mock.patch.object(rsyncd, "NotFound", FakeNotFound): + return rsyncd.ensure_rsync_daemon() + + +def current_source(client) -> str: + """The label a helper made by this code from this api container carries.""" + ensure(client) + return helper_runs(client)[0].kwargs["labels"][rsyncd.SOURCE_LABEL] + + +class TestConfig: + + def test_port(self): + assert rsyncd.get_rsyncd_port() == 8873 + assert rsyncd.get_rsyncd_port("port = 9000\n[media]\n") == 9000 + with pytest.raises(RuntimeError): + rsyncd.get_rsyncd_port("[media]\n") + + def test_unit_reads_the_same_config(self): + """A Raspberry Pi, Debian and Docker all serve by one config file.""" + unit = (REPOSITORY / "etc/raspberrypios/wrolpi-rsyncd.service").read_text() + assert "--config=/opt/wrolpi/controller/rsyncd.conf" in unit + assert rsyncd.RSYNCD_CONFIG == REPOSITORY / "controller/rsyncd.conf" + assert "User=wrolpi" in unit + + def test_units_installed_and_enabled(self): + for script in ("repair.sh", "pi-gen/stage2/04-wrolpi/03-run-chroot.sh", + "debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh"): + assert "wrolpi-rsyncd.service" in (REPOSITORY / script).read_text(), script + assert "rsync" in (REPOSITORY / "docker/api/Dockerfile").read_text() + + +@pytest.mark.skipif(not shutil.which("rsync") or b"rsync version 3" not in subprocess.run( + ["rsync", "--version"], capture_output=True).stdout, reason="needs rsync 3") +def test_daemon_serves_the_library_read_only(tmp_path): + """The daemon run with the real config (only its port and path changed) lists the library, never serves the + database in use or the TLS keys, serves the snapshot, and refuses a write.""" + media = tmp_path / "media" + for path in ("videos/a.mp4", "config/tags.yaml", "config/wrolpi.db", "config/wrolpi.db-wal", + "config/ssl/ca.key", "config/cookies.txt.enc", "config/.sync/config/wrolpi.db", + "config/.sync/config/.wrolpi.db.backup"): + (media / path).parent.mkdir(parents=True, exist_ok=True) + (media / path).write_text(path) + # Links that anyone able to write to the library could make, to what the daemon leaves out. Relative: rsync + # already makes an absolute link point nowhere. + (media / "videos/keys").symlink_to("../config/ssl") + (media / "videos/key").symlink_to("../config/ssl/ca.key") + with socket.socket() as probe: + probe.bind(("127.0.0.1", 0)) + port = probe.getsockname()[1] + config = rsyncd.read_rsyncd_config().replace("port = 8873", f"port = {port}\naddress = 127.0.0.1") \ + .replace("path = /media/wrolpi", f"path = {media}").replace("log file = /dev/stderr", "") + assert str(media) in config and str(port) in config + config_file = tmp_path / "rsyncd.conf" + config_file.write_text(config) + + daemon = subprocess.Popen(["rsync", "--daemon", "--no-detach", f"--config={config_file}"]) + try: + url = f"rsync://127.0.0.1:{port}/media/" + for _ in range(50): + listing = subprocess.run(["rsync", "-r", "--list-only", url], capture_output=True, text=True) + if listing.returncode == 0: + break + time.sleep(0.1) + assert listing.returncode == 0, listing.stderr + listed = {line.split()[-1] for line in listing.stdout.splitlines() if line.startswith("-")} + assert listed == {"videos/a.mp4", "config/tags.yaml", "config/.sync/config/wrolpi.db"} + + # Followed links are refused; without the option the links arrive as links, never as what they point at. + for option in ("--copy-links", "--copy-unsafe-links", "--copy-dirlinks"): + followed = subprocess.run(["rsync", "-r", option, url, str(tmp_path / "followed")], + capture_output=True, text=True) + assert followed.returncode != 0, option + assert "refuse" in followed.stderr, followed.stderr + leaked = [i for i in (tmp_path / "followed").rglob("*") if i.is_file() and "ssl/ca.key" in i.read_text()] + assert not leaked, "a link let a client read the TLS key" + copied = tmp_path / "copied" + assert subprocess.run(["rsync", "-a", url, f"{copied}/"], capture_output=True).returncode == 0 + assert (copied / "videos/key").is_symlink() and (copied / "videos/keys").is_symlink() + assert not (copied / "config/ssl").exists() + + (tmp_path / "new.txt").write_text("new") + write = subprocess.run(["rsync", str(tmp_path / "new.txt"), url], capture_output=True, text=True) + assert write.returncode != 0 and "read only" in write.stderr + assert not (media / "new.txt").exists() + finally: + daemon.terminate() + daemon.wait(timeout=10) + + +class TestEnsureRsyncDaemon: + + def test_not_in_docker(self): + with mock.patch.object(rsyncd, "can_manage_containers", return_value=False): + assert rsyncd.ensure_rsync_daemon()["success"] is False + + def test_creates_the_helper_from_the_api_container(self): + client = make_client() + assert ensure(client) == {"success": True, "action": "created"} + (call,) = helper_runs(client) + args, kwargs = call.args, call.kwargs + assert args[0] == "sha256:image", "the api container's own image" + assert kwargs["entrypoint"] == "sh" and "rsync --daemon" in args[1][1], "not the image's API entrypoint" + assert kwargs["volumes_from"] == ["wrolpi-api-1:ro"], "the api container's volumes, read-only" + assert kwargs["user"] == "1000:1000" + assert kwargs["ports"] == {"8873/tcp": 8873} + assert kwargs["environment"]["RSYNCD_CONF"] == rsyncd.read_rsyncd_config() + assert kwargs["name"] == "wrolpi-rsyncd-1" and kwargs["detach"] is True + assert kwargs["restart_policy"] == {"Name": "unless-stopped"} + + def test_keeps_its_own_helper(self): + helper = mock.Mock(status="running") + helper.labels = {rsyncd.SOURCE_LABEL: current_source(make_client())} + client = make_client(helper) + assert ensure(client) == {"success": True, "action": "kept"} + client.containers.run.assert_not_called(), "not even to look for rsync" + helper.start.assert_not_called() + helper.remove.assert_not_called() + + def test_starts_a_stopped_helper(self): + helper = mock.Mock(status="exited") + helper.labels = {rsyncd.SOURCE_LABEL: current_source(make_client())} + assert ensure(make_client(helper))["action"] == "kept" + helper.start.assert_called_once() + + def test_replaces_a_helper_made_from_another_api_container(self): + """A recreated api container may have other volumes; the helper must serve what the API has now.""" + helper = mock.Mock(status="running") + helper.labels = {rsyncd.SOURCE_LABEL: "made from an older api container"} + client = make_client(helper) + assert ensure(client)["action"] == "created" + helper.remove.assert_called_once_with(force=True) + assert len(helper_runs(client)) == 1 + + def test_no_helper_from_an_image_without_rsync(self): + """An api image built before rsync was added: a helper made from it would exit and be restarted forever. + None is made, an older helper is left serving, and the image is not tried again at every check.""" + helper = mock.Mock(status="running") + helper.labels = {rsyncd.SOURCE_LABEL: "made from an older api container"} + client = make_client(helper) + client.containers.run.side_effect = FakeContainerError("sh: 1: rsync: not found") + + for _ in range(3): + result = ensure(client) + assert result["success"] is False and "no rsync" in result["error"] + assert not helper_runs(client) + helper.remove.assert_not_called() + assert client.containers.run.call_count == 1, "the image is looked at once" + + def test_docker_failing_says_nothing_about_the_image(self): + client = make_client() + client.containers.run.side_effect = RuntimeError("docker is restarting") + assert ensure(client)["success"] is False + assert not rsyncd._images_without_rsync + + client.containers.run.side_effect = None + assert ensure(client)["action"] == "created" + + def test_never_raises(self): + client = make_client() + client.containers.run.side_effect = RuntimeError("no such image") + result = ensure(client) + assert result["success"] is False and "no such image" in result["error"] + + @pytest.mark.asyncio + async def test_checked_for_as_long_as_the_controller_runs(self, monkeypatch): + """The api container can be recreated while the Controller runs; the helper is checked again and again.""" + calls = [] + monkeypatch.setattr(rsyncd, "ensure_rsync_daemon", lambda: calls.append(1)) + monkeypatch.setattr(rsyncd, "CHECK_INTERVAL_SECONDS", 0.01) + task = asyncio.create_task(rsyncd.rsync_daemon_loop()) + await asyncio.sleep(0.2) + task.cancel() + assert len(calls) >= 3 + + def test_not_given_a_view_button(self): + assert "rsyncd" in NON_VIEWABLE_CONTAINERS diff --git a/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh b/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh index 503990a7..bb77fdac 100644 --- a/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh +++ b/debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh @@ -280,8 +280,8 @@ fi # with --no-block so systemd queues the starts after we exit. progress "Starting WROLPi services..." -systemctl enable wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service -systemctl start --no-block wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service +systemctl enable wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service wrolpi-rsyncd.service +systemctl start --no-block wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service wrolpi-rsyncd.service # If we did anything notable, leave a summary for the XFCE autostart to # show as a zenity dialog once the user lands on the desktop. diff --git a/docker/api/Dockerfile b/docker/api/Dockerfile index eca1d277..ac0f26dc 100644 --- a/docker/api/Dockerfile +++ b/docker/api/Dockerfile @@ -5,7 +5,7 @@ ENV DENO_TLS_CA_STORE=system WORKDIR /opt/wrolpi # Update dependencies for the services install. -RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip \ +RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip rsync \ build-essential libsqlite3-dev zlib1g-dev git # Install tippecanoe (for map search index building via tippecanoe-decode). diff --git a/etc/raspberrypios/wrolpi-rsyncd.service b/etc/raspberrypios/wrolpi-rsyncd.service new file mode 100644 index 00000000..a5191ed4 --- /dev/null +++ b/etc/raspberrypios/wrolpi-rsyncd.service @@ -0,0 +1,15 @@ +[Unit] +Description=WROLPi library for secondary WROLPis (rsync daemon) +PartOf=wrolpi.target +After=network.target + +[Service] +ExecStart=/usr/bin/rsync --daemon --no-detach --config=/opt/wrolpi/controller/rsyncd.conf +Group=wrolpi +User=wrolpi +Restart=on-failure +# Below everything a person is waiting on. +Nice=10 + +[Install] +WantedBy=default.target diff --git a/etc/raspberrypios/wrolpi.target b/etc/raspberrypios/wrolpi.target index 8551f153..a330cbd1 100644 --- a/etc/raspberrypios/wrolpi.target +++ b/etc/raspberrypios/wrolpi.target @@ -1,3 +1,3 @@ [Unit] Description=WROLPi services group -Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service wrolpi-kiwix.service +Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service wrolpi-kiwix.service wrolpi-rsyncd.service diff --git a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh index a79c1fb5..d1db81b4 100755 --- a/pi-gen/stage2/04-wrolpi/03-run-chroot.sh +++ b/pi-gen/stage2/04-wrolpi/03-run-chroot.sh @@ -106,6 +106,8 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/ systemctl enable wrolpi-first-startup.service # Enable Controller service to start on boot. systemctl enable wrolpi-controller.service +# Serve the library to secondary WROLPis. +systemctl enable wrolpi-rsyncd.service # Enable weekly certificate renewal check. systemctl enable wrolpi-cert-renew.timer diff --git a/repair.sh b/repair.sh index 6fc20375..ac77aa01 100755 --- a/repair.sh +++ b/repair.sh @@ -94,6 +94,7 @@ systemctl enable wrolpi-app.service systemctl enable wrolpi-kiwix.service systemctl enable wrolpi-help.service systemctl enable wrolpi-mounts.service +systemctl enable wrolpi-rsyncd.service systemctl enable wrolpi-cert-renew.timer # Repair Controller (offline-safe - no pip install) @@ -210,6 +211,8 @@ if [[ ${rpi} == true ]]; then fi systemctl restart wrolpi-help +# Restarted, not only started: its config may have changed. +systemctl restart wrolpi-rsyncd || : systemctl start wrolpi.target set +x diff --git a/wrolpi/sync/api.py b/wrolpi/sync/api.py index 9e2fc371..0d8fc858 100644 --- a/wrolpi/sync/api.py +++ b/wrolpi/sync/api.py @@ -12,9 +12,9 @@ from wrolpi.api_utils import api_app, json_response, perpetual_signal from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled from wrolpi.db import get_db_session from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled -from wrolpi.sync import backup, lib, schema +from wrolpi.sync import backup, lib, schema, snapshot from wrolpi.sync.config import get_sync_config -from wrolpi.sync.status import get_backup_status, get_sync_status +from wrolpi.sync.status import get_backup_status, get_snapshot_status, get_sync_status from wrolpi.vars import DOCKERIZED sync_bp = Blueprint('Sync', '/api/sync') @@ -51,6 +51,7 @@ async def perpetual_sync_worker(): # One loop, so a secondary's sync and its backup never run at once; the backup copies what the sync pulled. await lib.sync_tick() await backup.backup_tick() + await snapshot.snapshot_tick() @sync_bp.get('/info') @@ -78,7 +79,18 @@ async def get_manifest(_: Request, query: schema.ManifestRequest): @openapi.definition(summary="The progress of this secondary's sync") async def get_status(_: Request): return json_response(dict(sync=get_sync_status(), sync_running=bool(flags.sync_running.is_set()), - backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()))) + backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()), + snapshot=get_snapshot_status())) + + +@sync_bp.post('/snapshot') +@openapi.definition(summary="Make a snapshot of this WROLPi's database for a secondary to copy") +async def post_snapshot(_: Request): + if reason := snapshot.snapshot_skip_reason(): + raise InvalidConfig(reason) + # Made by the perpetual process; `/api/sync/status` says when it is done. + snapshot.request_snapshot() + return response.empty(HTTPStatus.ACCEPTED) @sync_bp.post('/run') diff --git a/wrolpi/sync/backup.py b/wrolpi/sync/backup.py index 1b6a66aa..0ebb1d76 100644 --- a/wrolpi/sync/backup.py +++ b/wrolpi/sync/backup.py @@ -171,6 +171,8 @@ def build_rsync_command(source: pathlib.Path, target: pathlib.Path, playlists_de '--info=progress2,stats2', '--no-human-readable', # The database is copied as a snapshot (see `snapshot_database`). '--exclude=/config/wrolpi.db', '--exclude=/config/wrolpi.db-*', '--exclude=/config/.wrolpi.db.backup', + # The snapshot kept for secondaries (wrolpi.sync.snapshot); the drive has its own. + '--exclude=/config/.sync/', # Hard links rebuilt from the configs when WROLPi starts. '--exclude=/tags/', # Files still being written by a sync, and rsync's own partials. diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py index 814380f4..7388f23e 100644 --- a/wrolpi/sync/lib.py +++ b/wrolpi/sync/lib.py @@ -35,7 +35,8 @@ from urllib.parse import quote import aiohttp import yaml -from sqlalchemy import func, select +from sqlalchemy import func, select, text +from sqlalchemy.exc import OperationalError from sqlalchemy.orm import Session from wrolpi import flags @@ -191,9 +192,20 @@ def get_sync_info(session: Session) -> dict: role=get_sync_config().role, media_directory=str(get_media_directory()), max_file_group_id=session.execute(select(func.max(FileGroup.id))).scalar() or 0, + # A secondary that replaces its database with this one must run code of the same migration. + database_revision=_database_revision(session), + # An unmounted media directory looks like an empty library. + media_mounted=bool(flags.media_mounted.is_set()), ) +def _database_revision(session: Session) -> Optional[str]: + try: + return session.execute(text('SELECT version_num FROM alembic_version')).scalar() + except OperationalError: + return None + + # --------------------------------------------------------------------------------------------------------------------- # Secondary: comparing and writing files. diff --git a/wrolpi/sync/snapshot.py b/wrolpi/sync/snapshot.py new file mode 100644 index 00000000..eca561d1 --- /dev/null +++ b/wrolpi/sync/snapshot.py @@ -0,0 +1,87 @@ +"""A primary WROLPi's database snapshot, for a secondary to copy. + +A secondary copies its primary's files with rsync (the daemon of `controller/rsyncd.conf`), and replaces its own +database with the primary's. A database that is in use cannot be copied as a file, so the secondary first asks the +primary for a snapshot, which the daemon then serves like any other file. The snapshot is kept between requests: an +unchanged database is not written again, and rsync has the previous copy to compare with. +""" +import asyncio +import pathlib +import shutil +from typing import Optional + +from wrolpi import flags +from wrolpi.common import logger, get_media_directory, is_secondary, wrol_mode_enabled +from wrolpi.dates import now +from wrolpi.db import get_db_file +from wrolpi.sync.backup import snapshot_database_if_changed +from wrolpi.sync.status import get_snapshot_status, update_snapshot_status +from wrolpi.vars import PYTEST + +logger = logger.getChild(__name__) + +# Hidden, so nothing indexes it; inside config, which nothing but the daemon serves. +SNAPSHOT_DIRECTORY = 'config/.sync' +# `snapshot_database` writes a media directory's `config/wrolpi.db`. +SNAPSHOT_PATH = f'{SNAPSHOT_DIRECTORY}/config/wrolpi.db' +# Never fill the drive with a copy of the database; leave this much free. +MIN_FREE_BYTES = 1024 ** 3 + + +def get_snapshot_directory() -> pathlib.Path: + return get_media_directory() / SNAPSHOT_DIRECTORY + + +def snapshot_skip_reason() -> Optional[str]: + """Why a snapshot cannot be made now, or None.""" + if is_secondary(): + return 'A secondary WROLPi is not copied; its primary is' + if wrol_mode_enabled(): + # Like the backup drive: nothing new is written in WROL Mode. A snapshot made earlier is still served. + return 'WROL Mode is enabled' + if not PYTEST and not flags.db_up.is_set(): + return 'The database is not up' + try: + size = get_db_file().stat().st_size + except FileNotFoundError: + return 'There is no database' + # The old snapshot is replaced only after the new one is complete, so both exist for a while. + if shutil.disk_usage(get_media_directory()).free - size < MIN_FREE_BYTES: + return 'Not enough free space for a copy of the database' + return None + + +def request_snapshot(): + update_snapshot_status(requested=True) + + +async def run_snapshot() -> dict: + """Make the snapshot (or keep the one there is, if the database has not changed). Returns the final status.""" + if reason := snapshot_skip_reason(): + logger.info(f'Not making a database snapshot: {reason}') + return update_snapshot_status(state='error', finished_at=now().isoformat(), last_error=reason) + + update_snapshot_status(state='running', started_at=now().isoformat(), finished_at=None, last_error=None) + try: + written = await asyncio.to_thread(snapshot_database_if_changed, get_snapshot_directory()) + size = (get_media_directory() / SNAPSHOT_PATH).stat().st_size + except Exception as e: + logger.error('Failed to make a database snapshot', exc_info=e) + return update_snapshot_status(state='error', finished_at=now().isoformat(), + last_error=str(e) or type(e).__name__) + return update_snapshot_status(state='idle', finished_at=now().isoformat(), path=SNAPSHOT_PATH, size=size, + written=written) + + +async def snapshot_tick(): + """Make a snapshot when one was requested.""" + status = get_snapshot_status() + if status['state'] == 'running': + # Only this loop makes snapshots, one at a time: one still marked running when a tick starts was cut off + # (its process was killed), and would otherwise hold back every later request. + status = update_snapshot_status(state='error', finished_at=now().isoformat(), + last_error='The snapshot was interrupted') + if not status['requested']: + return + update_snapshot_status(requested=False) + await run_snapshot() diff --git a/wrolpi/sync/status.py b/wrolpi/sync/status.py index 845b8e62..52f28f5c 100644 --- a/wrolpi/sync/status.py +++ b/wrolpi/sync/status.py @@ -48,6 +48,39 @@ def update_sync_status(**changes) -> dict: return status +def default_snapshot_status() -> dict: + return dict( + state='idle', + requested=False, + started_at=None, + finished_at=None, + last_error=None, + # Relative to the media directory; where a secondary finds the snapshot. + path=None, + size=0, + # False when the last request found the database unchanged and kept the snapshot it had. + written=False, + ) + + +# Kept out of `shared_ctx.status`, which every `/api/status` poll returns: only a secondary's Controller asks for it. +SNAPSHOT_STATUS_KEY = 'sync_snapshot_status' + + +def get_snapshot_status() -> dict: + from wrolpi.api_utils import api_app + return dict(api_app.shared_ctx.cache.get(SNAPSHOT_STATUS_KEY) or default_snapshot_status()) + + +def update_snapshot_status(**changes) -> dict: + """Merge `changes` into the shared snapshot status (see `update_sync_status`).""" + from wrolpi.api_utils import api_app + status = get_snapshot_status() + status.update(changes) + api_app.shared_ctx.cache[SNAPSHOT_STATUS_KEY] = status + return status + + def default_backup_status() -> dict: return dict( state='idle', diff --git a/wrolpi/sync/test/test_snapshot.py b/wrolpi/sync/test/test_snapshot.py new file mode 100644 index 00000000..6f5727b4 --- /dev/null +++ b/wrolpi/sync/test/test_snapshot.py @@ -0,0 +1,130 @@ +import contextlib +import shutil +import sqlite3 +from collections import namedtuple +from http import HTTPStatus + +import pytest + +from wrolpi.common import enable_wrol_mode +from wrolpi.db import get_db_file +from wrolpi.sync import backup, snapshot +from wrolpi.sync.status import get_snapshot_status, update_snapshot_status + + +@pytest.mark.asyncio +async def test_info_says_what_a_secondary_must_check(async_client, test_session, test_wrolpi_config): + """Before a secondary replaces its database with its primary's, it must know the primary's migration and that the + primary's media directory is really there.""" + request, response = await async_client.get('/api/sync/info') + assert response.status_code == HTTPStatus.OK + assert 'database_revision' in response.json + assert response.json['media_mounted'] in (True, False) + + +@pytest.mark.asyncio +async def test_snapshot(async_client, test_session, test_directory, test_wrolpi_config): + """A requested snapshot is a complete database where the rsync daemon serves it, and an unchanged database is + not written again.""" + with contextlib.closing(sqlite3.connect(get_db_file())) as live: + live.execute('CREATE TABLE snapshot_test (name TEXT)') + live.execute("INSERT INTO snapshot_test VALUES ('one')") + live.commit() + + request, response = await async_client.post('/api/sync/snapshot') + assert response.status_code == HTTPStatus.ACCEPTED + assert get_snapshot_status()['requested'] is True + assert not (test_directory / snapshot.SNAPSHOT_PATH).exists(), 'made by the perpetual process, not the request' + + await snapshot.snapshot_tick() + + status = get_snapshot_status() + assert status['state'] == 'idle' and status['last_error'] is None and status['requested'] is False + assert status['path'] == 'config/.sync/config/wrolpi.db' and status['written'] is True + copy = test_directory / status['path'] + assert status['size'] == copy.stat().st_size > 0 + with contextlib.closing(sqlite3.connect(f'file:{copy}?mode=ro', uri=True)) as connection: + assert connection.execute('SELECT name FROM snapshot_test').fetchall() == [('one',)] + + request, response = await async_client.get('/api/sync/status') + assert response.json['snapshot']['path'] == status['path'] + + # Nothing asked for: nothing made. + modified = copy.stat().st_mtime_ns + await snapshot.snapshot_tick() + assert copy.stat().st_mtime_ns == modified + + # Asked again with the database unchanged: the snapshot there is kept. + snapshot.request_snapshot() + await snapshot.snapshot_tick() + assert get_snapshot_status()['written'] is False and get_snapshot_status()['state'] == 'idle' + assert copy.stat().st_mtime_ns == modified + + # The database changed: written again. + with contextlib.closing(sqlite3.connect(get_db_file())) as live: + live.execute("INSERT INTO snapshot_test VALUES ('two')") + live.commit() + snapshot.request_snapshot() + await snapshot.snapshot_tick() + assert get_snapshot_status()['written'] is True + with contextlib.closing(sqlite3.connect(f'file:{copy}?mode=ro', uri=True)) as connection: + assert connection.execute('SELECT count(*) FROM snapshot_test').fetchone() == (2,) + + +@pytest.mark.asyncio +async def test_snapshot_refused(async_client, test_session, test_directory, test_wrolpi_config, make_secondary, + monkeypatch): + """No snapshot is made when it would fill the drive, or on a secondary (which nothing copies).""" + usage = namedtuple('usage', 'total used free') + monkeypatch.setattr(snapshot.shutil, 'disk_usage', lambda _: usage(10, 10, snapshot.MIN_FREE_BYTES)) + request, response = await async_client.post('/api/sync/snapshot') + assert response.status_code == HTTPStatus.BAD_REQUEST + assert 'free space' in response.json['error'] + # The space ran out between the request and the snapshot. + snapshot.request_snapshot() + await snapshot.snapshot_tick() + assert get_snapshot_status()['state'] == 'error' and 'free space' in get_snapshot_status()['last_error'] + assert not (test_directory / snapshot.SNAPSHOT_PATH).exists() + monkeypatch.setattr(snapshot.shutil, 'disk_usage', shutil.disk_usage) + + make_secondary('https://primary.example') + request, response = await async_client.post('/api/sync/snapshot') + assert response.status_code == HTTPStatus.BAD_REQUEST + assert not get_snapshot_status()['requested'] + + +@pytest.mark.asyncio +async def test_snapshot_refused_in_wrol_mode(async_client, test_session, test_directory, test_wrolpi_config): + """WROL Mode writes nothing new: no snapshot is made, whether it is asked for now or was asked for before.""" + snapshot.request_snapshot() + enable_wrol_mode() + + request, response = await async_client.post('/api/sync/snapshot') + assert response.status_code == HTTPStatus.BAD_REQUEST and 'WROL Mode' in response.json['error'] + + await snapshot.snapshot_tick() + assert get_snapshot_status()['state'] == 'error' and 'WROL Mode' in get_snapshot_status()['last_error'] + assert not (test_directory / snapshot.SNAPSHOT_PATH).exists() + + +@pytest.mark.asyncio +async def test_snapshot_interrupted(async_client, test_session, test_directory, test_wrolpi_config): + """A snapshot cut off by its process being killed stays marked as running. It must not hold back the next + request, nor be reported as running forever.""" + update_snapshot_status(state='running', started_at='2026-01-01T00:00:00+00:00') + + await snapshot.snapshot_tick() + assert get_snapshot_status()['state'] == 'error' + assert get_snapshot_status()['last_error'] == 'The snapshot was interrupted' + + update_snapshot_status(state='running') + snapshot.request_snapshot() + await snapshot.snapshot_tick() + assert get_snapshot_status()['state'] == 'idle' and get_snapshot_status()['written'] is True + assert (test_directory / snapshot.SNAPSHOT_PATH).is_file() + + +def test_backup_drive_does_not_copy_the_snapshot(test_directory, tmp_path): + """The backup drive gets its own snapshot; the one kept for secondaries is not copied to it as well.""" + command = backup.build_rsync_command(test_directory, tmp_path) + assert f'--exclude=/{snapshot.SNAPSHOT_DIRECTORY}/' in command