mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
A WROLPi serves its library over rsync, and makes a database snapshot on request (#988)
* A WROLPi serves its library over rsync, and makes a database snapshot on request The first part of replacing the sync: a secondary will copy its primary's whole media directory with rsync and replace its database with the primary's, instead of downloading the files of FileGroups over HTTP (which never copies an ignored directory, such as the map's). - An rsync daemon serves the media directory read-only on port 8873, on every WROLPi. It never serves the TLS keys, the cookies, or the database in use. On a Raspberry Pi and Debian it is the wrolpi-rsyncd unit; in Docker the Controller runs it in a helper container made from the api container (its image, user and volumes), so the media directory is still configured in one place. All three read controller/rsyncd.conf. - POST /api/sync/snapshot asks for a snapshot of the database, which the daemon serves from config/.sync. An unchanged database is not written again. - /api/sync/info also says the database's migration and whether the media directory is mounted. Nothing uses these yet; the existing sync is unchanged. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> * The rsync daemon follows no links, and no snapshot is made in WROL Mode - A client that asked the daemon to follow symbolic links could read what a link in the library points at. A relative link to config/ssl was read this way although that directory is left out of the module. The daemon now refuses the three options that follow links; a link is sent as a link. - A database snapshot is a new file on the primary's drive, so WROL Mode refuses it, as it does a backup. A snapshot made earlier is still served. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> * No rsync helper from an image without rsync, and the helper follows the api container Review found: - A helper made from an api image built before rsync was added exits at once, and Docker restarted it forever while the Controller reported it created. The image is now looked at first; no helper is made from one without rsync, an older helper is left serving, and the reason is logged once. - The helper was only checked when the Controller started, so a recreated or rebuilt api container left it stale. It is now checked every minute. - A snapshot cut off by its process being killed stayed marked as running and held back every later request. - The snapshot's half-written temporary file was served by the daemon. - The comment on the daemon's refused options claimed more than they give: a link that is part of the requested path is still followed. Co-Authored-By: Claude Fable 5.2 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.2 <noreply@anthropic.com>
This commit is contained in:
parent
bb7a66b0d6
commit
2ac4a3a294
19 changed files with 712 additions and 10 deletions
|
|
@ -81,6 +81,14 @@ DEFAULT_CONFIG = {
|
|||
"use_https": True,
|
||||
"description": "Kiwix/Zim server",
|
||||
},
|
||||
{
|
||||
"name": "wrolpi-rsyncd",
|
||||
"systemd_name": "wrolpi-rsyncd",
|
||||
"port": 8873,
|
||||
"viewable": False,
|
||||
"use_https": False,
|
||||
"description": "Library for secondary WROLPis (rsync)",
|
||||
},
|
||||
# System services
|
||||
{
|
||||
"name": "caddy",
|
||||
|
|
|
|||
|
|
@ -27,7 +27,7 @@ except ImportError:
|
|||
CONTAINER_PREFIX = os.environ.get("COMPOSE_PROJECT_NAME", "wrolpi")
|
||||
|
||||
# Containers that should not have a "View" button (non-HTTP services)
|
||||
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba"}
|
||||
NON_VIEWABLE_CONTAINERS = {"redis", "memcached", "samba", "rsyncd"}
|
||||
|
||||
# Containers that should use HTTPS
|
||||
HTTPS_CONTAINERS = {"web"}
|
||||
|
|
|
|||
133
controller/lib/rsyncd.py
Normal file
133
controller/lib/rsyncd.py
Normal file
|
|
@ -0,0 +1,133 @@
|
|||
"""
|
||||
The rsync daemon that serves this WROLPi's library to secondary WROLPis.
|
||||
|
||||
On a Raspberry Pi and on Debian it is the `wrolpi-rsyncd` systemd unit. In Docker the Controller runs it in a helper
|
||||
container made from the api container: the same image, user and volumes (read-only), so the media directory is
|
||||
configured in one place, the api service. Both read `controller/rsyncd.conf`.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import hashlib
|
||||
import logging
|
||||
import re
|
||||
from pathlib import Path
|
||||
|
||||
from controller.lib.docker_services import NotFound, _get_client, _get_container_name, can_manage_containers
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
RSYNCD_CONFIG = Path(__file__).parent.parent / "rsyncd.conf"
|
||||
# Named like a compose service's container, so the Control page can start, stop and restart it.
|
||||
HELPER_NAME = _get_container_name("rsyncd")
|
||||
# What a helper was made from; a helper made from anything else is replaced.
|
||||
SOURCE_LABEL = "wrolpi.rsyncd.source"
|
||||
# The api image starts the API; the helper starts rsync instead. The config arrives in the environment because the
|
||||
# helper shares no directory with the Controller.
|
||||
HELPER_COMMAND = ('printf "%s" "$RSYNCD_CONF" > /tmp/rsyncd.conf'
|
||||
' && exec rsync --daemon --no-detach --config=/tmp/rsyncd.conf')
|
||||
|
||||
|
||||
def read_rsyncd_config() -> str:
|
||||
return RSYNCD_CONFIG.read_text()
|
||||
|
||||
|
||||
def get_rsyncd_port(config: str = None) -> int:
|
||||
"""The port in rsyncd.conf, so the published port cannot differ from the one the daemon listens on."""
|
||||
match = re.search(r"^port\s*=\s*(\d+)\s*$", config or read_rsyncd_config(), re.MULTILINE)
|
||||
if not match:
|
||||
raise RuntimeError(f"No port in {RSYNCD_CONFIG}")
|
||||
return int(match.group(1))
|
||||
|
||||
|
||||
# The api container can be recreated or rebuilt while the Controller runs.
|
||||
CHECK_INTERVAL_SECONDS = 60
|
||||
# Images found to have no rsync (built before it was added), so they are not tried again at every check.
|
||||
_images_without_rsync = set()
|
||||
# The last failure logged, so a failure that lasts is logged once.
|
||||
_last_error = None
|
||||
|
||||
|
||||
def _image_has_rsync(client, image_id: str) -> bool:
|
||||
"""A helper made from an image without rsync exits at once and, restarted by Docker, never stops trying."""
|
||||
if image_id in _images_without_rsync:
|
||||
return False
|
||||
try:
|
||||
client.containers.run(image_id, ["-c", "command -v rsync"], entrypoint="sh", remove=True)
|
||||
return True
|
||||
except Exception as e:
|
||||
if getattr(e, "exit_status", None) is None:
|
||||
# Docker failed, not the command; this says nothing about the image.
|
||||
raise
|
||||
_images_without_rsync.add(image_id)
|
||||
return False
|
||||
|
||||
|
||||
def _failed(error: str) -> dict:
|
||||
global _last_error
|
||||
if error != _last_error:
|
||||
logger.warning("Could not start the rsync daemon container: %s", error)
|
||||
_last_error = error
|
||||
return {"success": False, "error": error}
|
||||
|
||||
|
||||
async def rsync_daemon_loop():
|
||||
"""Docker only: keep the helper matching the api container for as long as the Controller runs."""
|
||||
while True:
|
||||
await asyncio.to_thread(ensure_rsync_daemon)
|
||||
await asyncio.sleep(CHECK_INTERVAL_SECONDS)
|
||||
|
||||
|
||||
def ensure_rsync_daemon() -> dict:
|
||||
"""Docker only: make sure the helper container serves the api container's volumes.
|
||||
|
||||
A helper made from this api container, image and config is left alone (started if it stopped); any other is
|
||||
replaced. Never raises: a WROLPi without the daemon still works, it only cannot be copied by a secondary."""
|
||||
global _last_error
|
||||
if not can_manage_containers():
|
||||
return {"success": False, "error": "Docker management not available"}
|
||||
|
||||
try:
|
||||
client = _get_client()
|
||||
config = read_rsyncd_config()
|
||||
port = get_rsyncd_port(config)
|
||||
api = client.containers.get(_get_container_name("api"))
|
||||
source = hashlib.sha256(f"{api.id}:{api.image.id}:{config}".encode()).hexdigest()
|
||||
|
||||
try:
|
||||
helper = client.containers.get(HELPER_NAME)
|
||||
except NotFound:
|
||||
helper = None
|
||||
if helper is not None:
|
||||
if helper.labels.get(SOURCE_LABEL) == source:
|
||||
if helper.status != "running":
|
||||
logger.info("Starting the rsync daemon container %s", HELPER_NAME)
|
||||
helper.start()
|
||||
_last_error = None
|
||||
return {"success": True, "action": "kept"}
|
||||
|
||||
if not _image_has_rsync(client, api.image.id):
|
||||
# A helper from an older api container is left as it is; it still serves the library.
|
||||
return _failed("The api image has no rsync; rebuild it (docker compose build api)")
|
||||
|
||||
if helper is not None:
|
||||
logger.info("Replacing the rsync daemon container %s", HELPER_NAME)
|
||||
helper.remove(force=True)
|
||||
|
||||
client.containers.run(
|
||||
api.image.id,
|
||||
["-c", HELPER_COMMAND],
|
||||
entrypoint="sh",
|
||||
name=HELPER_NAME,
|
||||
detach=True,
|
||||
user=api.attrs["Config"].get("User") or None,
|
||||
volumes_from=[f"{api.name}:ro"],
|
||||
ports={f"{port}/tcp": port},
|
||||
environment={"RSYNCD_CONF": config},
|
||||
labels={SOURCE_LABEL: source},
|
||||
restart_policy={"Name": "unless-stopped"},
|
||||
)
|
||||
logger.info("Started the rsync daemon container %s on port %s", HELPER_NAME, port)
|
||||
_last_error = None
|
||||
return {"success": True, "action": "created"}
|
||||
except Exception as e:
|
||||
return _failed(str(e))
|
||||
|
|
@ -43,6 +43,7 @@ from controller.lib.config import (
|
|||
save_config,
|
||||
update_config,
|
||||
)
|
||||
from controller.lib.rsyncd import rsync_daemon_loop
|
||||
from controller.lib.scripts import get_script_status
|
||||
from controller.lib.docker_services import (
|
||||
can_manage_containers,
|
||||
|
|
@ -123,6 +124,10 @@ async def lifespan(app: FastAPI):
|
|||
else:
|
||||
logger.info("Primary drive not mounted - using default configuration")
|
||||
|
||||
# Docker: the library is served to secondary WROLPis from a helper container, kept matching the api container.
|
||||
if is_docker_mode():
|
||||
app.state.rsync_daemon_task = asyncio.create_task(rsync_daemon_loop())
|
||||
|
||||
# After the config (and any migrated hotspot settings) are loaded. Starting can take a while; do not block.
|
||||
from controller.lib.admin import start_hotspot_on_boot
|
||||
app.state.hotspot_on_boot_task = asyncio.create_task(asyncio.to_thread(start_hotspot_on_boot))
|
||||
|
|
@ -142,6 +147,8 @@ async def lifespan(app: FastAPI):
|
|||
await status_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if rsync_daemon_task := getattr(app.state, "rsync_daemon_task", None):
|
||||
rsync_daemon_task.cancel()
|
||||
|
||||
|
||||
# Create FastAPI app
|
||||
|
|
|
|||
22
controller/rsyncd.conf
Normal file
22
controller/rsyncd.conf
Normal file
|
|
@ -0,0 +1,22 @@
|
|||
# How a secondary WROLPi copies this WROLPi's library: an rsync daemon that serves the media directory read-only.
|
||||
# The library is no more private here than it is over HTTPS at /media/.
|
||||
#
|
||||
# One file for every deployment: the systemd unit reads it on a Raspberry Pi and on Debian, and the Controller hands
|
||||
# it to the helper container in Docker (see controller/lib/rsyncd.py).
|
||||
port = 8873
|
||||
# Run as the unprivileged wrolpi user: no chroot, and no connection limit (which needs a lock file under /var/run).
|
||||
use chroot = no
|
||||
log file = /dev/stderr
|
||||
|
||||
[media]
|
||||
path = /media/wrolpi
|
||||
read only = yes
|
||||
list = yes
|
||||
# A symbolic link is sent as a link. Asked to follow links while it copies a directory, the daemon would send
|
||||
# the files left out below through a link to them from elsewhere in the library. (A link that is itself part of
|
||||
# the path a client asks for is still followed: the files below are left out by where they are, nothing more.)
|
||||
refuse options = copy-links copy-unsafe-links copy-dirlinks
|
||||
# Never served: this machine's TLS keys and cookies, and the database in use (a copy of a database that is
|
||||
# being written can be torn; a secondary copies the snapshot under config/.sync instead), and a snapshot that
|
||||
# is still being written.
|
||||
exclude = /config/ssl/ /config/cookies.txt /config/cookies.txt.enc /config/wrolpi.db /config/wrolpi.db-wal /config/wrolpi.db-shm /config/wrolpi.db.lock .wrolpi.db.backup /lost+found/
|
||||
|
|
@ -57,7 +57,7 @@ class TestManagedServicesConfig:
|
|||
def test_managed_services_count(self):
|
||||
"""Should have expected number of managed services."""
|
||||
services = DEFAULT_CONFIG["managed_services"]
|
||||
assert len(services) == 11
|
||||
assert len(services) == 12
|
||||
|
||||
@pytest.mark.parametrize("name,expected", [
|
||||
("wrolpi-api",
|
||||
|
|
|
|||
236
controller/test/test_rsyncd.py
Normal file
236
controller/test/test_rsyncd.py
Normal file
|
|
@ -0,0 +1,236 @@
|
|||
"""
|
||||
Unit tests for controller.lib.rsyncd: the rsync daemon that serves the library to secondary WROLPis.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import time
|
||||
from pathlib import Path
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
||||
from controller.lib import rsyncd
|
||||
from controller.lib.docker_services import NON_VIEWABLE_CONTAINERS
|
||||
|
||||
REPOSITORY = Path(__file__).parent.parent.parent
|
||||
|
||||
|
||||
class FakeNotFound(Exception):
|
||||
pass
|
||||
|
||||
|
||||
def make_client(helper=None):
|
||||
api = mock.Mock()
|
||||
api.id, api.name, api.image.id = "api-id", "wrolpi-api-1", "sha256:image"
|
||||
api.attrs = {"Config": {"User": "1000:1000"}}
|
||||
client = mock.Mock()
|
||||
|
||||
def get(name):
|
||||
if name == "wrolpi-api-1":
|
||||
return api
|
||||
if helper is not None:
|
||||
return helper
|
||||
raise FakeNotFound(name)
|
||||
|
||||
client.containers.get.side_effect = get
|
||||
return client
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def forget_images():
|
||||
rsyncd._images_without_rsync.clear()
|
||||
rsyncd._last_error = None
|
||||
|
||||
|
||||
class FakeContainerError(Exception):
|
||||
exit_status = 127
|
||||
|
||||
|
||||
def helper_runs(client) -> list:
|
||||
"""The calls that made a helper (the others only look for rsync in the image)."""
|
||||
return [i for i in client.containers.run.call_args_list if i.kwargs.get("detach")]
|
||||
|
||||
|
||||
def ensure(client):
|
||||
with mock.patch.object(rsyncd, "can_manage_containers", return_value=True), \
|
||||
mock.patch.object(rsyncd, "_get_client", return_value=client), \
|
||||
mock.patch.object(rsyncd, "NotFound", FakeNotFound):
|
||||
return rsyncd.ensure_rsync_daemon()
|
||||
|
||||
|
||||
def current_source(client) -> str:
|
||||
"""The label a helper made by this code from this api container carries."""
|
||||
ensure(client)
|
||||
return helper_runs(client)[0].kwargs["labels"][rsyncd.SOURCE_LABEL]
|
||||
|
||||
|
||||
class TestConfig:
|
||||
|
||||
def test_port(self):
|
||||
assert rsyncd.get_rsyncd_port() == 8873
|
||||
assert rsyncd.get_rsyncd_port("port = 9000\n[media]\n") == 9000
|
||||
with pytest.raises(RuntimeError):
|
||||
rsyncd.get_rsyncd_port("[media]\n")
|
||||
|
||||
def test_unit_reads_the_same_config(self):
|
||||
"""A Raspberry Pi, Debian and Docker all serve by one config file."""
|
||||
unit = (REPOSITORY / "etc/raspberrypios/wrolpi-rsyncd.service").read_text()
|
||||
assert "--config=/opt/wrolpi/controller/rsyncd.conf" in unit
|
||||
assert rsyncd.RSYNCD_CONFIG == REPOSITORY / "controller/rsyncd.conf"
|
||||
assert "User=wrolpi" in unit
|
||||
|
||||
def test_units_installed_and_enabled(self):
|
||||
for script in ("repair.sh", "pi-gen/stage2/04-wrolpi/03-run-chroot.sh",
|
||||
"debian-live-config/config/includes.chroot/usr/local/sbin/wrolpi-bootstrap.sh"):
|
||||
assert "wrolpi-rsyncd.service" in (REPOSITORY / script).read_text(), script
|
||||
assert "rsync" in (REPOSITORY / "docker/api/Dockerfile").read_text()
|
||||
|
||||
|
||||
@pytest.mark.skipif(not shutil.which("rsync") or b"rsync version 3" not in subprocess.run(
|
||||
["rsync", "--version"], capture_output=True).stdout, reason="needs rsync 3")
|
||||
def test_daemon_serves_the_library_read_only(tmp_path):
|
||||
"""The daemon run with the real config (only its port and path changed) lists the library, never serves the
|
||||
database in use or the TLS keys, serves the snapshot, and refuses a write."""
|
||||
media = tmp_path / "media"
|
||||
for path in ("videos/a.mp4", "config/tags.yaml", "config/wrolpi.db", "config/wrolpi.db-wal",
|
||||
"config/ssl/ca.key", "config/cookies.txt.enc", "config/.sync/config/wrolpi.db",
|
||||
"config/.sync/config/.wrolpi.db.backup"):
|
||||
(media / path).parent.mkdir(parents=True, exist_ok=True)
|
||||
(media / path).write_text(path)
|
||||
# Links that anyone able to write to the library could make, to what the daemon leaves out. Relative: rsync
|
||||
# already makes an absolute link point nowhere.
|
||||
(media / "videos/keys").symlink_to("../config/ssl")
|
||||
(media / "videos/key").symlink_to("../config/ssl/ca.key")
|
||||
with socket.socket() as probe:
|
||||
probe.bind(("127.0.0.1", 0))
|
||||
port = probe.getsockname()[1]
|
||||
config = rsyncd.read_rsyncd_config().replace("port = 8873", f"port = {port}\naddress = 127.0.0.1") \
|
||||
.replace("path = /media/wrolpi", f"path = {media}").replace("log file = /dev/stderr", "")
|
||||
assert str(media) in config and str(port) in config
|
||||
config_file = tmp_path / "rsyncd.conf"
|
||||
config_file.write_text(config)
|
||||
|
||||
daemon = subprocess.Popen(["rsync", "--daemon", "--no-detach", f"--config={config_file}"])
|
||||
try:
|
||||
url = f"rsync://127.0.0.1:{port}/media/"
|
||||
for _ in range(50):
|
||||
listing = subprocess.run(["rsync", "-r", "--list-only", url], capture_output=True, text=True)
|
||||
if listing.returncode == 0:
|
||||
break
|
||||
time.sleep(0.1)
|
||||
assert listing.returncode == 0, listing.stderr
|
||||
listed = {line.split()[-1] for line in listing.stdout.splitlines() if line.startswith("-")}
|
||||
assert listed == {"videos/a.mp4", "config/tags.yaml", "config/.sync/config/wrolpi.db"}
|
||||
|
||||
# Followed links are refused; without the option the links arrive as links, never as what they point at.
|
||||
for option in ("--copy-links", "--copy-unsafe-links", "--copy-dirlinks"):
|
||||
followed = subprocess.run(["rsync", "-r", option, url, str(tmp_path / "followed")],
|
||||
capture_output=True, text=True)
|
||||
assert followed.returncode != 0, option
|
||||
assert "refuse" in followed.stderr, followed.stderr
|
||||
leaked = [i for i in (tmp_path / "followed").rglob("*") if i.is_file() and "ssl/ca.key" in i.read_text()]
|
||||
assert not leaked, "a link let a client read the TLS key"
|
||||
copied = tmp_path / "copied"
|
||||
assert subprocess.run(["rsync", "-a", url, f"{copied}/"], capture_output=True).returncode == 0
|
||||
assert (copied / "videos/key").is_symlink() and (copied / "videos/keys").is_symlink()
|
||||
assert not (copied / "config/ssl").exists()
|
||||
|
||||
(tmp_path / "new.txt").write_text("new")
|
||||
write = subprocess.run(["rsync", str(tmp_path / "new.txt"), url], capture_output=True, text=True)
|
||||
assert write.returncode != 0 and "read only" in write.stderr
|
||||
assert not (media / "new.txt").exists()
|
||||
finally:
|
||||
daemon.terminate()
|
||||
daemon.wait(timeout=10)
|
||||
|
||||
|
||||
class TestEnsureRsyncDaemon:
|
||||
|
||||
def test_not_in_docker(self):
|
||||
with mock.patch.object(rsyncd, "can_manage_containers", return_value=False):
|
||||
assert rsyncd.ensure_rsync_daemon()["success"] is False
|
||||
|
||||
def test_creates_the_helper_from_the_api_container(self):
|
||||
client = make_client()
|
||||
assert ensure(client) == {"success": True, "action": "created"}
|
||||
(call,) = helper_runs(client)
|
||||
args, kwargs = call.args, call.kwargs
|
||||
assert args[0] == "sha256:image", "the api container's own image"
|
||||
assert kwargs["entrypoint"] == "sh" and "rsync --daemon" in args[1][1], "not the image's API entrypoint"
|
||||
assert kwargs["volumes_from"] == ["wrolpi-api-1:ro"], "the api container's volumes, read-only"
|
||||
assert kwargs["user"] == "1000:1000"
|
||||
assert kwargs["ports"] == {"8873/tcp": 8873}
|
||||
assert kwargs["environment"]["RSYNCD_CONF"] == rsyncd.read_rsyncd_config()
|
||||
assert kwargs["name"] == "wrolpi-rsyncd-1" and kwargs["detach"] is True
|
||||
assert kwargs["restart_policy"] == {"Name": "unless-stopped"}
|
||||
|
||||
def test_keeps_its_own_helper(self):
|
||||
helper = mock.Mock(status="running")
|
||||
helper.labels = {rsyncd.SOURCE_LABEL: current_source(make_client())}
|
||||
client = make_client(helper)
|
||||
assert ensure(client) == {"success": True, "action": "kept"}
|
||||
client.containers.run.assert_not_called(), "not even to look for rsync"
|
||||
helper.start.assert_not_called()
|
||||
helper.remove.assert_not_called()
|
||||
|
||||
def test_starts_a_stopped_helper(self):
|
||||
helper = mock.Mock(status="exited")
|
||||
helper.labels = {rsyncd.SOURCE_LABEL: current_source(make_client())}
|
||||
assert ensure(make_client(helper))["action"] == "kept"
|
||||
helper.start.assert_called_once()
|
||||
|
||||
def test_replaces_a_helper_made_from_another_api_container(self):
|
||||
"""A recreated api container may have other volumes; the helper must serve what the API has now."""
|
||||
helper = mock.Mock(status="running")
|
||||
helper.labels = {rsyncd.SOURCE_LABEL: "made from an older api container"}
|
||||
client = make_client(helper)
|
||||
assert ensure(client)["action"] == "created"
|
||||
helper.remove.assert_called_once_with(force=True)
|
||||
assert len(helper_runs(client)) == 1
|
||||
|
||||
def test_no_helper_from_an_image_without_rsync(self):
|
||||
"""An api image built before rsync was added: a helper made from it would exit and be restarted forever.
|
||||
None is made, an older helper is left serving, and the image is not tried again at every check."""
|
||||
helper = mock.Mock(status="running")
|
||||
helper.labels = {rsyncd.SOURCE_LABEL: "made from an older api container"}
|
||||
client = make_client(helper)
|
||||
client.containers.run.side_effect = FakeContainerError("sh: 1: rsync: not found")
|
||||
|
||||
for _ in range(3):
|
||||
result = ensure(client)
|
||||
assert result["success"] is False and "no rsync" in result["error"]
|
||||
assert not helper_runs(client)
|
||||
helper.remove.assert_not_called()
|
||||
assert client.containers.run.call_count == 1, "the image is looked at once"
|
||||
|
||||
def test_docker_failing_says_nothing_about_the_image(self):
|
||||
client = make_client()
|
||||
client.containers.run.side_effect = RuntimeError("docker is restarting")
|
||||
assert ensure(client)["success"] is False
|
||||
assert not rsyncd._images_without_rsync
|
||||
|
||||
client.containers.run.side_effect = None
|
||||
assert ensure(client)["action"] == "created"
|
||||
|
||||
def test_never_raises(self):
|
||||
client = make_client()
|
||||
client.containers.run.side_effect = RuntimeError("no such image")
|
||||
result = ensure(client)
|
||||
assert result["success"] is False and "no such image" in result["error"]
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_checked_for_as_long_as_the_controller_runs(self, monkeypatch):
|
||||
"""The api container can be recreated while the Controller runs; the helper is checked again and again."""
|
||||
calls = []
|
||||
monkeypatch.setattr(rsyncd, "ensure_rsync_daemon", lambda: calls.append(1))
|
||||
monkeypatch.setattr(rsyncd, "CHECK_INTERVAL_SECONDS", 0.01)
|
||||
task = asyncio.create_task(rsyncd.rsync_daemon_loop())
|
||||
await asyncio.sleep(0.2)
|
||||
task.cancel()
|
||||
assert len(calls) >= 3
|
||||
|
||||
def test_not_given_a_view_button(self):
|
||||
assert "rsyncd" in NON_VIEWABLE_CONTAINERS
|
||||
|
|
@ -280,8 +280,8 @@ fi
|
|||
# with --no-block so systemd queues the starts after we exit.
|
||||
|
||||
progress "Starting WROLPi services..."
|
||||
systemctl enable wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service
|
||||
systemctl start --no-block wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service
|
||||
systemctl enable wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service wrolpi-rsyncd.service
|
||||
systemctl start --no-block wrolpi-api.service wrolpi-app.service wrolpi-kiwix.service wrolpi-rsyncd.service
|
||||
|
||||
# If we did anything notable, leave a summary for the XFCE autostart to
|
||||
# show as a zenity dialog once the user lands on the desktop.
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ ENV DENO_TLS_CA_STORE=system
|
|||
WORKDIR /opt/wrolpi
|
||||
|
||||
# Update dependencies for the services install.
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip \
|
||||
RUN apt-get update && apt-get install -y ffmpeg catdoc sysstat aria2 iputils-ping unrar-free unzip rsync \
|
||||
build-essential libsqlite3-dev zlib1g-dev git
|
||||
|
||||
# Install tippecanoe (for map search index building via tippecanoe-decode).
|
||||
|
|
|
|||
15
etc/raspberrypios/wrolpi-rsyncd.service
Normal file
15
etc/raspberrypios/wrolpi-rsyncd.service
Normal file
|
|
@ -0,0 +1,15 @@
|
|||
[Unit]
|
||||
Description=WROLPi library for secondary WROLPis (rsync daemon)
|
||||
PartOf=wrolpi.target
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
ExecStart=/usr/bin/rsync --daemon --no-detach --config=/opt/wrolpi/controller/rsyncd.conf
|
||||
Group=wrolpi
|
||||
User=wrolpi
|
||||
Restart=on-failure
|
||||
# Below everything a person is waiting on.
|
||||
Nice=10
|
||||
|
||||
[Install]
|
||||
WantedBy=default.target
|
||||
|
|
@ -1,3 +1,3 @@
|
|||
[Unit]
|
||||
Description=WROLPi services group
|
||||
Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service wrolpi-kiwix.service
|
||||
Wants=wrolpi-controller.service wrolpi-api.service wrolpi-app.service caddy.service wrolpi-kiwix.service wrolpi-rsyncd.service
|
||||
|
|
|
|||
|
|
@ -106,6 +106,8 @@ cp /opt/wrolpi/etc/raspberrypios/wrolpi.target /etc/systemd/system/
|
|||
systemctl enable wrolpi-first-startup.service
|
||||
# Enable Controller service to start on boot.
|
||||
systemctl enable wrolpi-controller.service
|
||||
# Serve the library to secondary WROLPis.
|
||||
systemctl enable wrolpi-rsyncd.service
|
||||
# Enable weekly certificate renewal check.
|
||||
systemctl enable wrolpi-cert-renew.timer
|
||||
|
||||
|
|
|
|||
|
|
@ -94,6 +94,7 @@ systemctl enable wrolpi-app.service
|
|||
systemctl enable wrolpi-kiwix.service
|
||||
systemctl enable wrolpi-help.service
|
||||
systemctl enable wrolpi-mounts.service
|
||||
systemctl enable wrolpi-rsyncd.service
|
||||
systemctl enable wrolpi-cert-renew.timer
|
||||
|
||||
# Repair Controller (offline-safe - no pip install)
|
||||
|
|
@ -210,6 +211,8 @@ if [[ ${rpi} == true ]]; then
|
|||
fi
|
||||
|
||||
systemctl restart wrolpi-help
|
||||
# Restarted, not only started: its config may have changed.
|
||||
systemctl restart wrolpi-rsyncd || :
|
||||
systemctl start wrolpi.target
|
||||
|
||||
set +x
|
||||
|
|
|
|||
|
|
@ -12,9 +12,9 @@ from wrolpi.api_utils import api_app, json_response, perpetual_signal
|
|||
from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled
|
||||
from wrolpi.sync import backup, lib, schema
|
||||
from wrolpi.sync import backup, lib, schema, snapshot
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.sync.status import get_backup_status, get_sync_status
|
||||
from wrolpi.sync.status import get_backup_status, get_snapshot_status, get_sync_status
|
||||
from wrolpi.vars import DOCKERIZED
|
||||
|
||||
sync_bp = Blueprint('Sync', '/api/sync')
|
||||
|
|
@ -51,6 +51,7 @@ async def perpetual_sync_worker():
|
|||
# One loop, so a secondary's sync and its backup never run at once; the backup copies what the sync pulled.
|
||||
await lib.sync_tick()
|
||||
await backup.backup_tick()
|
||||
await snapshot.snapshot_tick()
|
||||
|
||||
|
||||
@sync_bp.get('/info')
|
||||
|
|
@ -78,7 +79,18 @@ async def get_manifest(_: Request, query: schema.ManifestRequest):
|
|||
@openapi.definition(summary="The progress of this secondary's sync")
|
||||
async def get_status(_: Request):
|
||||
return json_response(dict(sync=get_sync_status(), sync_running=bool(flags.sync_running.is_set()),
|
||||
backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set())))
|
||||
backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()),
|
||||
snapshot=get_snapshot_status()))
|
||||
|
||||
|
||||
@sync_bp.post('/snapshot')
|
||||
@openapi.definition(summary="Make a snapshot of this WROLPi's database for a secondary to copy")
|
||||
async def post_snapshot(_: Request):
|
||||
if reason := snapshot.snapshot_skip_reason():
|
||||
raise InvalidConfig(reason)
|
||||
# Made by the perpetual process; `/api/sync/status` says when it is done.
|
||||
snapshot.request_snapshot()
|
||||
return response.empty(HTTPStatus.ACCEPTED)
|
||||
|
||||
|
||||
@sync_bp.post('/run')
|
||||
|
|
|
|||
|
|
@ -171,6 +171,8 @@ def build_rsync_command(source: pathlib.Path, target: pathlib.Path, playlists_de
|
|||
'--info=progress2,stats2', '--no-human-readable',
|
||||
# The database is copied as a snapshot (see `snapshot_database`).
|
||||
'--exclude=/config/wrolpi.db', '--exclude=/config/wrolpi.db-*', '--exclude=/config/.wrolpi.db.backup',
|
||||
# The snapshot kept for secondaries (wrolpi.sync.snapshot); the drive has its own.
|
||||
'--exclude=/config/.sync/',
|
||||
# Hard links rebuilt from the configs when WROLPi starts.
|
||||
'--exclude=/tags/',
|
||||
# Files still being written by a sync, and rsync's own partials.
|
||||
|
|
|
|||
|
|
@ -35,7 +35,8 @@ from urllib.parse import quote
|
|||
|
||||
import aiohttp
|
||||
import yaml
|
||||
from sqlalchemy import func, select
|
||||
from sqlalchemy import func, select, text
|
||||
from sqlalchemy.exc import OperationalError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi import flags
|
||||
|
|
@ -191,9 +192,20 @@ def get_sync_info(session: Session) -> dict:
|
|||
role=get_sync_config().role,
|
||||
media_directory=str(get_media_directory()),
|
||||
max_file_group_id=session.execute(select(func.max(FileGroup.id))).scalar() or 0,
|
||||
# A secondary that replaces its database with this one must run code of the same migration.
|
||||
database_revision=_database_revision(session),
|
||||
# An unmounted media directory looks like an empty library.
|
||||
media_mounted=bool(flags.media_mounted.is_set()),
|
||||
)
|
||||
|
||||
|
||||
def _database_revision(session: Session) -> Optional[str]:
|
||||
try:
|
||||
return session.execute(text('SELECT version_num FROM alembic_version')).scalar()
|
||||
except OperationalError:
|
||||
return None
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
# Secondary: comparing and writing files.
|
||||
|
||||
|
|
|
|||
87
wrolpi/sync/snapshot.py
Normal file
87
wrolpi/sync/snapshot.py
Normal file
|
|
@ -0,0 +1,87 @@
|
|||
"""A primary WROLPi's database snapshot, for a secondary to copy.
|
||||
|
||||
A secondary copies its primary's files with rsync (the daemon of `controller/rsyncd.conf`), and replaces its own
|
||||
database with the primary's. A database that is in use cannot be copied as a file, so the secondary first asks the
|
||||
primary for a snapshot, which the daemon then serves like any other file. The snapshot is kept between requests: an
|
||||
unchanged database is not written again, and rsync has the previous copy to compare with.
|
||||
"""
|
||||
import asyncio
|
||||
import pathlib
|
||||
import shutil
|
||||
from typing import Optional
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import logger, get_media_directory, is_secondary, wrol_mode_enabled
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_file
|
||||
from wrolpi.sync.backup import snapshot_database_if_changed
|
||||
from wrolpi.sync.status import get_snapshot_status, update_snapshot_status
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
# Hidden, so nothing indexes it; inside config, which nothing but the daemon serves.
|
||||
SNAPSHOT_DIRECTORY = 'config/.sync'
|
||||
# `snapshot_database` writes a media directory's `config/wrolpi.db`.
|
||||
SNAPSHOT_PATH = f'{SNAPSHOT_DIRECTORY}/config/wrolpi.db'
|
||||
# Never fill the drive with a copy of the database; leave this much free.
|
||||
MIN_FREE_BYTES = 1024 ** 3
|
||||
|
||||
|
||||
def get_snapshot_directory() -> pathlib.Path:
|
||||
return get_media_directory() / SNAPSHOT_DIRECTORY
|
||||
|
||||
|
||||
def snapshot_skip_reason() -> Optional[str]:
|
||||
"""Why a snapshot cannot be made now, or None."""
|
||||
if is_secondary():
|
||||
return 'A secondary WROLPi is not copied; its primary is'
|
||||
if wrol_mode_enabled():
|
||||
# Like the backup drive: nothing new is written in WROL Mode. A snapshot made earlier is still served.
|
||||
return 'WROL Mode is enabled'
|
||||
if not PYTEST and not flags.db_up.is_set():
|
||||
return 'The database is not up'
|
||||
try:
|
||||
size = get_db_file().stat().st_size
|
||||
except FileNotFoundError:
|
||||
return 'There is no database'
|
||||
# The old snapshot is replaced only after the new one is complete, so both exist for a while.
|
||||
if shutil.disk_usage(get_media_directory()).free - size < MIN_FREE_BYTES:
|
||||
return 'Not enough free space for a copy of the database'
|
||||
return None
|
||||
|
||||
|
||||
def request_snapshot():
|
||||
update_snapshot_status(requested=True)
|
||||
|
||||
|
||||
async def run_snapshot() -> dict:
|
||||
"""Make the snapshot (or keep the one there is, if the database has not changed). Returns the final status."""
|
||||
if reason := snapshot_skip_reason():
|
||||
logger.info(f'Not making a database snapshot: {reason}')
|
||||
return update_snapshot_status(state='error', finished_at=now().isoformat(), last_error=reason)
|
||||
|
||||
update_snapshot_status(state='running', started_at=now().isoformat(), finished_at=None, last_error=None)
|
||||
try:
|
||||
written = await asyncio.to_thread(snapshot_database_if_changed, get_snapshot_directory())
|
||||
size = (get_media_directory() / SNAPSHOT_PATH).stat().st_size
|
||||
except Exception as e:
|
||||
logger.error('Failed to make a database snapshot', exc_info=e)
|
||||
return update_snapshot_status(state='error', finished_at=now().isoformat(),
|
||||
last_error=str(e) or type(e).__name__)
|
||||
return update_snapshot_status(state='idle', finished_at=now().isoformat(), path=SNAPSHOT_PATH, size=size,
|
||||
written=written)
|
||||
|
||||
|
||||
async def snapshot_tick():
|
||||
"""Make a snapshot when one was requested."""
|
||||
status = get_snapshot_status()
|
||||
if status['state'] == 'running':
|
||||
# Only this loop makes snapshots, one at a time: one still marked running when a tick starts was cut off
|
||||
# (its process was killed), and would otherwise hold back every later request.
|
||||
status = update_snapshot_status(state='error', finished_at=now().isoformat(),
|
||||
last_error='The snapshot was interrupted')
|
||||
if not status['requested']:
|
||||
return
|
||||
update_snapshot_status(requested=False)
|
||||
await run_snapshot()
|
||||
|
|
@ -48,6 +48,39 @@ def update_sync_status(**changes) -> dict:
|
|||
return status
|
||||
|
||||
|
||||
def default_snapshot_status() -> dict:
|
||||
return dict(
|
||||
state='idle',
|
||||
requested=False,
|
||||
started_at=None,
|
||||
finished_at=None,
|
||||
last_error=None,
|
||||
# Relative to the media directory; where a secondary finds the snapshot.
|
||||
path=None,
|
||||
size=0,
|
||||
# False when the last request found the database unchanged and kept the snapshot it had.
|
||||
written=False,
|
||||
)
|
||||
|
||||
|
||||
# Kept out of `shared_ctx.status`, which every `/api/status` poll returns: only a secondary's Controller asks for it.
|
||||
SNAPSHOT_STATUS_KEY = 'sync_snapshot_status'
|
||||
|
||||
|
||||
def get_snapshot_status() -> dict:
|
||||
from wrolpi.api_utils import api_app
|
||||
return dict(api_app.shared_ctx.cache.get(SNAPSHOT_STATUS_KEY) or default_snapshot_status())
|
||||
|
||||
|
||||
def update_snapshot_status(**changes) -> dict:
|
||||
"""Merge `changes` into the shared snapshot status (see `update_sync_status`)."""
|
||||
from wrolpi.api_utils import api_app
|
||||
status = get_snapshot_status()
|
||||
status.update(changes)
|
||||
api_app.shared_ctx.cache[SNAPSHOT_STATUS_KEY] = status
|
||||
return status
|
||||
|
||||
|
||||
def default_backup_status() -> dict:
|
||||
return dict(
|
||||
state='idle',
|
||||
|
|
|
|||
130
wrolpi/sync/test/test_snapshot.py
Normal file
130
wrolpi/sync/test/test_snapshot.py
Normal file
|
|
@ -0,0 +1,130 @@
|
|||
import contextlib
|
||||
import shutil
|
||||
import sqlite3
|
||||
from collections import namedtuple
|
||||
from http import HTTPStatus
|
||||
|
||||
import pytest
|
||||
|
||||
from wrolpi.common import enable_wrol_mode
|
||||
from wrolpi.db import get_db_file
|
||||
from wrolpi.sync import backup, snapshot
|
||||
from wrolpi.sync.status import get_snapshot_status, update_snapshot_status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_info_says_what_a_secondary_must_check(async_client, test_session, test_wrolpi_config):
|
||||
"""Before a secondary replaces its database with its primary's, it must know the primary's migration and that the
|
||||
primary's media directory is really there."""
|
||||
request, response = await async_client.get('/api/sync/info')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert 'database_revision' in response.json
|
||||
assert response.json['media_mounted'] in (True, False)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot(async_client, test_session, test_directory, test_wrolpi_config):
|
||||
"""A requested snapshot is a complete database where the rsync daemon serves it, and an unchanged database is
|
||||
not written again."""
|
||||
with contextlib.closing(sqlite3.connect(get_db_file())) as live:
|
||||
live.execute('CREATE TABLE snapshot_test (name TEXT)')
|
||||
live.execute("INSERT INTO snapshot_test VALUES ('one')")
|
||||
live.commit()
|
||||
|
||||
request, response = await async_client.post('/api/sync/snapshot')
|
||||
assert response.status_code == HTTPStatus.ACCEPTED
|
||||
assert get_snapshot_status()['requested'] is True
|
||||
assert not (test_directory / snapshot.SNAPSHOT_PATH).exists(), 'made by the perpetual process, not the request'
|
||||
|
||||
await snapshot.snapshot_tick()
|
||||
|
||||
status = get_snapshot_status()
|
||||
assert status['state'] == 'idle' and status['last_error'] is None and status['requested'] is False
|
||||
assert status['path'] == 'config/.sync/config/wrolpi.db' and status['written'] is True
|
||||
copy = test_directory / status['path']
|
||||
assert status['size'] == copy.stat().st_size > 0
|
||||
with contextlib.closing(sqlite3.connect(f'file:{copy}?mode=ro', uri=True)) as connection:
|
||||
assert connection.execute('SELECT name FROM snapshot_test').fetchall() == [('one',)]
|
||||
|
||||
request, response = await async_client.get('/api/sync/status')
|
||||
assert response.json['snapshot']['path'] == status['path']
|
||||
|
||||
# Nothing asked for: nothing made.
|
||||
modified = copy.stat().st_mtime_ns
|
||||
await snapshot.snapshot_tick()
|
||||
assert copy.stat().st_mtime_ns == modified
|
||||
|
||||
# Asked again with the database unchanged: the snapshot there is kept.
|
||||
snapshot.request_snapshot()
|
||||
await snapshot.snapshot_tick()
|
||||
assert get_snapshot_status()['written'] is False and get_snapshot_status()['state'] == 'idle'
|
||||
assert copy.stat().st_mtime_ns == modified
|
||||
|
||||
# The database changed: written again.
|
||||
with contextlib.closing(sqlite3.connect(get_db_file())) as live:
|
||||
live.execute("INSERT INTO snapshot_test VALUES ('two')")
|
||||
live.commit()
|
||||
snapshot.request_snapshot()
|
||||
await snapshot.snapshot_tick()
|
||||
assert get_snapshot_status()['written'] is True
|
||||
with contextlib.closing(sqlite3.connect(f'file:{copy}?mode=ro', uri=True)) as connection:
|
||||
assert connection.execute('SELECT count(*) FROM snapshot_test').fetchone() == (2,)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_refused(async_client, test_session, test_directory, test_wrolpi_config, make_secondary,
|
||||
monkeypatch):
|
||||
"""No snapshot is made when it would fill the drive, or on a secondary (which nothing copies)."""
|
||||
usage = namedtuple('usage', 'total used free')
|
||||
monkeypatch.setattr(snapshot.shutil, 'disk_usage', lambda _: usage(10, 10, snapshot.MIN_FREE_BYTES))
|
||||
request, response = await async_client.post('/api/sync/snapshot')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert 'free space' in response.json['error']
|
||||
# The space ran out between the request and the snapshot.
|
||||
snapshot.request_snapshot()
|
||||
await snapshot.snapshot_tick()
|
||||
assert get_snapshot_status()['state'] == 'error' and 'free space' in get_snapshot_status()['last_error']
|
||||
assert not (test_directory / snapshot.SNAPSHOT_PATH).exists()
|
||||
monkeypatch.setattr(snapshot.shutil, 'disk_usage', shutil.disk_usage)
|
||||
|
||||
make_secondary('https://primary.example')
|
||||
request, response = await async_client.post('/api/sync/snapshot')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert not get_snapshot_status()['requested']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_refused_in_wrol_mode(async_client, test_session, test_directory, test_wrolpi_config):
|
||||
"""WROL Mode writes nothing new: no snapshot is made, whether it is asked for now or was asked for before."""
|
||||
snapshot.request_snapshot()
|
||||
enable_wrol_mode()
|
||||
|
||||
request, response = await async_client.post('/api/sync/snapshot')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST and 'WROL Mode' in response.json['error']
|
||||
|
||||
await snapshot.snapshot_tick()
|
||||
assert get_snapshot_status()['state'] == 'error' and 'WROL Mode' in get_snapshot_status()['last_error']
|
||||
assert not (test_directory / snapshot.SNAPSHOT_PATH).exists()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_snapshot_interrupted(async_client, test_session, test_directory, test_wrolpi_config):
|
||||
"""A snapshot cut off by its process being killed stays marked as running. It must not hold back the next
|
||||
request, nor be reported as running forever."""
|
||||
update_snapshot_status(state='running', started_at='2026-01-01T00:00:00+00:00')
|
||||
|
||||
await snapshot.snapshot_tick()
|
||||
assert get_snapshot_status()['state'] == 'error'
|
||||
assert get_snapshot_status()['last_error'] == 'The snapshot was interrupted'
|
||||
|
||||
update_snapshot_status(state='running')
|
||||
snapshot.request_snapshot()
|
||||
await snapshot.snapshot_tick()
|
||||
assert get_snapshot_status()['state'] == 'idle' and get_snapshot_status()['written'] is True
|
||||
assert (test_directory / snapshot.SNAPSHOT_PATH).is_file()
|
||||
|
||||
|
||||
def test_backup_drive_does_not_copy_the_snapshot(test_directory, tmp_path):
|
||||
"""The backup drive gets its own snapshot; the one kept for secondaries is not copied to it as well."""
|
||||
command = backup.build_rsync_command(test_directory, tmp_path)
|
||||
assert f'--exclude=/{snapshot.SNAPSHOT_DIRECTORY}/' in command
|
||||
Loading…
Add table
Add a link
Reference in a new issue