mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
A secondary deletes its own stale write-ahead logs, and the sync leaves no stray files (#994)
* A secondary deletes its own stale write-ahead logs, and the sync leaves no stray files Found by the first sync between two real machines: - The primary's unfinished files (SQLite logs, partial downloads) were excluded from the copy, and an excluded path is never deleted on the receiving side. So a secondary kept its own write-ahead log beside a map search database the primary had replaced, and SQLite would apply that log to the new database. These patterns are now only hidden on the primary's side, so the secondary's stale ones are deleted (to the trash). - Reading the incoming snapshot to check it made a -wal and -shm beside it, which stayed behind when the snapshot was moved into place. It is now opened immutable. - The primary's info, written by the Controller (root), belonged to root. Files the Controller writes in the media directory now belong to its owner, like the directories it makes. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * With deletes turned off, a secondary still drops its stale unfinished files Review found that the hide rules of the previous commit only take effect through --delete, which is not given when delete_after_days is null (never delete). A secondary then kept its own write-ahead log beside a database the primary had replaced, and SQLite applied it to the new database. Without a trash, --delete is now given with every path protected except the unfinished patterns (and what is inside an unfinished folder), so only those are removed. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
ea7cc4dec2
commit
6772762ce9
2 changed files with 87 additions and 15 deletions
|
|
@ -82,8 +82,12 @@ LOCAL_PATHS = (
|
|||
f"/{SYNC_DIRECTORY}",
|
||||
f"/{TRASH_DIRECTORY}", "/lost+found",
|
||||
)
|
||||
# Anywhere in the library: files still being written on the primary.
|
||||
UNFINISHED_PATTERNS = (PARTIAL_DIRECTORY, ".partial", "*.part", "*.aria2", "*.ytdl", "*.db-wal", "*.db-shm")
|
||||
# Anywhere in the library: files still being written on the primary, which are not copied. Hidden from the
|
||||
# primary's side only (rsync's "hide"), not excluded: the secondary's own are stale, and must go. An excluded
|
||||
# path is never deleted, so a secondary kept the write-ahead log of a map search database whose database the
|
||||
# primary had replaced, and SQLite would apply that log to the new database. (rsync keeps its own partial
|
||||
# directory itself.)
|
||||
UNFINISHED_PATTERNS = (".partial", "*.part", "*.aria2", "*.ytdl", "*.db-wal", "*.db-shm")
|
||||
|
||||
|
||||
class SyncError(Exception):
|
||||
|
|
@ -171,11 +175,14 @@ def checked_path(path: Path) -> Path:
|
|||
|
||||
def write_file(file: Path, text: str):
|
||||
"""Write `file` in the media directory without following a link: to a new file of its own, renamed into
|
||||
place."""
|
||||
place. It belongs to the media directory's owner, like everything else in the library."""
|
||||
checked_path(file)
|
||||
owner = get_media_directory().stat()
|
||||
temporary = file.with_name(f".{file.name}.{uuid.uuid4().hex}.tmp")
|
||||
descriptor = os.open(temporary, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o644)
|
||||
with os.fdopen(descriptor, "w") as fh:
|
||||
with contextlib.suppress(PermissionError):
|
||||
os.fchown(fh.fileno(), owner.st_uid, owner.st_gid)
|
||||
fh.write(text)
|
||||
os.replace(temporary, file)
|
||||
|
||||
|
|
@ -252,7 +259,16 @@ def build_files_command(primary_url: str, media_directory: Path, trash: Optional
|
|||
for playlists in dict.fromkeys(str(i or "").strip("/") for i in playlists_destinations):
|
||||
if playlists:
|
||||
arguments.append(f"--exclude=/{rsync_literal(playlists)}")
|
||||
arguments.extend(f"--exclude={i}" for i in UNFINISHED_PATTERNS)
|
||||
arguments.extend(f"--filter=H {i}" for i in UNFINISHED_PATTERNS)
|
||||
if trash is None:
|
||||
# Nothing of the library is deleted, but the secondary's stale unfinished files must still go: a log left
|
||||
# beside a database the primary replaced would be applied to it. Only those are at risk (R); everything
|
||||
# else is protected (P) from the --delete that removes them.
|
||||
arguments.append("--delete")
|
||||
for pattern in UNFINISHED_PATTERNS:
|
||||
# The pattern itself, and what is inside it when it is a folder (".partial").
|
||||
arguments.extend([f"--filter=R {pattern}", f"--filter=R {pattern}/**"])
|
||||
arguments.append("--filter=P *")
|
||||
# Folders the owner chose not to copy.
|
||||
for directory in excluded_directories or []:
|
||||
if directory := str(directory).strip("/"):
|
||||
|
|
@ -516,7 +532,8 @@ def replace_database(media_directory: Path):
|
|||
database complete."""
|
||||
database = checked_path(media_directory / "config/wrolpi.db")
|
||||
incoming = checked_path(media_directory / INCOMING_DATABASE)
|
||||
with contextlib.closing(sqlite3.connect(f"file:{incoming}?mode=ro", uri=True)) as connection:
|
||||
# Immutable: read without making a -wal and -shm beside it, which would be left behind when it is moved.
|
||||
with contextlib.closing(sqlite3.connect(f"file:{incoming}?mode=ro&immutable=1", uri=True)) as connection:
|
||||
if connection.execute("PRAGMA schema_version").fetchone()[0] == 0:
|
||||
raise SyncError("The snapshot from the primary is not a database")
|
||||
if database.is_file():
|
||||
|
|
|
|||
|
|
@ -111,9 +111,11 @@ class TestCommand:
|
|||
assert f"--backup-dir={media}/.sync-trash/2026-11-18" in command
|
||||
for excluded in ("/tags", "/my playlists", "/lists", "/config/wrolpi.yaml", "/config/sync.yaml", "/config/ssl",
|
||||
"/config/wrolpi.db", "/config/wrolpi.db-wal", "/config/.sync", "/.sync-trash",
|
||||
"/config/sync_state.yaml", "/config/.wrol_mode", "*.part", ".partial", "*.db-wal",
|
||||
"/old \\[stuff]", "/a/b"):
|
||||
"/config/sync_state.yaml", "/config/.wrol_mode", "/old \\[stuff]", "/a/b"):
|
||||
assert f"--exclude={excluded}" in command, excluded
|
||||
# Unfinished files are only hidden from the primary's side, so the secondary's stale ones are deleted.
|
||||
for hidden in ("*.part", ".partial", "*.db-wal", "*.db-shm"):
|
||||
assert f"--filter=H {hidden}" in command and f"--exclude={hidden}" not in command, hidden
|
||||
# A pattern ending in a slash matches only a directory; a file or link of that name would get through.
|
||||
assert not [i for i in command if i.startswith("--exclude=") and i.endswith("/")]
|
||||
assert command.count("--exclude=/my playlists") == 1
|
||||
|
|
@ -134,8 +136,14 @@ class TestCommand:
|
|||
f"{media}/config/.sync/incoming/wrolpi.db"]
|
||||
|
||||
def test_never_deletes_without_a_trash(self, media):
|
||||
"""Without a trash nothing of the library is deleted; only the secondary's stale unfinished files are."""
|
||||
command = sync.build_files_command(PRIMARY, media, None)
|
||||
assert "--delete" not in command and "--backup" not in command
|
||||
assert "--backup" not in command
|
||||
protect = command.index("--filter=P *")
|
||||
for pattern in sync.UNFINISHED_PATTERNS:
|
||||
# rsync uses the first rule that matches: the risk rules must come before the protect rule.
|
||||
assert command.index(f"--filter=R {pattern}") < protect, pattern
|
||||
assert command.index(f"--filter=R {pattern}/**") < protect, pattern
|
||||
|
||||
@pytest.mark.parametrize("url, expected", [
|
||||
("https://10.0.0.5", "rsync://10.0.0.5:8873/media/"),
|
||||
|
|
@ -188,6 +196,15 @@ class TestRootInAnotherUsersDirectory:
|
|||
sync.make_directory(media / sync.SYNC_DIRECTORY / "incoming")
|
||||
assert not (tmp_path / "incoming").exists() and not (tmp_path / "primary-info.yaml").exists()
|
||||
|
||||
def test_files_belong_to_the_library_owner(self, media, monkeypatch):
|
||||
chowned = []
|
||||
monkeypatch.setattr(sync.os, "fchown", lambda fd, uid, gid: chowned.append((uid, gid)))
|
||||
sync.make_directory(media / sync.SYNC_DIRECTORY)
|
||||
sync.write_file(media / sync.PRIMARY_INFO, "media_directory: /media/wrolpi\n")
|
||||
owner = media.stat()
|
||||
assert chowned == [(owner.st_uid, owner.st_gid)]
|
||||
assert (media / sync.PRIMARY_INFO).read_text() == "media_directory: /media/wrolpi\n"
|
||||
|
||||
def test_directories_belong_to_the_library_owner(self, media, monkeypatch):
|
||||
"""rsync does not run as root, and must write into what the Controller made."""
|
||||
chowned = []
|
||||
|
|
@ -283,22 +300,31 @@ class TestPurge:
|
|||
class TestReplaceDatabase:
|
||||
|
||||
def test_replaces_and_removes_the_old_log(self, media):
|
||||
"""The old write-ahead log left beside the new database would be applied to it."""
|
||||
"""A log left beside the old database (by an API that was killed) would be applied to the new one."""
|
||||
database = media / "config/wrolpi.db"
|
||||
make_database(database, "this secondary's")
|
||||
live = sqlite3.connect(database)
|
||||
live.execute("PRAGMA journal_mode=WAL")
|
||||
live.execute("UPDATE t SET name = 'written to the log'")
|
||||
live.commit()
|
||||
assert database.with_name("wrolpi.db-wal").stat().st_size > 0
|
||||
make_database(media / sync.INCOMING_DATABASE, "from the primary")
|
||||
log = database.with_name("wrolpi.db-wal")
|
||||
left_behind = log.read_bytes()
|
||||
assert left_behind
|
||||
live.close()
|
||||
log.write_bytes(left_behind)
|
||||
incoming = media / sync.INCOMING_DATABASE
|
||||
make_database(incoming, "from the primary")
|
||||
with sqlite3.connect(incoming) as connection:
|
||||
# The snapshot of a database in WAL mode is in WAL mode too.
|
||||
connection.execute("PRAGMA journal_mode=WAL")
|
||||
connection.close()
|
||||
|
||||
sync.replace_database(media)
|
||||
live.close()
|
||||
|
||||
assert read_database(database) == "from the primary"
|
||||
assert not database.with_name("wrolpi.db-wal").exists() and not database.with_name("wrolpi.db-shm").exists()
|
||||
assert not (media / sync.INCOMING_DATABASE).exists()
|
||||
# Nothing is left where the snapshot was; a log there would be applied to the next snapshot.
|
||||
assert not list(incoming.parent.iterdir()), list(incoming.parent.iterdir())
|
||||
assert read_database(database) == "from the primary"
|
||||
|
||||
def test_refuses_what_is_not_a_database(self, media):
|
||||
make_database(media / "config/wrolpi.db", "this secondary's")
|
||||
|
|
@ -479,6 +505,8 @@ def test_sync_with_a_real_daemon(media, tmp_path, primary_api, monkeypatch):
|
|||
# Where this machine has a directory of its own, the primary has a file of that name.
|
||||
(primary / "config/backup").write_text("not a directory on the primary")
|
||||
write(media, {"config/backup/tags-20261118.yaml": "this machine's config backup"})
|
||||
# The secondary's own: a log beside a search database the primary replaced, and an unfinished download.
|
||||
write(media, {"map/region.search.db-wal": "this machine's stale log", "map/.partial/old.pmtiles": "stale"})
|
||||
write(media, {
|
||||
"videos/changed.txt": "older", "videos/deleted on the primary.txt": "gone there",
|
||||
"tags/mine/x.txt": "this machine's links", "config/wrolpi.yaml": "hotspot_ssid: mine\n",
|
||||
|
|
@ -530,8 +558,11 @@ def test_sync_with_a_real_daemon(media, tmp_path, primary_api, monkeypatch):
|
|||
assert (media / "config/backup/tags-20261118.yaml").read_text() == "this machine's config backup"
|
||||
assert (media / "videos/inside").is_symlink() and (media / "videos/inside").read_text() == "a video"
|
||||
assert not (media / "videos/outside").is_symlink() and not (media / "videos/climbs out").is_symlink()
|
||||
# Unfinished files and the excluded folder stay on the primary.
|
||||
# Unfinished files and the excluded folder stay on the primary; the secondary's own stale ones are gone
|
||||
# (to the trash, like any deleted file).
|
||||
assert not (media / "map/.partial").exists() and not (media / "map/region.search.db-wal").exists()
|
||||
assert (media / ".sync-trash" / datetime.date.today().isoformat() / "map/region.search.db-wal").read_text() \
|
||||
== "this machine's stale log"
|
||||
assert not (media / "excluded").exists()
|
||||
# The database is the primary's snapshot, never its database in use; the library settings come along.
|
||||
assert read_database(media / "config/wrolpi.db") == "from the primary"
|
||||
|
|
@ -616,3 +647,27 @@ def test_a_primary_that_stops_answering_ends_rsync(media, monkeypatch):
|
|||
gave_up.cancel()
|
||||
sync._cancel.clear()
|
||||
assert code != 0
|
||||
|
||||
|
||||
@pytest.mark.skipif(not RSYNC_3, reason="needs rsync 3")
|
||||
def test_never_delete_still_removes_stale_logs(tmp_path):
|
||||
"""With deletes turned off, a secondary keeps every file its primary no longer has, but not its own write-ahead
|
||||
log beside a database the primary replaced, which SQLite would apply to the new database."""
|
||||
primary, secondary = tmp_path / "primary", tmp_path / "secondary"
|
||||
make_database(primary / "map/region.search.db", "the primary's new index")
|
||||
write(secondary, {"map/region.search.db": "old", "map/region.search.db-wal": "this machine's stale log",
|
||||
"map/.partial/old.pmtiles": "stale", "videos/deleted on the primary.mp4": "kept",
|
||||
"kept folder/a.txt": "kept"})
|
||||
command = sync.build_files_command(PRIMARY, secondary, None)
|
||||
# From a local folder rather than a daemon, which takes no connection timeout.
|
||||
command = [i for i in command if not i.startswith("--contimeout")]
|
||||
command[-2] = f"{primary}/"
|
||||
result = subprocess.run(["rsync", *command], capture_output=True, text=True)
|
||||
assert result.returncode == 0, result.stdout + result.stderr
|
||||
|
||||
assert not (secondary / "map/region.search.db-wal").exists()
|
||||
assert not (secondary / "map/.partial/old.pmtiles").exists()
|
||||
assert (secondary / "videos/deleted on the primary.mp4").read_text() == "kept"
|
||||
assert (secondary / "kept folder/a.txt").read_text() == "kept"
|
||||
assert read_database(secondary / "map/region.search.db") == "the primary's new index"
|
||||
assert not (secondary / ".sync-trash").exists()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue