mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
The backup drive keeps deleted files for 30 days, then removes them (#999)
* The backup drive keeps deleted files for 30 days, then removes them The fifth part of replacing the sync. A backup drive never deleted anything, so it grew forever and kept every file the media directory had ever held. Now, like a secondary: - A file deleted from the media directory, or the old version of one that changed, is moved to a dated folder in .backup-trash on the backup drive. Folders older than backup_delete_after_days (sync.yaml; 30 by default, also for drives set up before this; none never deletes) are removed, before and after each backup, and the oldest go sooner when the drive is nearly full. - A backup refuses a media directory that is not mounted, or has no config/wrolpi.yaml: copied with deletes, an empty folder would move the whole backup drive to the trash. - The Backup Drive panel has the same "Delete after N days / Never delete them" control as the Sync panel, now one component, and refuses an empty number of days. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * A nearly full drive keeps the trash it just filled, and makes room for the database Review found, in the backup drive's trash and in the secondary's sync, which share its shape: - Below 1 GiB free, the purge after a copy removed the newest trash folder too. Moving a file to the trash is a rename, which frees nothing, so on a nearly full drive a deleted file was gone at the end of the same run instead of 30 days later. The newest folder is now only removed by age (0 days still deletes at once). - The purge made room only down to 1 GiB, so a database larger than the free space failed to be copied every time while the trash was full. Before the database is written (the backup's snapshot, the secondary's copy of its primary's), the trash now makes room for the whole database. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
38885990f6
commit
a5ab01198b
14 changed files with 345 additions and 50 deletions
|
|
@ -574,8 +574,12 @@ def directory_size(directory: Path) -> int:
|
|||
|
||||
def purge_trash(media_directory: Path, delete_after_days: Optional[int], today: datetime.date = None,
|
||||
min_free_bytes: int = MIN_FREE_BYTES) -> list:
|
||||
"""Remove the trash folders that have waited `delete_after_days` and, while the drive is nearly full, the
|
||||
oldest of the rest. A folder is as old as its name says, whatever its files' times are. Returns their names."""
|
||||
"""Remove the trash folders that have waited `delete_after_days` and, while the drive has less than
|
||||
`min_free_bytes` free, the oldest of the rest. A folder is as old as its name says, whatever its files' times
|
||||
are. Returns their names.
|
||||
|
||||
The newest folder is never removed for space: moving a file to the trash is a rename, which frees nothing, so a
|
||||
nearly full drive would otherwise delete what this very sync moved there at once."""
|
||||
trash = media_directory / TRASH_DIRECTORY
|
||||
if trash.is_symlink() or not trash.is_dir():
|
||||
return []
|
||||
|
|
@ -587,9 +591,11 @@ def purge_trash(media_directory: Path, delete_after_days: Optional[int], today:
|
|||
with contextlib.suppress(ValueError):
|
||||
dated.append((datetime.date.fromisoformat(folder.name), folder))
|
||||
purged = []
|
||||
for day, folder in sorted(dated):
|
||||
dated.sort()
|
||||
newest = dated[-1][1] if dated else None
|
||||
for day, folder in dated:
|
||||
expired = delete_after_days is not None and (today - day).days >= delete_after_days
|
||||
nearly_full = shutil.disk_usage(media_directory).free < min_free_bytes
|
||||
nearly_full = folder != newest and shutil.disk_usage(media_directory).free < min_free_bytes
|
||||
if not expired and not nearly_full:
|
||||
continue
|
||||
logger.info("Removing the sync trash of %s (%s)", folder.name, "expired" if expired else "drive nearly full")
|
||||
|
|
@ -630,7 +636,8 @@ def run_sync(runner=None) -> dict:
|
|||
snapshot = request_snapshot(primary_url)
|
||||
# The trash is what fills the drive, and a full drive fails every run: make room before the copy, not
|
||||
# only after a run that succeeded.
|
||||
purge_trash(media_directory, delete_after_days)
|
||||
# Room for the primary's database, too, before anything is stopped.
|
||||
purge_trash(media_directory, delete_after_days, min_free_bytes=int(snapshot.get("size") or 0) + MIN_FREE_BYTES)
|
||||
if shutil.disk_usage(media_directory).free - int(snapshot.get("size") or 0) < MIN_FREE_BYTES:
|
||||
raise SyncError("Not enough free space for the primary's database")
|
||||
_check_cancel()
|
||||
|
|
|
|||
|
|
@ -849,3 +849,27 @@ class TestRecoveryAndSchedule:
|
|||
monkeypatch.setattr(sync, "update_status", watch)
|
||||
sync.run_sync(FakeRunner(media))
|
||||
assert locked_when_saved == [True]
|
||||
|
||||
|
||||
def test_purge_never_removes_the_newest_for_space(media, monkeypatch):
|
||||
"""Moving a file to the trash frees nothing, so on a nearly full drive the trash this sync just filled would be
|
||||
removed by the same sync. Only age removes the newest folder."""
|
||||
trash = media / ".sync-trash"
|
||||
for name in ("2026-11-17", "2026-11-18"):
|
||||
(trash / name).mkdir(parents=True)
|
||||
usage = namedtuple("usage", "total used free")
|
||||
monkeypatch.setattr(sync.shutil, "disk_usage", lambda _: usage(100, 100, 0))
|
||||
assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=10) == ["2026-11-17"]
|
||||
assert (trash / "2026-11-18").is_dir()
|
||||
assert sync.purge_trash(media, 0, datetime.date(2026, 11, 18), min_free_bytes=10) == ["2026-11-18"]
|
||||
|
||||
|
||||
def test_room_is_made_for_the_primarys_database(media, primary_api, monkeypatch):
|
||||
"""Before the API is stopped, the trash makes room for the primary's database, not only the usual 1 GiB."""
|
||||
purges = []
|
||||
monkeypatch.setattr(sync, "purge_trash",
|
||||
lambda media_directory, days, today=None, min_free_bytes=sync.MIN_FREE_BYTES:
|
||||
purges.append(min_free_bytes))
|
||||
sync.run_sync(FakeRunner(media))
|
||||
# The snapshot of primary_api is 1000 bytes.
|
||||
assert purges[0] == 1000 + sync.MIN_FREE_BYTES
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue