mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
The backup drive keeps deleted files for 30 days, then removes them (#999)
* The backup drive keeps deleted files for 30 days, then removes them The fifth part of replacing the sync. A backup drive never deleted anything, so it grew forever and kept every file the media directory had ever held. Now, like a secondary: - A file deleted from the media directory, or the old version of one that changed, is moved to a dated folder in .backup-trash on the backup drive. Folders older than backup_delete_after_days (sync.yaml; 30 by default, also for drives set up before this; none never deletes) are removed, before and after each backup, and the oldest go sooner when the drive is nearly full. - A backup refuses a media directory that is not mounted, or has no config/wrolpi.yaml: copied with deletes, an empty folder would move the whole backup drive to the trash. - The Backup Drive panel has the same "Delete after N days / Never delete them" control as the Sync panel, now one component, and refuses an empty number of days. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * A nearly full drive keeps the trash it just filled, and makes room for the database Review found, in the backup drive's trash and in the secondary's sync, which share its shape: - Below 1 GiB free, the purge after a copy removed the newest trash folder too. Moving a file to the trash is a rename, which frees nothing, so on a nearly full drive a deleted file was gone at the end of the same run instead of 30 days later. The newest folder is now only removed by age (0 days still deletes at once). - The purge made room only down to 1 GiB, so a database larger than the free space failed to be copied every time while the trash was full. Before the database is written (the backup's snapshot, the secondary's copy of its primary's), the trash now makes room for the whole database. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
38885990f6
commit
a5ab01198b
14 changed files with 345 additions and 50 deletions
|
|
@ -387,6 +387,7 @@ export async function getSettings() {
|
|||
const content = await response.json();
|
||||
return {
|
||||
archive_destination: content.archive_destination,
|
||||
backup_delete_after_days: content.backup_delete_after_days,
|
||||
backup_device_uuid: content.backup_device_uuid,
|
||||
backup_directory: content.backup_directory,
|
||||
backup_excluded_directories: content.backup_excluded_directories,
|
||||
|
|
|
|||
|
|
@ -300,6 +300,34 @@ export function ExcludedDirectories({value, onChange, source, disabled, note}) {
|
|||
</div>
|
||||
}
|
||||
|
||||
/** True if `days` (a number, or the text of the days box) is a whole number of days. */
|
||||
export function validDays(days) {
|
||||
return /^\d+$/.test(String(days ?? '').trim());
|
||||
}
|
||||
|
||||
/**
|
||||
* How long deleted files wait in a trash folder before they are removed; `value` null is "never". An empty box is
|
||||
* not "delete at once": callers refuse to save it (see `validDays`).
|
||||
*/
|
||||
export function DeleteDelay({label, value, onChange, disabled, description}) {
|
||||
const never = value === null;
|
||||
return <div style={{margin: '0.5em 0'}}>
|
||||
<label><b>{label}</b></label>
|
||||
<div style={{display: 'flex', gap: '1em', alignItems: 'center', flexWrap: 'wrap'}}>
|
||||
<span style={{display: 'flex', gap: '0.5em', alignItems: 'center'}}>
|
||||
Delete after
|
||||
<TextInput type='number' min={0} max={3650} style={{width: '6em'}} aria-label={`${label}: days`}
|
||||
value={never ? '' : value} disabled={disabled || never}
|
||||
onChange={e => onChange(e.currentTarget.value)}/>
|
||||
days
|
||||
</span>
|
||||
<Checkbox checked={never} disabled={disabled} label='Never delete them'
|
||||
aria-label={`${label}: never delete them`} onChange={() => onChange(never ? 30 : null)}/>
|
||||
</div>
|
||||
<p style={{color: 'var(--muted)', fontSize: '0.9em'}}>{description}</p>
|
||||
</div>
|
||||
}
|
||||
|
||||
export function SyncSection() {
|
||||
const {settings, fetchSettings} = React.useContext(SettingsContext);
|
||||
const {sync, running, fetchSync} = useSyncStatus();
|
||||
|
|
@ -341,7 +369,7 @@ export function SyncSection() {
|
|||
const save = async () => {
|
||||
// An empty box is not "delete at once": that would empty the trash at the next sync.
|
||||
const days = String(state.sync_delete_after_days ?? '').trim();
|
||||
if (secondary && !neverDelete && !/^\d+$/.test(days)) {
|
||||
if (secondary && !neverDelete && !validDays(days)) {
|
||||
toast({
|
||||
type: 'error', title: 'Sync settings were not saved', time: 5000,
|
||||
description: 'Enter how many days to keep deleted files, or choose to never delete them.',
|
||||
|
|
@ -420,25 +448,11 @@ export function SyncSection() {
|
|||
Within the window, this secondary syncs every 15 minutes. Leave both empty to sync only with
|
||||
"Sync Now".
|
||||
</p>
|
||||
<div style={{margin: '0.5em 0'}}>
|
||||
<label><b>Files the primary deletes</b></label>
|
||||
<div style={{display: 'flex', gap: '1em', alignItems: 'center', flexWrap: 'wrap'}}>
|
||||
<span style={{display: 'flex', gap: '0.5em', alignItems: 'center'}}>
|
||||
Delete after
|
||||
<TextInput type='number' min={0} max={3650} style={{width: '6em'}} aria-label='Days'
|
||||
value={neverDelete ? '' : state.sync_delete_after_days}
|
||||
disabled={wrolMode || neverDelete}
|
||||
onChange={e => setState({...state, sync_delete_after_days: e.currentTarget.value})}/>
|
||||
days
|
||||
</span>
|
||||
<Checkbox checked={neverDelete} disabled={wrolMode} label='Never delete them'
|
||||
onChange={() => setState({...state, sync_delete_after_days: neverDelete ? 30 : null})}/>
|
||||
</div>
|
||||
<p style={{color: 'var(--muted)', fontSize: '0.9em'}}>
|
||||
A file the primary deletes, or the old version of one it changes, waits in a trash folder
|
||||
(<code>.sync-trash</code>) for this long. It is removed sooner if the drive is nearly full.
|
||||
</p>
|
||||
</div>
|
||||
<DeleteDelay label='Files the primary deletes' value={state.sync_delete_after_days} disabled={wrolMode}
|
||||
onChange={value => setState({...state, sync_delete_after_days: value})}
|
||||
description={<>A file the primary deletes, or the old version of one it changes, waits in a
|
||||
trash folder (<code>.sync-trash</code>) for this long. It is removed sooner if the drive
|
||||
is nearly full.</>}/>
|
||||
</>}
|
||||
|
||||
{/* The primary's folders can only be listed once this WROLPi knows its primary. */}
|
||||
|
|
@ -531,6 +545,7 @@ export function BackupDriveSection() {
|
|||
const [windowStart, setWindowStart] = React.useState('');
|
||||
const [windowEnd, setWindowEnd] = React.useState('');
|
||||
const [excluded, setExcluded] = React.useState([]);
|
||||
const [deleteAfterDays, setDeleteAfterDays] = React.useState(30);
|
||||
const [saving, setSaving] = React.useState(false);
|
||||
|
||||
const savedDirectory = settings?.backup_directory || '';
|
||||
|
|
@ -554,6 +569,12 @@ export function BackupDriveSection() {
|
|||
setExcluded(JSON.parse(savedExcluded));
|
||||
}, [savedExcluded]);
|
||||
|
||||
// null: never delete.
|
||||
const savedDeleteAfterDays = settings?.backup_delete_after_days;
|
||||
React.useEffect(() => {
|
||||
setDeleteAfterDays(savedDeleteAfterDays === undefined ? 30 : savedDeleteAfterDays);
|
||||
}, [savedDeleteAfterDays]);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (dockerized === false) {
|
||||
getDisks().then(setDisks).catch(e => setDisksError(e.message));
|
||||
|
|
@ -573,6 +594,13 @@ export function BackupDriveSection() {
|
|||
const running = status?.flags?.backup_running === true;
|
||||
|
||||
const save = async () => {
|
||||
if (drive && deleteAfterDays !== null && !validDays(deleteAfterDays)) {
|
||||
toast({
|
||||
type: 'error', title: 'Backup drive was not saved', time: 5000,
|
||||
description: 'Enter how many days to keep deleted files, or choose to never delete them.',
|
||||
});
|
||||
return;
|
||||
}
|
||||
setSaving(true);
|
||||
try {
|
||||
const chosen = options.find(i => i.value === drive);
|
||||
|
|
@ -583,6 +611,8 @@ export function BackupDriveSection() {
|
|||
backup_window_start: chosen ? windowStart : '',
|
||||
backup_window_end: chosen ? windowEnd : '',
|
||||
backup_excluded_directories: excluded,
|
||||
// The API takes -1 for "never".
|
||||
backup_delete_after_days: deleteAfterDays === null ? -1 : Number(String(deleteAfterDays).trim()),
|
||||
});
|
||||
if (response?.ok) {
|
||||
toast({type: 'success', title: 'Backup drive saved', time: 3000});
|
||||
|
|
@ -605,7 +635,8 @@ export function BackupDriveSection() {
|
|||
<Header as='h3' icon='hdd'>Backup Drive</Header>
|
||||
<p>
|
||||
Copy this WROLPi's media directory to a second drive, so the drive can replace the media drive if it
|
||||
fails. Files deleted here are kept on the backup drive.
|
||||
fails. A file deleted here, or the old version of one that changes, is kept on the backup drive for as
|
||||
long as you choose below.
|
||||
</p>
|
||||
{disksError && <Message kind='error' icon='warning' title='Could not list drives'>{disksError}</Message>}
|
||||
<div style={{margin: '0.5em 0', maxWidth: '30em'}}>
|
||||
|
|
@ -638,6 +669,11 @@ export function BackupDriveSection() {
|
|||
Within the window, the backup drive is updated every hour. Leave both empty to back up only with
|
||||
"Back Up Now".
|
||||
</p>
|
||||
<DeleteDelay label='Files deleted here' value={deleteAfterDays} onChange={setDeleteAfterDays}
|
||||
disabled={wrolMode}
|
||||
description={<>A file deleted from this WROLPi, or the old version of one that changes,
|
||||
waits on the backup drive in a trash folder (<code>.backup-trash</code>) for this long.
|
||||
It is removed sooner if the backup drive is nearly full.</>}/>
|
||||
<ExcludedDirectories value={excluded} onChange={setExcluded} source='local' disabled={wrolMode}/>
|
||||
</>}
|
||||
<div style={{display: 'flex', gap: '0.5em', flexWrap: 'wrap'}}>
|
||||
|
|
|
|||
|
|
@ -168,10 +168,10 @@ describe('SyncSection', () => {
|
|||
|
||||
it('saves how long deleted files are kept, or never to delete them', async () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_delete_after_days: 7})});
|
||||
expect(screen.getByLabelText('Days')).toHaveValue(7);
|
||||
expect(screen.getByLabelText('Files the primary deletes: days')).toHaveValue(7);
|
||||
|
||||
fireEvent.click(screen.getByLabelText('Never delete them'));
|
||||
expect(screen.getByLabelText('Days')).toBeDisabled();
|
||||
fireEvent.click(screen.getByLabelText('Files the primary deletes: never delete them'));
|
||||
expect(screen.getByLabelText('Files the primary deletes: days')).toBeDisabled();
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({sync_delete_after_days: -1})));
|
||||
|
|
@ -179,13 +179,13 @@ describe('SyncSection', () => {
|
|||
|
||||
it('an empty number of days is not saved as "delete at once"', async () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_delete_after_days: 30})});
|
||||
fireEvent.change(screen.getByLabelText('Days'), {target: {value: ''}});
|
||||
fireEvent.change(screen.getByLabelText('Files the primary deletes: days'), {target: {value: ''}});
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: 'error', title: 'Sync settings were not saved'})));
|
||||
expect(saveSyncSettings).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.change(screen.getByLabelText('Days'), {target: {value: '0'}});
|
||||
fireEvent.change(screen.getByLabelText('Files the primary deletes: days'), {target: {value: '0'}});
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({sync_delete_after_days: 0})));
|
||||
|
|
@ -193,7 +193,7 @@ describe('SyncSection', () => {
|
|||
|
||||
it('a secondary that never deletes shows it', () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_delete_after_days: null})});
|
||||
expect(screen.getByLabelText('Never delete them')).toBeChecked();
|
||||
expect(screen.getByLabelText('Files the primary deletes: never delete them')).toBeChecked();
|
||||
});
|
||||
|
||||
it('WROL Mode stops syncing', () => {
|
||||
|
|
@ -221,9 +221,31 @@ describe('BackupDriveSection', () => {
|
|||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
|
||||
backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup',
|
||||
backup_window_start: '', backup_window_end: '', backup_excluded_directories: [],
|
||||
backup_delete_after_days: 30,
|
||||
}));
|
||||
});
|
||||
|
||||
it('saves how long deleted files are kept on the backup drive', async () => {
|
||||
const settings = {settings: settingsFixture({
|
||||
backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup',
|
||||
backup_delete_after_days: 7})};
|
||||
renderWithProviders(<BackupDriveSection/>, {settings});
|
||||
const days = await screen.findByLabelText('Files deleted here: days');
|
||||
expect(days).toHaveValue(7);
|
||||
|
||||
// An empty box is not "delete at once".
|
||||
fireEvent.change(days, {target: {value: ''}});
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: 'error', title: 'Backup drive was not saved'})));
|
||||
expect(saveSyncSettings).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.click(screen.getByLabelText('Files deleted here: never delete them'));
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({backup_delete_after_days: -1})));
|
||||
});
|
||||
|
||||
it('shows the backup and runs it on request', async () => {
|
||||
const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup',
|
||||
backup_device_uuid: 'uuid-backup'})};
|
||||
|
|
@ -299,6 +321,7 @@ describe('BackupDriveSection folder', () => {
|
|||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
|
||||
backup_directory: '/media/wrolpi_backup/wrolpi/library', backup_device_uuid: 'uuid-backup',
|
||||
backup_window_start: '', backup_window_end: '', backup_excluded_directories: [],
|
||||
backup_delete_after_days: 30,
|
||||
}));
|
||||
});
|
||||
|
||||
|
|
@ -334,6 +357,7 @@ describe('BackupDriveSection window', () => {
|
|||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
|
||||
backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup',
|
||||
backup_window_start: '01:00', backup_window_end: '04:30', backup_excluded_directories: [],
|
||||
backup_delete_after_days: 30,
|
||||
}));
|
||||
});
|
||||
});
|
||||
|
|
|
|||
|
|
@ -45,6 +45,7 @@ import {defaultTheme} from './themes/names';
|
|||
/** GET /api/settings */
|
||||
export const settingsFixture = (overrides = {}) => ({
|
||||
archive_destination: 'archive/%(domain_tag)s/%(domain)s',
|
||||
backup_delete_after_days: 30,
|
||||
backup_device_uuid: null,
|
||||
backup_directory: null,
|
||||
backup_excluded_directories: [],
|
||||
|
|
|
|||
|
|
@ -574,8 +574,12 @@ def directory_size(directory: Path) -> int:
|
|||
|
||||
def purge_trash(media_directory: Path, delete_after_days: Optional[int], today: datetime.date = None,
|
||||
min_free_bytes: int = MIN_FREE_BYTES) -> list:
|
||||
"""Remove the trash folders that have waited `delete_after_days` and, while the drive is nearly full, the
|
||||
oldest of the rest. A folder is as old as its name says, whatever its files' times are. Returns their names."""
|
||||
"""Remove the trash folders that have waited `delete_after_days` and, while the drive has less than
|
||||
`min_free_bytes` free, the oldest of the rest. A folder is as old as its name says, whatever its files' times
|
||||
are. Returns their names.
|
||||
|
||||
The newest folder is never removed for space: moving a file to the trash is a rename, which frees nothing, so a
|
||||
nearly full drive would otherwise delete what this very sync moved there at once."""
|
||||
trash = media_directory / TRASH_DIRECTORY
|
||||
if trash.is_symlink() or not trash.is_dir():
|
||||
return []
|
||||
|
|
@ -587,9 +591,11 @@ def purge_trash(media_directory: Path, delete_after_days: Optional[int], today:
|
|||
with contextlib.suppress(ValueError):
|
||||
dated.append((datetime.date.fromisoformat(folder.name), folder))
|
||||
purged = []
|
||||
for day, folder in sorted(dated):
|
||||
dated.sort()
|
||||
newest = dated[-1][1] if dated else None
|
||||
for day, folder in dated:
|
||||
expired = delete_after_days is not None and (today - day).days >= delete_after_days
|
||||
nearly_full = shutil.disk_usage(media_directory).free < min_free_bytes
|
||||
nearly_full = folder != newest and shutil.disk_usage(media_directory).free < min_free_bytes
|
||||
if not expired and not nearly_full:
|
||||
continue
|
||||
logger.info("Removing the sync trash of %s (%s)", folder.name, "expired" if expired else "drive nearly full")
|
||||
|
|
@ -630,7 +636,8 @@ def run_sync(runner=None) -> dict:
|
|||
snapshot = request_snapshot(primary_url)
|
||||
# The trash is what fills the drive, and a full drive fails every run: make room before the copy, not
|
||||
# only after a run that succeeded.
|
||||
purge_trash(media_directory, delete_after_days)
|
||||
# Room for the primary's database, too, before anything is stopped.
|
||||
purge_trash(media_directory, delete_after_days, min_free_bytes=int(snapshot.get("size") or 0) + MIN_FREE_BYTES)
|
||||
if shutil.disk_usage(media_directory).free - int(snapshot.get("size") or 0) < MIN_FREE_BYTES:
|
||||
raise SyncError("Not enough free space for the primary's database")
|
||||
_check_cancel()
|
||||
|
|
|
|||
|
|
@ -849,3 +849,27 @@ class TestRecoveryAndSchedule:
|
|||
monkeypatch.setattr(sync, "update_status", watch)
|
||||
sync.run_sync(FakeRunner(media))
|
||||
assert locked_when_saved == [True]
|
||||
|
||||
|
||||
def test_purge_never_removes_the_newest_for_space(media, monkeypatch):
|
||||
"""Moving a file to the trash frees nothing, so on a nearly full drive the trash this sync just filled would be
|
||||
removed by the same sync. Only age removes the newest folder."""
|
||||
trash = media / ".sync-trash"
|
||||
for name in ("2026-11-17", "2026-11-18"):
|
||||
(trash / name).mkdir(parents=True)
|
||||
usage = namedtuple("usage", "total used free")
|
||||
monkeypatch.setattr(sync.shutil, "disk_usage", lambda _: usage(100, 100, 0))
|
||||
assert sync.purge_trash(media, 30, datetime.date(2026, 11, 18), min_free_bytes=10) == ["2026-11-17"]
|
||||
assert (trash / "2026-11-18").is_dir()
|
||||
assert sync.purge_trash(media, 0, datetime.date(2026, 11, 18), min_free_bytes=10) == ["2026-11-18"]
|
||||
|
||||
|
||||
def test_room_is_made_for_the_primarys_database(media, primary_api, monkeypatch):
|
||||
"""Before the API is stopped, the trash makes room for the primary's database, not only the usual 1 GiB."""
|
||||
purges = []
|
||||
monkeypatch.setattr(sync, "purge_trash",
|
||||
lambda media_directory, days, today=None, min_free_bytes=sync.MIN_FREE_BYTES:
|
||||
purges.append(min_free_bytes))
|
||||
sync.run_sync(FakeRunner(media))
|
||||
# The snapshot of primary_api is 1000 bytes.
|
||||
assert purges[0] == 1000 + sync.MIN_FREE_BYTES
|
||||
|
|
|
|||
|
|
@ -178,6 +178,7 @@ def get_settings(_: Request):
|
|||
settings = {
|
||||
'archive_destination': wrolpi_config.archive_destination,
|
||||
'backup_device_uuid': sync_config.backup_device_uuid,
|
||||
'backup_delete_after_days': sync_config.backup_delete_after_days,
|
||||
'backup_directory': sync_config.backup_directory,
|
||||
'backup_excluded_directories': sync_config.backup_excluded_directories,
|
||||
'backup_window_end': sync_config.backup_window_end,
|
||||
|
|
|
|||
|
|
@ -9,6 +9,8 @@ from wrolpi.errors import InvalidDownload, ValidationError
|
|||
|
||||
@dataclass
|
||||
class SettingsResponse:
|
||||
# None never deletes.
|
||||
backup_delete_after_days: Optional[int]
|
||||
backup_device_uuid: Optional[str]
|
||||
backup_directory: Optional[str]
|
||||
backup_excluded_directories: List[str]
|
||||
|
|
|
|||
|
|
@ -149,9 +149,10 @@ async def put_settings(_: Request, body: schema.SyncSettingsRequest):
|
|||
if value and not re.match(r'^([01]\d|2[0-3]):[0-5]\d$', value):
|
||||
raise InvalidConfig(f'{request_key} must be in HH:MM format')
|
||||
changes[config_key] = value or None
|
||||
if body.sync_delete_after_days is not None:
|
||||
days = int(body.sync_delete_after_days)
|
||||
changes['delete_after_days'] = None if days < 0 else days
|
||||
for request_key, config_key in (('sync_delete_after_days', 'delete_after_days'),
|
||||
('backup_delete_after_days', 'backup_delete_after_days')):
|
||||
if (days := getattr(body, request_key)) is not None:
|
||||
changes[config_key] = None if int(days) < 0 else int(days)
|
||||
for key in ('sync_excluded_directories', 'backup_excluded_directories'):
|
||||
if (value := getattr(body, key)) is not None:
|
||||
changes[key] = lib.normalize_excluded_directories(value)
|
||||
|
|
|
|||
|
|
@ -3,9 +3,11 @@
|
|||
The backup drive mirrors the media directory, so if the media drive dies the backup drive can be mounted as the media
|
||||
directory and WROLPi starts from it:
|
||||
|
||||
1. rsync copies every file that is new or changed (never `--delete`: a file deleted from the media directory stays on
|
||||
the backup drive). The Tags Directory and the playlists directory are not copied; they are hard links that
|
||||
WROLPi rebuilds from the configs when it starts.
|
||||
1. rsync copies every file that is new or changed. A file deleted from the media directory, or the old version of one
|
||||
that changed, is moved to a dated folder in `.backup-trash` on the backup drive and removed after
|
||||
`backup_delete_after_days` (sync.yaml), or sooner when the drive is nearly full; with no number of days nothing
|
||||
is deleted. The Tags Directory and the playlists directory are not copied; they are hard links that WROLPi
|
||||
rebuilds from the configs when it starts.
|
||||
2. The database is not copied by rsync (a live database copied as files can be torn); SQLite's backup API writes a
|
||||
consistent snapshot instead.
|
||||
|
||||
|
|
@ -15,6 +17,7 @@ leaves an empty directory on the system drive, which must never be filled.
|
|||
"""
|
||||
import asyncio
|
||||
import contextlib
|
||||
import datetime
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
|
|
@ -37,6 +40,10 @@ logger = logger.getChild(__name__)
|
|||
|
||||
# Within the sync window, back up again this long after the last backup finished; each backup walks the whole library.
|
||||
BACKUP_REPEAT_SECONDS = 60 * 60
|
||||
# On the backup drive: what the media directory deleted or replaced, in a folder per day.
|
||||
TRASH_DIRECTORY = '.backup-trash'
|
||||
# Never fill the backup drive with the trash; below this, the oldest trash goes first.
|
||||
MIN_FREE_BYTES = 1024 ** 3
|
||||
# rsync keeps an interrupted file here (on the backup drive) so the next backup resumes it.
|
||||
PARTIAL_DIRECTORY = '.rsync-partial'
|
||||
# rsync exit codes: 23 = some files could not be read, 24 = some files vanished while copying.
|
||||
|
|
@ -156,8 +163,9 @@ def rsync_literal(path: str) -> str:
|
|||
|
||||
|
||||
def build_rsync_command(source: pathlib.Path, target: pathlib.Path, playlists_destination: str = None,
|
||||
excluded_directories: List[str] = None) -> List[str]:
|
||||
"""The rsync that copies `source` (the media directory) to `target` (the backup drive), never deleting."""
|
||||
excluded_directories: List[str] = None, trash: pathlib.Path = None) -> List[str]:
|
||||
"""The rsync that copies `source` (the media directory) to `target` (the backup drive). What `source` no longer
|
||||
has, and the old versions of what changed, are moved to `trash`; without one nothing is deleted."""
|
||||
command = list()
|
||||
# The backup should not slow the WROLPi down.
|
||||
if ionice := shutil.which('ionice'):
|
||||
|
|
@ -169,6 +177,12 @@ def build_rsync_command(source: pathlib.Path, target: pathlib.Path, playlists_de
|
|||
'--archive',
|
||||
'--partial', f'--partial-dir={PARTIAL_DIRECTORY}',
|
||||
'--info=progress2,stats2', '--no-human-readable',
|
||||
])
|
||||
if trash is not None:
|
||||
# An excluded path is never deleted: the trash, the drive's database snapshot and rsync's partials stay.
|
||||
command.extend(['--delete', '--backup', f'--backup-dir={trash}'])
|
||||
command.extend([
|
||||
f'--exclude=/{TRASH_DIRECTORY}',
|
||||
# The database is copied as a snapshot (see `snapshot_database`).
|
||||
'--exclude=/config/wrolpi.db', '--exclude=/config/wrolpi.db-*', '--exclude=/config/.wrolpi.db.backup',
|
||||
# The snapshot kept for secondaries (wrolpi.sync.snapshot); the drive has its own.
|
||||
|
|
@ -266,6 +280,12 @@ def backup_skip_reason() -> Optional[str]:
|
|||
return 'WROL Mode is enabled'
|
||||
if not PYTEST and not flags.db_up.is_set():
|
||||
return 'The database is not up'
|
||||
# An unmounted media directory is an empty folder: copying it with --delete would move the whole backup drive to
|
||||
# the trash.
|
||||
if not PYTEST and get_wrolpi_config().require_media_mounted and not flags.media_mounted.is_set():
|
||||
return 'The media directory is not mounted'
|
||||
if not (get_media_directory() / 'config/wrolpi.yaml').is_file():
|
||||
return 'The media directory holds no WROLPi library'
|
||||
if not shutil.which('rsync'):
|
||||
return 'rsync is not installed'
|
||||
if reason := check_backup_target(config.backup_directory, config.backup_device_uuid):
|
||||
|
|
@ -370,8 +390,14 @@ async def run_backup() -> dict:
|
|||
with flags.backup_running:
|
||||
# A folder on the drive is created once the drive has been checked (`backup_skip_reason`).
|
||||
target.mkdir(parents=True, exist_ok=True)
|
||||
delete_after_days = config.backup_delete_after_days
|
||||
# Room first: the trash is what fills a backup drive, and a full drive fails every backup.
|
||||
await asyncio.to_thread(purge_trash, target, delete_after_days)
|
||||
trash = None
|
||||
if delete_after_days is not None:
|
||||
trash = target / TRASH_DIRECTORY / datetime.date.today().isoformat()
|
||||
command = build_rsync_command(get_media_directory(), target, get_wrolpi_config().playlists_destination,
|
||||
config.backup_excluded_directories)
|
||||
config.backup_excluded_directories, trash)
|
||||
code, errors = await _run_rsync(command, target, uuid)
|
||||
if code == RSYNC_PARTIAL:
|
||||
# Unreadable files; everything else was copied.
|
||||
|
|
@ -384,7 +410,11 @@ async def run_backup() -> dict:
|
|||
update_backup_status(phase='database')
|
||||
if reason := check_backup_target(str(target), uuid):
|
||||
raise BackupError(f'Stopped: {reason}')
|
||||
# Room for the database snapshot, which is written beside the drive's old one before it replaces it.
|
||||
room = get_db_file().stat().st_size + MIN_FREE_BYTES
|
||||
await asyncio.to_thread(purge_trash, target, delete_after_days, None, room)
|
||||
await asyncio.to_thread(snapshot_database_if_changed, target)
|
||||
await asyncio.to_thread(purge_trash, target, delete_after_days)
|
||||
except BackupCancelled as e:
|
||||
logger.warning(f'Backup stopped: {e}')
|
||||
return update_backup_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e),
|
||||
|
|
@ -410,6 +440,38 @@ async def run_backup() -> dict:
|
|||
cancel_requested=False)
|
||||
|
||||
|
||||
def purge_trash(target: pathlib.Path, delete_after_days: Optional[int], today: datetime.date = None,
|
||||
min_free_bytes: int = MIN_FREE_BYTES) -> List[str]:
|
||||
"""Remove the trash folders on the backup drive that have waited `delete_after_days` and, while the drive has
|
||||
less than `min_free_bytes` free, the oldest of the rest. A folder is as old as its name says. Returns their
|
||||
names.
|
||||
|
||||
The newest folder is never removed for space: moving a file to the trash is a rename, which frees nothing, so a
|
||||
nearly full drive would otherwise delete what this very backup moved there at once."""
|
||||
trash = target / TRASH_DIRECTORY
|
||||
if trash.is_symlink() or not trash.is_dir():
|
||||
return []
|
||||
today = today or datetime.date.today()
|
||||
dated = []
|
||||
for folder in trash.iterdir():
|
||||
if folder.is_symlink() or not folder.is_dir():
|
||||
continue
|
||||
with contextlib.suppress(ValueError):
|
||||
dated.append((datetime.date.fromisoformat(folder.name), folder))
|
||||
purged = []
|
||||
dated.sort()
|
||||
newest = dated[-1][1] if dated else None
|
||||
for day, folder in dated:
|
||||
expired = delete_after_days is not None and (today - day).days >= delete_after_days
|
||||
nearly_full = folder != newest and shutil.disk_usage(target).free < min_free_bytes
|
||||
if not expired and not nearly_full:
|
||||
continue
|
||||
logger.info(f'Removing the backup trash of {folder.name} ({"expired" if expired else "drive nearly full"})')
|
||||
shutil.rmtree(folder, ignore_errors=True)
|
||||
purged.append(folder.name)
|
||||
return purged
|
||||
|
||||
|
||||
def backup_is_due(status: dict, local_now) -> bool:
|
||||
"""True if the backup window is open and the last backup finished long enough ago."""
|
||||
config = get_sync_config()
|
||||
|
|
|
|||
|
|
@ -32,6 +32,9 @@ class SyncConfigValidator:
|
|||
backup_window_end: str = None
|
||||
# Folders (relative to the media directory) never copied to the backup drive.
|
||||
backup_excluded_directories: list[str] = field(default_factory=list)
|
||||
# Days the backup drive keeps a file deleted (or replaced) in the media directory, in `.backup-trash`; None never
|
||||
# deletes.
|
||||
backup_delete_after_days: int = None
|
||||
# When a backup last finished without errors (ISO 8601); kept so it is known after a restart.
|
||||
backup_last_success_at: str = None
|
||||
|
||||
|
|
@ -55,6 +58,7 @@ class SyncConfig(ConfigFile):
|
|||
backup_window_start=None,
|
||||
backup_window_end=None,
|
||||
backup_excluded_directories=[],
|
||||
backup_delete_after_days=DEFAULT_DELETE_AFTER_DAYS,
|
||||
backup_last_success_at=None,
|
||||
)
|
||||
validator = SyncConfigValidator
|
||||
|
|
@ -112,6 +116,13 @@ class SyncConfig(ConfigFile):
|
|||
def backup_excluded_directories(self) -> List[str]:
|
||||
return list(self._config.get('backup_excluded_directories') or [])
|
||||
|
||||
@property
|
||||
def backup_delete_after_days(self) -> Optional[int]:
|
||||
"""None never deletes. A sync.yaml written before this setting has no key, and gets the default."""
|
||||
if 'backup_delete_after_days' not in self._config:
|
||||
return DEFAULT_DELETE_AFTER_DAYS
|
||||
return self._config['backup_delete_after_days']
|
||||
|
||||
@property
|
||||
def backup_last_success_at(self) -> Optional[str]:
|
||||
return self._config.get('backup_last_success_at')
|
||||
|
|
|
|||
|
|
@ -29,6 +29,8 @@ class SyncSettingsRequest:
|
|||
backup_window_end: Optional[str] = None
|
||||
# Days a secondary keeps a file its primary deleted or replaced; -1 never deletes.
|
||||
sync_delete_after_days: Optional[int] = None
|
||||
# Days the backup drive keeps a file deleted or replaced in the media directory; -1 never deletes.
|
||||
backup_delete_after_days: Optional[int] = None
|
||||
# Folders, relative to the media directory, never copied ([] clears).
|
||||
sync_excluded_directories: Optional[List[str]] = None
|
||||
backup_excluded_directories: Optional[List[str]] = None
|
||||
|
|
@ -36,5 +38,6 @@ class SyncSettingsRequest:
|
|||
def __post_init__(self):
|
||||
if self.role is not None and self.role not in ('primary', 'secondary'):
|
||||
raise ValidationError('role must be "primary" or "secondary"')
|
||||
if self.sync_delete_after_days is not None and not -1 <= int(self.sync_delete_after_days) <= 3650:
|
||||
raise ValidationError('sync_delete_after_days must be between 0 and 3650, or -1 to never delete')
|
||||
for key in ('sync_delete_after_days', 'backup_delete_after_days'):
|
||||
if (value := getattr(self, key)) is not None and not -1 <= int(value) <= 3650:
|
||||
raise ValidationError(f'{key} must be between 0 and 3650, or -1 to never delete')
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import collections
|
||||
import contextlib
|
||||
import os
|
||||
import pathlib
|
||||
|
|
@ -6,7 +7,7 @@ import shutil
|
|||
import sqlite3
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import datetime, timezone
|
||||
from datetime import date, datetime, timezone
|
||||
|
||||
import pytest
|
||||
import yaml
|
||||
|
|
@ -36,6 +37,9 @@ def backup_drive(test_directory, test_wrolpi_config, monkeypatch):
|
|||
drive = pathlib.Path(tmp_dir).resolve()
|
||||
monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: None)
|
||||
get_sync_config()._config.update(backup_directory=str(drive), backup_device_uuid='uuid-1')
|
||||
# A real media directory always has its wrolpi.yaml; a backup refuses one without it.
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
(test_directory / 'config/wrolpi.yaml').write_text('version: 1\n')
|
||||
yield drive
|
||||
|
||||
|
||||
|
|
@ -44,12 +48,19 @@ def test_build_rsync_command(monkeypatch):
|
|||
command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), 'playlists')
|
||||
assert command[:5] == ['/usr/bin/ionice', '-c3', '/usr/bin/nice', '-n', '19']
|
||||
assert command[-2:] == ['/media/wrolpi/', '/media/backup/'], 'trailing slashes copy the contents, not the folder'
|
||||
assert not any(i.startswith('--delete') for i in command), 'a backup never deletes'
|
||||
assert not any(i.startswith('--delete') for i in command), 'without a trash nothing is deleted'
|
||||
assert '-H' not in command and '--hard-links' not in command
|
||||
for exclude in ('/config/wrolpi.db', '/config/wrolpi.db-*', '/tags/', '/playlists/', '.*.partial',
|
||||
'.rsync-partial/', '/lost+found/'):
|
||||
assert f'--exclude={exclude}' in command, exclude
|
||||
|
||||
# With a trash, what the media directory no longer has is moved there; the trash itself is never touched.
|
||||
trash = pathlib.Path('/media/backup/.backup-trash/2026-11-19')
|
||||
command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), 'playlists',
|
||||
trash=trash)
|
||||
assert '--delete' in command and '--backup' in command and f'--backup-dir={trash}' in command
|
||||
assert '--exclude=/.backup-trash' in command
|
||||
|
||||
# ionice and nice are optional.
|
||||
monkeypatch.setattr(backup.shutil, 'which', lambda name: '/usr/bin/rsync' if name == 'rsync' else None)
|
||||
command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), None)
|
||||
|
|
@ -245,7 +256,7 @@ def test_snapshot_database_only_when_changed(async_client, test_directory, tmp_p
|
|||
@pytest.mark.asyncio
|
||||
async def test_run_backup(async_client, test_directory, backup_drive, events_fixture):
|
||||
"""Files and configs are copied with their modification times; the database is a snapshot; hard-link directories
|
||||
are not copied; nothing on the backup drive is deleted; a second backup copies nothing."""
|
||||
are not copied; what the media directory no longer has waits in today's trash; a second backup copies nothing."""
|
||||
files = {
|
||||
'videos/channel/video.mp4': b'video',
|
||||
'videos/channel/video.info.json': b'{}',
|
||||
|
|
@ -261,8 +272,8 @@ async def test_run_backup(async_client, test_directory, backup_drive, events_fix
|
|||
with sqlite3.connect(test_directory / 'config/wrolpi.db') as db:
|
||||
db.execute('CREATE TABLE t (v TEXT)')
|
||||
db.execute("INSERT INTO t VALUES ('row')")
|
||||
# Only on the backup drive: never deleted.
|
||||
(backup_drive / 'old.txt').write_text('deleted from the media directory long ago')
|
||||
# Only on the backup drive: deleted from the media directory since the last backup.
|
||||
(backup_drive / 'old.txt').write_text('deleted from the media directory')
|
||||
|
||||
status = await backup.run_backup()
|
||||
|
||||
|
|
@ -274,16 +285,70 @@ async def test_run_backup(async_client, test_directory, backup_drive, events_fix
|
|||
assert not (backup_drive / 'tags').exists()
|
||||
assert not (backup_drive / 'playlists').exists()
|
||||
assert not (backup_drive / 'videos/channel/.video.mp4.partial').exists()
|
||||
assert (backup_drive / 'old.txt').exists()
|
||||
assert not (backup_drive / 'old.txt').exists()
|
||||
trash = backup_drive / '.backup-trash' / date.today().isoformat()
|
||||
assert (trash / 'old.txt').read_text() == 'deleted from the media directory'
|
||||
with sqlite3.connect(backup_drive / 'config/wrolpi.db') as copy:
|
||||
assert copy.execute('SELECT v FROM t').fetchall() == [('row',)]
|
||||
assert status['files_transferred'] == 3
|
||||
# The three files above, and wrolpi.yaml.
|
||||
assert status['files_transferred'] == 4
|
||||
assert status['cancel_requested'] is False
|
||||
events_fixture.assert_has_event('backup_completed')
|
||||
|
||||
status = await backup.run_backup()
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert status['files_transferred'] == 0
|
||||
assert (trash / 'old.txt').exists(), 'the trash is not itself deleted by the next backup'
|
||||
assert (backup_drive / 'config/wrolpi.db').is_file(), "the drive's database snapshot is kept"
|
||||
|
||||
|
||||
@requires_rsync
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_backup_never_deletes(async_client, test_directory, backup_drive):
|
||||
"""With no number of days, a backup deletes nothing (as backups did before the trash)."""
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
with sqlite3.connect(test_directory / 'config/wrolpi.db') as db:
|
||||
db.execute('CREATE TABLE t (v TEXT)')
|
||||
(backup_drive / 'old.txt').write_text('deleted from the media directory')
|
||||
get_sync_config()._config['backup_delete_after_days'] = None
|
||||
|
||||
status = await backup.run_backup()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
assert (backup_drive / 'old.txt').exists()
|
||||
assert not (backup_drive / '.backup-trash').exists()
|
||||
|
||||
|
||||
def test_purge_trash(tmp_path, monkeypatch):
|
||||
"""A trash folder is as old as its name says; while the drive is nearly full the oldest goes first; a link is
|
||||
never followed."""
|
||||
trash = tmp_path / '.backup-trash'
|
||||
for name in ('2026-10-18', '2026-10-19', '2026-10-20', '2026-11-18', 'not a date'):
|
||||
(trash / name).mkdir(parents=True)
|
||||
# 31 and 30 days old: expired. 29 days: kept.
|
||||
today = date(2026, 11, 18)
|
||||
assert backup.purge_trash(tmp_path, 30, today, min_free_bytes=0) == ['2026-10-18', '2026-10-19']
|
||||
assert backup.purge_trash(tmp_path, None, today, min_free_bytes=0) == [], 'never'
|
||||
|
||||
usage = collections.namedtuple('usage', 'total used free')
|
||||
monkeypatch.setattr(backup.shutil, 'disk_usage',
|
||||
lambda _: usage(100, 0, 5 if (trash / '2026-10-20').exists() else 50))
|
||||
assert backup.purge_trash(tmp_path, 30, today, min_free_bytes=10) == ['2026-10-20']
|
||||
monkeypatch.undo()
|
||||
|
||||
elsewhere = tmp_path / 'elsewhere'
|
||||
(elsewhere / '2020-01-01').mkdir(parents=True)
|
||||
(trash / '2020-01-02').symlink_to(elsewhere / '2020-01-01')
|
||||
assert backup.purge_trash(tmp_path, 30, today, min_free_bytes=0) == []
|
||||
assert (elsewhere / '2020-01-01').is_dir()
|
||||
|
||||
|
||||
def test_backup_refuses_an_empty_media_directory(async_client, test_directory, backup_drive):
|
||||
"""An unmounted media directory is an empty folder: backing it up with deletes would move the whole backup
|
||||
drive to the trash."""
|
||||
assert backup.backup_skip_reason() is None
|
||||
(test_directory / 'config/wrolpi.yaml').unlink()
|
||||
assert backup.backup_skip_reason() == 'The media directory holds no WROLPi library'
|
||||
|
||||
|
||||
@requires_rsync
|
||||
|
|
@ -462,3 +527,43 @@ async def test_failed_backup_keeps_last_success(async_client, test_directory, te
|
|||
|
||||
assert status['state'] == 'error'
|
||||
assert get_sync_config().backup_last_success_at == '2026-11-01T02:00:00+00:00'
|
||||
|
||||
|
||||
def test_purge_never_removes_the_newest_for_space(tmp_path, monkeypatch):
|
||||
"""Moving a file to the trash frees nothing, so on a nearly full drive the trash this backup just filled would be
|
||||
removed by the same backup. Only age removes the newest folder."""
|
||||
trash = tmp_path / '.backup-trash'
|
||||
for name in ('2026-11-17', '2026-11-18'):
|
||||
(trash / name).mkdir(parents=True)
|
||||
usage = collections.namedtuple('usage', 'total used free')
|
||||
monkeypatch.setattr(backup.shutil, 'disk_usage', lambda _: usage(100, 100, 0))
|
||||
assert backup.purge_trash(tmp_path, 30, date(2026, 11, 18), min_free_bytes=10) == ['2026-11-17']
|
||||
assert (trash / '2026-11-18').is_dir()
|
||||
# 0 days is "delete at once", whatever the space.
|
||||
assert backup.purge_trash(tmp_path, 0, date(2026, 11, 18), min_free_bytes=10) == ['2026-11-18']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_room_is_made_for_the_database_snapshot(async_client, test_directory, backup_drive, monkeypatch):
|
||||
"""The snapshot is written beside the drive's old one: before it, the trash makes room for the whole database,
|
||||
not only the usual 1 GiB."""
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
with sqlite3.connect(test_directory / 'config/wrolpi.db') as db:
|
||||
db.execute('CREATE TABLE t (v TEXT)')
|
||||
events = []
|
||||
|
||||
async def run_rsync(command, target, uuid):
|
||||
events.append('rsync')
|
||||
return 0, []
|
||||
|
||||
monkeypatch.setattr(backup, '_run_rsync', run_rsync)
|
||||
monkeypatch.setattr(backup, 'purge_trash', lambda target, days, today=None, min_free_bytes=backup.MIN_FREE_BYTES:
|
||||
events.append(('purge', min_free_bytes)))
|
||||
monkeypatch.setattr(backup, 'snapshot_database_if_changed', lambda target: events.append('snapshot'))
|
||||
|
||||
status = await backup.run_backup()
|
||||
|
||||
assert status['state'] == 'idle', status['last_error']
|
||||
database = (test_directory / 'config/wrolpi.db').stat().st_size
|
||||
assert events == [('purge', backup.MIN_FREE_BYTES), 'rsync', ('purge', database + backup.MIN_FREE_BYTES),
|
||||
'snapshot', ('purge', backup.MIN_FREE_BYTES)]
|
||||
|
|
|
|||
|
|
@ -187,3 +187,20 @@ def test_apply_primary_wrolpi_config(async_client, test_directory, test_wrolpi_c
|
|||
|
||||
(sync_directory / 'primary-wrolpi.yaml').write_text('not: [a config')
|
||||
assert lib.apply_primary_wrolpi_config() == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_backup_delete_after_days_setting(async_client, test_session, test_directory, test_wrolpi_config,
|
||||
await_switches):
|
||||
"""How long the backup drive keeps what the media directory deleted: 30 days unless set, -1 for never."""
|
||||
request, response = await async_client.get('/api/settings')
|
||||
assert response.json['backup_delete_after_days'] == 30
|
||||
|
||||
for sent, kept in ((7, 7), (-1, None)):
|
||||
request, response = await async_client.put('/api/sync/settings',
|
||||
content=json.dumps(dict(backup_delete_after_days=sent)))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
assert get_sync_config().backup_delete_after_days == kept
|
||||
|
||||
get_sync_config()._config.pop('backup_delete_after_days', None)
|
||||
assert get_sync_config().backup_delete_after_days == 30, 'a sync.yaml from before the setting gets the default'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue