}
+ {folders && folders.map(folder => )}
+
+
+
+
+
+
+}
+
+/** The folders a sync or a backup does not copy, and the button that chooses them. */
+export function ExcludedDirectories({value, onChange, source, disabled}) {
+ const [open, setOpen] = React.useState(false);
+ const excluded = value || [];
+ return
+
+ {excluded.length === 0
+ ?
Nothing excluded; every folder is copied.
+ :
+ {excluded.map(path =>
+ {path}
+ onChange(excluded.filter(i => i !== path))}/>
+
)}
+
}
+
+
+ Files already copied into an excluded folder are kept; delete them yourself to free space.
+
+ A secondary WROLPi keeps a copy of a primary WROLPi's library: its files, tags,
+ channels, domains, playlists, and other library settings. Files deleted on the primary are kept.
+
+
+
+
+
+ {secondary && <>
+
+ Every sync replaces this WROLPi's library with the primary's; changes made here are lost.
+ It does not download, and it is unavailable while it syncs.
+
+
+ Within the window, this secondary syncs every 15 minutes. Leave both empty to sync only with
+ "Sync Now".
+
+ >}
+
+ {/* The primary's folders can only be listed once this WROLPi knows its primary. */}
+ {secondary && saved && setState({
+ ...state,
+ sync_excluded_directories: value
+ })}/>}
+
+
+
+ {saved && }
+ {saved && }
+
+
+ {testResult && (testResult.ok
+ ?
+ WROLPi {testResult.info?.version} at {settings.sync_primary_url}
+
+ : {testResult.error})}
+
+ {wrolMode && settings.role === 'secondary' &&
+ WROL Mode is enabled; this WROLPi does not sync.
+ }
+
+ {settings.role === 'secondary' && }
+
+}
+
+
+/** The drives a backup can be written to: mounted under /media, and not the media drive. */
+export function backupDriveOptions(disks, mediaDirectory) {
+ const media = mediaDirectory || '/media/wrolpi';
+ return (disks || [])
+ .filter(d => d.mountpoint && d.uuid && d.mountpoint.startsWith('/media/')
+ && d.mountpoint !== media && d.mountpoint !== '/media/wrolpi_temp_onboarding')
+ .map(d => ({value: d.mountpoint, uuid: d.uuid, label: `${d.label || d.name} (${d.size}) at ${d.mountpoint}`}));
+}
+
+/** Split a saved backup directory into the drive it is on (its option) and the folder on that drive. */
+export function splitBackupDirectory(directory, uuid, options) {
+ if (!directory) {
+ return {drive: '', folder: ''};
+ }
+ const onDrive = options.filter(i => directory === i.value || directory.startsWith(`${i.value}/`));
+ // The drive with the saved UUID, else the deepest mount point that holds the directory.
+ const drive = onDrive.find(i => i.uuid === uuid)
+ || onDrive.sort((a, b) => b.value.length - a.value.length)[0];
+ if (!drive) {
+ return {drive: directory, folder: ''};
+ }
+ return {drive: drive.value, folder: directory.slice(drive.value.length).replace(/^\/+/, '')};
+}
+
+/** The progress and outcome of copying the media directory to the backup drive. */
+export function BackupStatus({backup, running}) {
+ if (!backup) {
+ return null;
+ }
+ return
+ Copy this WROLPi's media directory to a second drive, so the drive can replace the media drive if it
+ fails. Files deleted here are kept on the backup drive.
+
+ {disksError && {disksError}}
+
+
+
+ {drive &&
+
+ setFolder(e.currentTarget.value)}/>
+
+ The media directory is copied into this folder; leave it empty to use the whole drive.
+
+
}
+ {drive && <>
+
+
+
+ setWindowStart(e.currentTarget.value)}/>
+
+
+
+ setWindowEnd(e.currentTarget.value)}/>
+
+
+
+ Within the window, the backup drive is updated every hour. Leave both empty to back up only with
+ "Back Up Now".
+
+
+ >}
+
+
+ {savedDirectory && !running && }
+ {/* The database snapshot runs to the end; a half-written database is worse than waiting for it. */}
+ {running && }
+
+ {savedDirectory && }
+
+}
+
+/** The Sync tab of /admin. */
+export function SyncPage() {
+ useTitle('Sync');
+ return
+
+
+
+
+}
diff --git a/app/src/components/admin/Sync.test.js b/app/src/components/admin/Sync.test.js
new file mode 100644
index 00000000..627e1c54
--- /dev/null
+++ b/app/src/components/admin/Sync.test.js
@@ -0,0 +1,343 @@
+import React from 'react';
+import {fireEvent, screen, waitFor} from '@testing-library/react';
+import {renderWithProviders} from '../../test-utils';
+import {backupStatusFixture, settingsFixture, statusFixture, syncStatusFixture} from '../../test-fixtures';
+import {
+ BackupDriveSection, backupDriveOptions, ExcludedDirectories, isExcludedPath, SecondaryRoleMessage, simplifyExcluded,
+ splitBackupDirectory, SyncingOverlay, SyncSection,
+} from './Sync';
+
+jest.mock('../../api/controller', () => ({
+ getDisks: jest.fn(),
+}));
+
+jest.mock('../../api', () => ({
+ cancelBackup: jest.fn(),
+ runBackup: jest.fn(),
+ cancelSync: jest.fn(),
+ runSync: jest.fn(),
+ saveSyncSettings: jest.fn(),
+ testSync: jest.fn(),
+ getSyncDirectories: jest.fn(),
+ PrimaryUnreachableError: class PrimaryUnreachableError extends Error {
+ },
+}));
+
+import {
+ cancelBackup, cancelSync, getSyncDirectories, PrimaryUnreachableError, runBackup, runSync, saveSyncSettings, testSync,
+} from '../../api';
+import {getDisks} from '../../api/controller';
+
+beforeEach(() => {
+ // react-scripts resets mock implementations before each test.
+ cancelSync.mockResolvedValue({ok: true});
+ runSync.mockResolvedValue({ok: true});
+ saveSyncSettings.mockResolvedValue({ok: true});
+ testSync.mockResolvedValue({ok: true, info: {version: '0.30.5-beta'}});
+ runBackup.mockResolvedValue({ok: true});
+ cancelBackup.mockResolvedValue({ok: true});
+ getDisks.mockResolvedValue(disks);
+ getSyncDirectories.mockImplementation(async path => folderTree[path] || []);
+});
+
+const folderTree = {
+ '': [{name: 'Movies', path: 'Movies'}, {name: 'videos', path: 'videos'}],
+ 'videos': [{name: 'channel', path: 'videos/channel'}],
+ 'videos/channel': [],
+};
+
+const disks = [
+ {name: 'sda1', size: '9.1T', mountpoint: '/media/wrolpi_backup', label: 'Decay2of2', uuid: 'uuid-backup'},
+ {name: 'sdb1', size: '9.1T', mountpoint: '/media/wrolpi', label: 'Decay1of2', uuid: 'uuid-media'},
+ {name: 'mmcblk0p2', size: '118.6G', mountpoint: '/', label: 'rootfs', uuid: 'uuid-root'},
+ {name: 'sdc1', size: '64G', mountpoint: null, label: 'stick', uuid: 'uuid-stick'},
+];
+
+const secondary = (overrides = {}) => ({
+ settings: settingsFixture({role: 'secondary', sync_primary_url: 'https://192.168.0.10', ...overrides}),
+});
+
+describe('SecondaryRoleMessage', () => {
+ it('warns on a secondary', () => {
+ renderWithProviders(, {settings: secondary()});
+ expect(screen.getByText('Secondary WROLPi')).toBeInTheDocument();
+ expect(screen.getByText('https://192.168.0.10')).toBeInTheDocument();
+ });
+
+ it('is absent on a primary', () => {
+ renderWithProviders();
+ expect(screen.queryByText('Secondary WROLPi')).not.toBeInTheDocument();
+ });
+
+ it('goes away in WROL Mode, which stops syncing', () => {
+ renderWithProviders(, {settings: secondary({wrol_mode: true})});
+ expect(screen.queryByText('Secondary WROLPi')).not.toBeInTheDocument();
+ });
+});
+
+describe('SyncingOverlay', () => {
+ const syncing = (sync = {}) => ({
+ status: statusFixture({
+ flags: {sync_running: true},
+ sync: syncStatusFixture({state: 'running', phase: 'files', groups_seen: 50, max_id: 100, ...sync}),
+ }),
+ });
+
+ it('covers the app while a secondary syncs', () => {
+ renderWithProviders(, {settings: secondary(), status: syncing()});
+ expect(screen.getByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
+ expect(screen.getByText('Syncing from the primary')).toBeInTheDocument();
+
+ fireEvent.click(screen.getByText('Stop Sync'));
+ expect(cancelSync).toHaveBeenCalled();
+ });
+
+ it('is absent when not syncing', () => {
+ renderWithProviders(, {settings: secondary()});
+ expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
+ });
+
+ it('is absent on a primary', () => {
+ renderWithProviders(, {status: syncing()});
+ expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
+ });
+});
+
+describe('SyncSection', () => {
+ it('a primary shows only its role', () => {
+ // Even with a backup drive: the backup has its own window, in its own panel.
+ renderWithProviders(, {settings: {settings: settingsFixture({
+ backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup'})}});
+ expect(screen.getByText('This WROLPi is')).toBeInTheDocument();
+ expect(screen.queryByText('Primary address')).not.toBeInTheDocument();
+ expect(screen.queryByText('Sync Window Start')).not.toBeInTheDocument();
+ expect(screen.queryByText('Sync Now')).not.toBeInTheDocument();
+ });
+
+ it('a secondary shows its primary, window, and sync controls', async () => {
+ const status = {status: statusFixture({sync: syncStatusFixture({last_error: 'The primary did not answer'})})};
+ renderWithProviders(, {settings: secondary({sync_window_start: '01:00'}), status});
+ expect(screen.getByDisplayValue('https://192.168.0.10')).toBeInTheDocument();
+ expect(screen.getByDisplayValue('01:00')).toBeInTheDocument();
+ expect(screen.getByText('The primary did not answer')).toBeInTheDocument();
+
+ fireEvent.click(screen.getByText('Sync Now'));
+ await waitFor(() => expect(runSync).toHaveBeenCalled());
+
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
+ role: 'secondary', sync_primary_url: 'https://192.168.0.10', sync_window_start: '01:00',
+ sync_window_end: '', sync_excluded_directories: [],
+ }));
+ });
+
+ it('WROL Mode stops syncing', () => {
+ renderWithProviders(, {settings: secondary({wrol_mode: true})});
+ expect(screen.getByText('Syncing is stopped')).toBeInTheDocument();
+ expect(screen.getByText('Sync Now').closest('button')).toBeDisabled();
+ });
+});
+
+describe('backupDriveOptions', () => {
+ it('lists mounted drives under /media that are not the media drive', () => {
+ expect(backupDriveOptions(disks, '/media/wrolpi')).toEqual([
+ {value: '/media/wrolpi_backup', uuid: 'uuid-backup', label: 'Decay2of2 (9.1T) at /media/wrolpi_backup'},
+ ]);
+ });
+});
+
+describe('BackupDriveSection', () => {
+ it('saves the chosen drive with its UUID', async () => {
+ renderWithProviders();
+ await waitFor(() => expect(getDisks).toHaveBeenCalled());
+ fireEvent.click(screen.getByDisplayValue('None'));
+ fireEvent.click(await screen.findByText('Decay2of2 (9.1T) at /media/wrolpi_backup'));
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
+ backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup',
+ backup_window_start: '', backup_window_end: '', backup_excluded_directories: [],
+ }));
+ });
+
+ it('shows the backup and runs it on request', async () => {
+ const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup',
+ backup_device_uuid: 'uuid-backup'})};
+ const status = {status: statusFixture({backup: backupStatusFixture({last_error: 'rsync failed (12)'})})};
+ renderWithProviders(, {settings, status});
+ expect(await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup')).toBeInTheDocument();
+ expect(screen.getByText('rsync failed (12)')).toBeInTheDocument();
+ fireEvent.click(screen.getByText('Back Up Now'));
+ await waitFor(() => expect(runBackup).toHaveBeenCalled());
+ });
+
+ it('keeps an unplugged drive selected', async () => {
+ getDisks.mockResolvedValue([]);
+ const settings = {settings: settingsFixture({backup_directory: '/media/gone', backup_device_uuid: 'x'})};
+ renderWithProviders(, {settings});
+ expect(await screen.findByDisplayValue('/media/gone (not mounted)')).toBeInTheDocument();
+ });
+
+ it('offers to stop a running backup', async () => {
+ const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup',
+ backup_device_uuid: 'uuid-backup'})};
+ const status = {status: statusFixture({flags: {backup_running: true},
+ backup: backupStatusFixture({state: 'running', phase: 'files', percent: 40})})};
+ renderWithProviders(, {settings, status});
+ fireEvent.click(await screen.findByText('Stop Backup'));
+ expect(cancelBackup).toHaveBeenCalled();
+ });
+
+ it('is absent in Docker, which cannot reach other drives', () => {
+ renderWithProviders(, {status: {status: statusFixture({dockerized: true})}});
+ expect(screen.queryByText('Backup Drive')).not.toBeInTheDocument();
+ expect(getDisks).not.toHaveBeenCalled();
+ });
+});
+
+describe('splitBackupDirectory', () => {
+ const options = [
+ {value: '/media/backup', uuid: 'uuid-backup'},
+ {value: '/media/backup/nested', uuid: 'uuid-nested'},
+ ];
+
+ it('finds the drive and the folder on it', () => {
+ expect(splitBackupDirectory('/media/backup', 'uuid-backup', options))
+ .toEqual({drive: '/media/backup', folder: ''});
+ expect(splitBackupDirectory('/media/backup/wrolpi/library', 'uuid-backup', options))
+ .toEqual({drive: '/media/backup', folder: 'wrolpi/library'});
+ });
+
+ it('prefers the drive with the saved UUID', () => {
+ expect(splitBackupDirectory('/media/backup/nested/wrolpi', 'uuid-nested', options))
+ .toEqual({drive: '/media/backup/nested', folder: 'wrolpi'});
+ expect(splitBackupDirectory('/media/backup/nested/wrolpi', 'uuid-backup', options))
+ .toEqual({drive: '/media/backup', folder: 'nested/wrolpi'});
+ });
+
+ it('does not mistake a similarly named drive', () => {
+ expect(splitBackupDirectory('/media/backup2/wrolpi', 'x', options)).toEqual({drive: '/media/backup2/wrolpi', folder: ''});
+ });
+
+ it('is empty without a backup', () => {
+ expect(splitBackupDirectory('', '', options)).toEqual({drive: '', folder: ''});
+ });
+});
+
+describe('BackupDriveSection folder', () => {
+ it('saves a folder on the chosen drive', async () => {
+ renderWithProviders();
+ await waitFor(() => expect(getDisks).toHaveBeenCalled());
+ fireEvent.click(screen.getByDisplayValue('None'));
+ fireEvent.click(await screen.findByText('Decay2of2 (9.1T) at /media/wrolpi_backup'));
+ fireEvent.change(screen.getByPlaceholderText('The whole drive'), {target: {value: '/wrolpi/library/'}});
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
+ backup_directory: '/media/wrolpi_backup/wrolpi/library', backup_device_uuid: 'uuid-backup',
+ backup_window_start: '', backup_window_end: '', backup_excluded_directories: [],
+ }));
+ });
+
+ it('shows the saved folder', async () => {
+ const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup/wrolpi',
+ backup_device_uuid: 'uuid-backup'})};
+ renderWithProviders(, {settings});
+ expect(await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup')).toBeInTheDocument();
+ expect(screen.getByPlaceholderText('The whole drive')).toHaveValue('wrolpi');
+ });
+});
+
+describe('BackupDriveSection database phase', () => {
+ it('cannot be stopped while the database is copied', async () => {
+ const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup',
+ backup_device_uuid: 'uuid-backup'})};
+ const status = {status: statusFixture({flags: {backup_running: true},
+ backup: backupStatusFixture({state: 'running', phase: 'database'})})};
+ renderWithProviders(, {settings, status});
+ expect((await screen.findByText('Stop Backup')).closest('button')).toBeDisabled();
+ });
+});
+
+describe('BackupDriveSection window', () => {
+ it('saves its own window with the drive', async () => {
+ const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup',
+ backup_device_uuid: 'uuid-backup', backup_window_start: '01:00', backup_window_end: '05:00'})};
+ renderWithProviders(, {settings});
+ await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup');
+ expect(screen.getByLabelText('Backup Window Start')).toHaveValue('01:00');
+ fireEvent.change(screen.getByLabelText('Backup Window End'), {target: {value: '04:30'}});
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
+ backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup',
+ backup_window_start: '01:00', backup_window_end: '04:30', backup_excluded_directories: [],
+ }));
+ });
+});
+
+
+describe('excluded folders', () => {
+ it('isExcludedPath matches a folder and what is inside it, not a similar name', () => {
+ expect(isExcludedPath('Movies', ['Movies'])).toBe(true);
+ expect(isExcludedPath('Movies/a', ['Movies'])).toBe(true);
+ expect(isExcludedPath('Movies2', ['Movies'])).toBe(false);
+ });
+
+ it('simplifyExcluded drops folders inside another', () => {
+ expect(simplifyExcluded(['videos/channel', 'Movies', 'videos', 'Movies'])).toEqual(['Movies', 'videos']);
+ });
+
+ it('lists the excluded folders and removes one', () => {
+ const onChange = jest.fn();
+ renderWithProviders();
+ expect(screen.getByText('Movies')).toBeInTheDocument();
+ fireEvent.click(screen.getByLabelText('Copy Movies again'));
+ expect(onChange).toHaveBeenCalledWith(['videos/channel']);
+ });
+
+ it('chooses folders in the tree; a folder inside a checked folder is excluded with it', async () => {
+ const onChange = jest.fn();
+ renderWithProviders();
+ expect(screen.getByText('Nothing excluded; every folder is copied.')).toBeInTheDocument();
+ fireEvent.click(screen.getByText('Choose Folders'));
+ await screen.findByLabelText('videos');
+ expect(getSyncDirectories).toHaveBeenCalledWith('', 'primary');
+
+ fireEvent.click(screen.getByLabelText('Open videos'));
+ const child = await screen.findByLabelText('videos/channel');
+ expect(getSyncDirectories).toHaveBeenCalledWith('videos', 'primary');
+ fireEvent.click(child);
+ fireEvent.click(screen.getByLabelText('videos'));
+ await waitFor(() => expect(screen.getByLabelText('videos/channel')).toBeDisabled());
+ expect(screen.getByLabelText('videos/channel')).toBeChecked();
+ expect(screen.getByText('excluded with its parent')).toBeInTheDocument();
+
+ fireEvent.click(screen.getByText('Done'));
+ expect(onChange).toHaveBeenCalledWith(['videos']);
+ });
+
+ it('says when the primary does not answer', async () => {
+ getSyncDirectories.mockRejectedValue(new PrimaryUnreachableError('connection refused'));
+ renderWithProviders();
+ fireEvent.click(screen.getByText('Choose Folders'));
+ expect(await screen.findByText('The primary did not answer: connection refused')).toBeInTheDocument();
+ });
+
+ it('a secondary saves its excluded folders', async () => {
+ renderWithProviders(, {settings: secondary({sync_excluded_directories: ['Movies']})});
+ expect(screen.getByText('Movies')).toBeInTheDocument();
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
+ expect.objectContaining({sync_excluded_directories: ['Movies']})));
+ });
+
+ it('the backup drive saves its own excluded folders', async () => {
+ const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup',
+ backup_device_uuid: 'uuid-backup', backup_excluded_directories: ['Movies']})};
+ renderWithProviders(, {settings});
+ await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup');
+ fireEvent.click(screen.getByLabelText('Copy Movies again'));
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
+ expect.objectContaining({backup_excluded_directories: []})));
+ });
+});
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index a5beeeb5..612d3e13 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -2119,6 +2119,12 @@ export const useWROLMode = () => {
return settings ? settings.wrol_mode : null;
}
+export const useSyncRole = () => {
+ // This WROLPi's role ('primary' or 'secondary', see Sync); during fetch this returns null.
+ const {settings} = useContext(SettingsContext);
+ return settings ? settings.role || 'primary' : null;
+}
+
export const useDockerized = () => {
const {status} = React.useContext(StatusContext);
return status && status['dockerized'] === true;
diff --git a/app/src/test-fixtures.js b/app/src/test-fixtures.js
index 7746d375..d988878d 100644
--- a/app/src/test-fixtures.js
+++ b/app/src/test-fixtures.js
@@ -45,6 +45,11 @@ import {defaultTheme} from './themes/names';
/** GET /api/settings */
export const settingsFixture = (overrides = {}) => ({
archive_destination: 'archive/%(domain_tag)s/%(domain)s',
+ backup_device_uuid: null,
+ backup_directory: null,
+ backup_excluded_directories: [],
+ backup_window_end: null,
+ backup_window_start: null,
check_for_upgrades: true,
download_daily_limit_global: null,
download_daily_limit_per_domain: null,
@@ -74,7 +79,12 @@ export const settingsFixture = (overrides = {}) => ({
repos_destination: 'repos/%(repo_tag)s/%(repo_name)s',
require_cookies_unlocked: false,
require_media_mounted: false,
+ role: 'primary',
save_ffprobe_json: true,
+ sync_excluded_directories: [],
+ sync_primary_url: null,
+ sync_window_end: null,
+ sync_window_start: null,
tags_directory: true,
throttle_on_startup: false,
timezone: null,
@@ -85,6 +95,52 @@ export const settingsFixture = (overrides = {}) => ({
...overrides,
});
+/** The `sync` of GET /api/status: a secondary that has never synced. */
+export const syncStatusFixture = (overrides = {}) => ({
+ state: 'idle',
+ phase: null,
+ requested: false,
+ force: false,
+ cancel_requested: false,
+ started_at: null,
+ finished_at: null,
+ last_success_at: null,
+ last_error: null,
+ last_skip_reason: null,
+ primary_url: null,
+ primary_version: null,
+ groups_seen: 0,
+ max_id: 0,
+ files_checked: 0,
+ files_to_download: 0,
+ files_downloaded: 0,
+ bytes_downloaded: 0,
+ files_failed: 0,
+ files_excluded: 0,
+ configs_imported: [],
+ disk_full: false,
+ ...overrides,
+});
+
+/** The `backup` of GET /api/status: a backup drive that has never been written. */
+export const backupStatusFixture = (overrides = {}) => ({
+ state: 'idle',
+ phase: null,
+ requested: false,
+ cancel_requested: false,
+ started_at: null,
+ finished_at: null,
+ last_success_at: null,
+ last_error: null,
+ last_skip_reason: null,
+ directory: null,
+ bytes_transferred: 0,
+ percent: 0,
+ files_transferred: 0,
+ files_to_check: 0,
+ ...overrides,
+});
+
/**
* GET /api/status
*
@@ -122,6 +178,10 @@ export const statusFixture = (overrides = {}) => ({
memory_stats: {total: 4000000000, used: 1000000000, free: 3000000000, cached: 500000000},
processes_stats: [],
sanic_workers: {},
+ // A secondary's sync (wrolpi.sync); idle on a primary.
+ sync: syncStatusFixture(),
+ // A backup to a second drive (wrolpi.sync.backup); idle without one.
+ backup: backupStatusFixture(),
version: '1.0.0',
wrol_mode: false,
// Upgrade info, seeded by the API and refreshed by its upgrade check.
diff --git a/docker/web/Caddyfile b/docker/web/Caddyfile
index 1d745032..368d518c 100644
--- a/docker/web/Caddyfile
+++ b/docker/web/Caddyfile
@@ -8,6 +8,12 @@
:443 {
tls /etc/ssl/caddy/cert.crt /etc/ssl/caddy/cert.key
+ # Access log: `docker compose logs -f web`.
+ log {
+ output stdout
+ format console
+ }
+
header Access-Control-Allow-Origin "*"
header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS"
header Access-Control-Allow-Headers "Content-Type, Accept"
diff --git a/main.py b/main.py
index 91748324..1778af70 100755
--- a/main.py
+++ b/main.py
@@ -14,7 +14,7 @@ from wrolpi import root_api # noqa
from wrolpi.api_utils import api_app, perpetual_signal, per_worker_task
from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \
cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, have_internet_connection, wrol_mode_enabled, \
- create_empty_config_files, import_file_configs, TRACE_LEVEL
+ create_empty_config_files, import_file_configs, TRACE_LEVEL, is_secondary
from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts
from wrolpi.dates import Seconds
from modules.videos.cookies import lock_cookies
@@ -286,7 +286,8 @@ async def start_single_tasks(app: Sanic):
# Hotspot/throttle on startup are now handled by the Controller service.
- if get_wrolpi_config().download_on_startup and not wrol_mode_enabled():
+ # A secondary never downloads; its library comes from its primary (see wrolpi.sync).
+ if get_wrolpi_config().download_on_startup and not wrol_mode_enabled() and not is_secondary():
# Only start downloading when prerequisites have been met.
try:
async with flags.have_internet.wait_for(timeout=30):
diff --git a/scripts/vulture_baseline.txt b/scripts/vulture_baseline.txt
index b8fd5403..ba2fd89f 100644
--- a/scripts/vulture_baseline.txt
+++ b/scripts/vulture_baseline.txt
@@ -39,6 +39,7 @@ wrolpi/media_path.py: unused method 'process_result_value'
wrolpi/media_path.py: unused variable 'PATH_TYPE'
wrolpi/media_path.py: unused variable 'dialect'
wrolpi/media_path.py: unused variable 'dialect'
+wrolpi/sync/conftest.py: unused method 'log_message'
wrolpi/tags.py: unused function 'get_name_by_id_key'
wrolpi/test/common.py: unused class 'PytestCase'
wrolpi/test/common.py: unused method 'assertDictContains'
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 45bc7db8..321b5902 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -490,6 +490,24 @@ def library_on_internal_disk(media_directory: Path = None) -> bool:
DB_CONFIG_FILE_NAMES ={'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml',
'playlists.yaml', 'repos.yaml'}
+# The library configs a secondary WROLPi copies from its primary (see wrolpi.sync). A secondary never saves these;
+# the primary's copy is the only one that changes them. repos.yaml is absent because git clones are never synced.
+SYNCED_CONFIG_FILE_NAMES = frozenset({
+ 'archives_downloader.yaml',
+ 'bookmarks.yaml',
+ 'channels.yaml',
+ 'domains.yaml',
+ 'download_manager.yaml',
+ 'inventory/catalog.yaml',
+ 'map_pins.yaml',
+ 'playlists.yaml',
+ 'tags.yaml',
+ 'videos_downloader.yaml',
+ 'zims.yaml',
+})
+# MultiFileConfig names (directories of configs) a secondary copies from its primary.
+SYNCED_MULTIFILE_CONFIG_NAMES = frozenset({'inventory'})
+
class _DatabaseChanged(Exception):
"""The database changed a config's rows between an import's snapshot and its write."""
@@ -733,6 +751,11 @@ class ConfigFile:
from wrolpi.api_utils import api_app
from wrolpi.events import Events
+ if self.file_name in SYNCED_CONFIG_FILE_NAMES and is_secondary():
+ # The primary's copy of this config is the only one that changes; the next sync replaces this one.
+ logger_.debug(f'Not saving {self.file_name} on a secondary WROLPi')
+ return
+
file = file or self.get_file()
# Compare basenames so a nested file_name (e.g. 'inventory/catalog.yaml') is accepted.
if pathlib.Path(self.file_name).name != file.name:
@@ -1004,6 +1027,8 @@ class ConfigFile:
from wrolpi.db import get_db_session
with get_db_session() as session:
db_version = get_config_version(session, self.file_name)
+ # A secondary's synced configs are the primary's; its database never outranks them (see wrolpi.sync).
+ forced = forced or (self.file_name in SYNCED_CONFIG_FILE_NAMES and is_secondary())
decision = IMPORT if forced else decide(file_version, db_version)
logger_.debug(f'{self.file_name}: file version {file_version}, database version {db_version}: {decision}')
return decision, db_version
@@ -1281,6 +1306,10 @@ class MultiFileConfig:
"""Write one entity's in-memory config to its YAML file (version bump + daily backup).
The caller MUST already hold `save_lock()`."""
+ if self.name in SYNCED_MULTIFILE_CONFIG_NAMES and is_secondary():
+ # The primary's copy is the only one that changes; the next sync replaces this one.
+ logger_.debug(f'Not saving {self.name}/{slug} on a secondary WROLPi')
+ return
file = self.get_file(slug)
if PYTEST and not is_tempfile(file):
raise ValueError(f'Refusing to save config file while testing: {file}')
@@ -1490,6 +1519,10 @@ def get_all_configs() -> Dict[str, ConfigFile]:
if bookmarks_config := get_bookmarks_config():
all_configs[bookmarks_config.file_name] = bookmarks_config
+ from wrolpi.sync.config import get_sync_config
+ if sync_config := get_sync_config():
+ all_configs[sync_config.file_name] = sync_config
+
from wrolpi.collections.config import get_playlists_config
if playlists_config := get_playlists_config():
all_configs[playlists_config.file_name] = playlists_config
@@ -1527,6 +1560,14 @@ async def import_file_configs():
except Exception as e:
logger.warning(f'Failed to import wrolpi config: {e}')
+ # Whether this WROLPi is a secondary decides how the other configs import (see `ConfigFile.version_decision`).
+ from wrolpi.sync.config import get_sync_config
+ try:
+ await asyncio.to_thread(get_sync_config().import_config)
+ logger.debug('sync config imported')
+ except Exception as e:
+ logger.warning(f'Failed to import sync config: {e}')
+
if not wrolpi_config.successful_import:
return
if wrolpi_config.wrol_mode:
@@ -2095,6 +2136,13 @@ def wrol_mode_enabled() -> bool:
return get_wrolpi_config().wrol_mode
+def is_secondary() -> bool:
+ """Return True if this WROLPi copies its library from a primary WROLPi (see wrolpi.sync)."""
+ # wrolpi.sync.config imports this module.
+ from wrolpi.sync.config import get_sync_config
+ return get_sync_config().role == 'secondary'
+
+
def wrol_mode_check(func):
"""Wraps a function so that it cannot be called when WROL Mode is enabled."""
@@ -2120,6 +2168,11 @@ def enable_wrol_mode():
get_wrolpi_config().wrol_mode = True
from wrolpi.downloader import download_manager
download_manager.stop()
+ # A secondary in WROL Mode is in use; a running sync stops between pages. A backup stops too.
+ from wrolpi.sync.lib import request_sync_cancel
+ from wrolpi.sync.backup import request_backup_cancel
+ request_sync_cancel()
+ request_backup_cancel()
async def disable_wrol_mode():
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index be9eba13..f569f6e4 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -52,6 +52,7 @@ from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.models import Directory, FileGroup
from wrolpi.files.worker import file_worker
from wrolpi.switches import await_switches as await_switches_
+from wrolpi.sync.config import get_sync_config
from wrolpi.tags import Tag, upsert_tag
from wrolpi.vars import PROJECT_DIR
@@ -236,6 +237,15 @@ def test_directory() -> Generator[Path, Any, None]:
yield tmp_path
+@pytest.fixture(autouse=True)
+def test_sync_config():
+ """Every test starts as a primary with no sync settings (sync.yaml is in memory unless a test saves it)."""
+ config = get_sync_config()
+ config._config.update(copy.deepcopy(config.default_config))
+ yield
+ config._config.update(copy.deepcopy(config.default_config))
+
+
@pytest.fixture
def test_wrolpi_config(test_directory) -> Generator[Path, Any, None]:
"""
diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py
index 9fb35bf8..75715cdb 100644
--- a/wrolpi/contexts.py
+++ b/wrolpi/contexts.py
@@ -5,6 +5,7 @@ import queue
from sanic import Sanic
from wrolpi.common import logger
+from wrolpi.sync.status import default_sync_status, default_backup_status
from wrolpi.vars import LOG_LEVEL_INT
logger = logger.getChild(__name__)
@@ -41,6 +42,7 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.map_pins_config = manager.dict()
app.shared_ctx.flasher_config = manager.dict()
app.shared_ctx.bookmarks_config = manager.dict()
+ app.shared_ctx.sync_config = manager.dict()
app.shared_ctx.recently_viewed_config = manager.dict()
# Shared dicts.
app.shared_ctx.uploaded_files = manager.dict()
@@ -65,6 +67,9 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.download_manager_stopped.set()
app.shared_ctx.single_tasks_started = multiprocessing.Event()
+ # Set while a secondary syncs (wrolpi.sync); the API refuses requests. An Event so the request middleware can
+ # check it without a manager round trip.
+ app.shared_ctx.sync_api_locked = multiprocessing.Event()
app.shared_ctx.flags_initialized = multiprocessing.Event()
# Transcode
@@ -152,6 +157,8 @@ def reset_shared_contexts(app: Sanic):
current_commit=None,
commits_behind=0,
git_branch=None,
+ sync=default_sync_status(),
+ backup=default_backup_status(),
))
app.shared_ctx.cache.clear()
# Secure cookies
@@ -240,6 +247,7 @@ def reset_shared_contexts(app: Sanic):
# Clear events_history in place to preserve references held by fixtures
del app.shared_ctx.events_history[:]
app.shared_ctx.single_tasks_started.clear()
+ app.shared_ctx.sync_api_locked.clear()
app.shared_ctx.flags_initialized.clear()
if hasattr(app.shared_ctx, 'flag_holders'):
app.shared_ctx.flag_holders.clear()
@@ -351,6 +359,12 @@ def initialize_configs_contexts(app: Sanic):
except Exception as e:
logger.error(f'Failed to initialize in-memory bookmarks config: {e}')
+ try:
+ from wrolpi.sync.config import SYNC_CONFIG
+ SYNC_CONFIG.initialize(app.shared_ctx.sync_config)
+ except Exception as e:
+ logger.error(f'Failed to initialize in-memory sync config: {e}')
+
try:
from wrolpi.files.progress import RECENTLY_VIEWED_CONFIG
RECENTLY_VIEWED_CONFIG.initialize(app.shared_ctx.recently_viewed_config)
diff --git a/wrolpi/dates.py b/wrolpi/dates.py
index 539ecab8..713373a3 100644
--- a/wrolpi/dates.py
+++ b/wrolpi/dates.py
@@ -1,5 +1,5 @@
import logging
-from datetime import datetime, timezone
+from datetime import datetime, timezone, time
from enum import Enum
from typing import Union, List
@@ -216,3 +216,13 @@ def date_range_to_where(wheres: list, params: dict, from_year: int, to_year: int
params['to_year'] = to_year
return wheres, params
+
+
+def is_within_daily_window(start: str | None, end: str | None, current: time) -> bool:
+ """True if `current` is within the daily window from `start` to `end` ('HH:MM'). A window whose end is before
+ its start crosses midnight (e.g. 22:00-06:00). Callers decide what a missing window means."""
+ start_time = datetime.strptime(start, '%H:%M').time()
+ end_time = datetime.strptime(end, '%H:%M').time()
+ if start_time <= end_time:
+ return start_time <= current < end_time
+ return current >= start_time or current < end_time
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index 13b65aca..573c9769 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -33,10 +33,10 @@ from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which, run_command, CommandResult
from wrolpi.common import loaded, Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \
wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \
- get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain
+ get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain, is_secondary
from wrolpi.config_import import ConfigSection, Unreadable, chunks
from wrolpi.config_version import collection_kinds, count_config_change, register_ownership
-from wrolpi.dates import TZDateTime, now, Seconds
+from wrolpi.dates import TZDateTime, now, Seconds, is_within_daily_window
from wrolpi.db import get_db_session, session_curs, serializer
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \
ValidationError, DownloadError
@@ -817,14 +817,8 @@ class DownloadManager:
return True
current = self._get_local_time()
- start_time = datetime.strptime(start, "%H:%M").time()
- end_time = datetime.strptime(end, "%H:%M").time()
-
- if start_time <= end_time:
- within = start_time <= current < end_time
- else:
- # Overnight window (e.g., 22:00-06:00)
- within = current >= start_time or current < end_time
+ # Overnight windows (e.g., 22:00-06:00) are supported.
+ within = is_within_daily_window(start, end, current)
# Diagnostic: log only when the window opens/closes, including the computed local time and
# timezone, so a clock/timezone mismatch (e.g. system UTC-5 vs configured America/Denver)
@@ -845,6 +839,9 @@ class DownloadManager:
Single source of truth for `can_download`; returning a reason string (rather than a bare
bool) lets the dispatcher log *why* a cycle was skipped."""
+ if is_secondary():
+ # A secondary's library comes from its primary (see wrolpi.sync).
+ return 'secondary role'
if PYTEST:
return None
if self.is_disabled:
diff --git a/wrolpi/errors.py b/wrolpi/errors.py
index 3f2a0062..c2852ad6 100644
--- a/wrolpi/errors.py
+++ b/wrolpi/errors.py
@@ -45,6 +45,18 @@ class WROLModeEnabled(APIError):
status_code = HTTPStatus.FORBIDDEN
+class SyncInProgress(APIError):
+ code = 'SYNCING'
+ summary = 'This WROLPi is copying its library from its primary. Try again when the sync finishes.'
+ status_code = HTTPStatus.SERVICE_UNAVAILABLE
+
+
+class SecondaryRole(APIError):
+ code = 'SECONDARY_ROLE'
+ summary = 'This method is disabled on a secondary WROLPi.'
+ status_code = HTTPStatus.FORBIDDEN
+
+
class InvalidDownload(APIError):
code = 'INVALID_DOWNLOAD'
summary = 'The URL cannot be downloaded.'
diff --git a/wrolpi/events.py b/wrolpi/events.py
index 5a88bcd5..d4e52d29 100644
--- a/wrolpi/events.py
+++ b/wrolpi/events.py
@@ -40,6 +40,22 @@ class Events:
def send_downloads_disabled(message: str = None):
send_event('downloads_disabled', message, subject='downloads')
+ @staticmethod
+ def send_backup_completed(message: str):
+ send_event('backup_completed', message, subject='backup')
+
+ @staticmethod
+ def send_backup_failed(message: str):
+ send_event('backup_failed', message, subject='backup')
+
+ @staticmethod
+ def send_sync_completed(message: str):
+ send_event('sync_completed', message, subject='sync')
+
+ @staticmethod
+ def send_sync_failed(message: str):
+ send_event('sync_failed', message, subject='sync')
+
@staticmethod
def send_user_notify(message: str, url: str = None):
send_event('user_notify_message', message, subject='user_notify', url=url)
diff --git a/wrolpi/flags.py b/wrolpi/flags.py
index 139970a7..88f3c2a5 100644
--- a/wrolpi/flags.py
+++ b/wrolpi/flags.py
@@ -161,6 +161,11 @@ map_search_building = Flag('map_search_building')
# The global refresh has been performed. This is False on a fresh instance of WROLPi.
refresh_complete = Flag('refresh_complete', store_db=True)
+# A secondary WROLPi is copying its library from its primary (see wrolpi.sync). Its API refuses requests meanwhile.
+sync_running = Flag('sync_running')
+# This WROLPi is copying its media directory to its backup drive (see wrolpi.sync.backup).
+backup_running = Flag('backup_running')
+
# Cookies flags for yt-dlp.
cookies_exist = Flag('cookies_exist')
cookies_unlocked = Flag('cookies_unlocked')
@@ -171,6 +176,7 @@ def get_flags() -> dict:
flags = dict(
cookies_exist=cookies_exist.is_set(),
cookies_unlocked=cookies_unlocked.is_set(),
+ backup_running=backup_running.is_set(),
db_up=db_up.is_set(),
file_worker_busy=file_worker_busy.is_set(),
file_worker_cleanup=file_worker_cleanup.is_set(),
@@ -184,6 +190,7 @@ def get_flags() -> dict:
media_mounted=media_mounted.is_set(),
outdated_zims=outdated_zims.is_set(),
refresh_complete=refresh_complete.is_set(),
+ sync_running=sync_running.is_set(),
)
return flags
diff --git a/wrolpi/perpetual.py b/wrolpi/perpetual.py
index afa76abb..80c40d17 100644
--- a/wrolpi/perpetual.py
+++ b/wrolpi/perpetual.py
@@ -55,7 +55,8 @@ SHARED_LOCK_HEAL_TIMEOUT = 10
# when the recorded holder pid is dead, because a live worker may be inside the block right now.
# file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and
# process_queue already drops a stale set on its next idle tick.
-PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active')
+PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active', 'sync_running',
+ 'backup_running')
SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup',
'map_search_building')
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index a9ccb412..c778e8f5 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -28,18 +28,20 @@ from wrolpi.bookmarks_api import bookmarks_bp
from wrolpi.collections.api import collection_bp
from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \
- set_global_log_level, get_relative_to_media_directory, search_other_estimates, set_system_timezone
+ set_global_log_level, get_relative_to_media_directory, search_other_estimates, set_system_timezone, is_secondary
from wrolpi.config_api import config_bp
from wrolpi.db_query import db_query_bp
from wrolpi.dates import now
from wrolpi.downloader import download_manager
-from wrolpi.errors import WROLModeEnabled, InvalidConfig, ValidationError
+from wrolpi.errors import WROLModeEnabled, InvalidConfig, ValidationError, SecondaryRole
from wrolpi.events import get_events, Events
from wrolpi.files import files_bp
from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count
from wrolpi.jobs import jobs_bp
from wrolpi.log_levels import int_to_name
from wrolpi.speedtest import speedtest_bp
+from wrolpi.sync.api import sync_bp
+from wrolpi.sync.config import get_sync_config
from wrolpi.tags import Tag
from wrolpi.vars import DOCKERIZED, IS_RPI, IS_RPI4, IS_RPI5, API_HOST, API_PORT, API_WORKERS, API_DEBUG, \
API_ACCESS_LOG, truthy_arg, API_AUTO_RELOAD
@@ -69,6 +71,7 @@ api_app.blueprint(jobs_bp)
api_app.blueprint(map_bp)
api_app.blueprint(repos_bp)
api_app.blueprint(speedtest_bp)
+api_app.blueprint(sync_bp)
api_app.blueprint(videos_bp)
api_app.blueprint(zim_bp)
@@ -164,6 +167,7 @@ async def echo(request: Request):
)
def get_settings(_: Request):
wrolpi_config = get_wrolpi_config()
+ sync_config = get_sync_config()
# Drop special directories (e.g. zims) that must never be ignored, if present from older configs.
from wrolpi.files.lib import sanitize_ignored_directories
@@ -173,6 +177,11 @@ def get_settings(_: Request):
settings = {
'archive_destination': wrolpi_config.archive_destination,
+ 'backup_device_uuid': sync_config.backup_device_uuid,
+ 'backup_directory': sync_config.backup_directory,
+ 'backup_excluded_directories': sync_config.backup_excluded_directories,
+ 'backup_window_end': sync_config.backup_window_end,
+ 'backup_window_start': sync_config.backup_window_start,
'download_manager_disabled': download_manager.is_disabled,
'download_manager_stopped': download_manager.is_stopped,
'download_daily_limit_global': wrolpi_config.download_daily_limit_global,
@@ -205,7 +214,12 @@ def get_settings(_: Request):
'zims_destination': wrolpi_config.zims_destination,
'require_cookies_unlocked': wrolpi_config.require_cookies_unlocked,
'require_media_mounted': wrolpi_config.require_media_mounted,
+ 'role': sync_config.role,
'save_ffprobe_json': wrolpi_config.save_ffprobe_json,
+ 'sync_primary_url': sync_config.primary_url,
+ 'sync_excluded_directories': sync_config.sync_excluded_directories,
+ 'sync_window_end': sync_config.window_end,
+ 'sync_window_start': sync_config.window_start,
'timezone': wrolpi_config.timezone,
}
return json_response(settings)
@@ -494,6 +508,8 @@ async def kill_downloads(_: Request):
@api_bp.post('/download/enable')
@openapi.description('Enable and start downloading.')
async def enable_downloads(_: Request):
+ if is_secondary():
+ raise SecondaryRole('Downloads are disabled on a secondary WROLPi; its primary downloads.')
await download_manager.enable()
return response.empty()
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index a8019cb0..03b74d0f 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -9,6 +9,11 @@ from wrolpi.errors import InvalidDownload, ValidationError
@dataclass
class SettingsResponse:
+ backup_device_uuid: Optional[str]
+ backup_directory: Optional[str]
+ backup_excluded_directories: List[str]
+ backup_window_end: Optional[str]
+ backup_window_start: Optional[str]
download_manager_disabled: bool
download_manager_stopped: bool
download_daily_limit_global: Optional[int]
@@ -38,6 +43,11 @@ class SettingsResponse:
nav_color: str
media_directory: str
require_cookies_unlocked: bool
+ role: str
+ sync_primary_url: Optional[str]
+ sync_excluded_directories: List[str]
+ sync_window_end: Optional[str]
+ sync_window_start: Optional[str]
require_media_mounted: bool
save_ffprobe_json: bool
tags_directory: bool
@@ -116,6 +126,7 @@ class SettingsRequest:
@dataclass
class FlagsStatusResponse:
+ backup_running: bool
db_up: bool
file_worker_busy: bool
file_worker_cleanup: bool
@@ -128,6 +139,7 @@ class FlagsStatusResponse:
media_mounted: bool
outdated_zims: bool
refresh_complete: bool
+ sync_running: bool
@dataclass
@@ -187,6 +199,52 @@ class DownloadsSummaryResponse:
daily_limit_reached: bool
+@dataclass
+class SyncStatusResponse:
+ """The progress of a secondary WROLPi copying its library from its primary (see wrolpi.sync)."""
+ state: str # 'idle', 'running', or 'error'
+ phase: Optional[str] # 'files', 'configs', or None
+ requested: bool
+ force: bool
+ cancel_requested: bool
+ started_at: Optional[str]
+ finished_at: Optional[str]
+ last_success_at: Optional[str]
+ last_error: Optional[str]
+ last_skip_reason: Optional[str]
+ primary_url: Optional[str]
+ primary_version: Optional[str]
+ groups_seen: int
+ max_id: int
+ files_checked: int
+ files_to_download: int
+ files_downloaded: int
+ bytes_downloaded: int
+ files_failed: int
+ files_excluded: int
+ configs_imported: List[str]
+ disk_full: bool
+
+
+@dataclass
+class BackupStatusResponse:
+ """The progress of copying this WROLPi's media directory to its backup drive (see wrolpi.sync.backup)."""
+ state: str # 'idle', 'running', or 'error'
+ phase: Optional[str] # 'files', 'database', or None
+ requested: bool
+ cancel_requested: bool
+ started_at: Optional[str]
+ finished_at: Optional[str]
+ last_success_at: Optional[str]
+ last_error: Optional[str]
+ last_skip_reason: Optional[str]
+ directory: Optional[str]
+ bytes_transferred: int
+ percent: int
+ files_transferred: int
+ files_to_check: int
+
+
@dataclass
class StatusResponse:
"""The response of GET /api/status.
@@ -201,6 +259,7 @@ class StatusResponse:
`disk_bandwidth_stats` and `nic_bandwidth_stats` were declared here after the handler had
stopped returning them; hotspot and throttle moved to the Controller.
"""
+ backup: BackupStatusResponse
cpu_stats: CPUStatusResponse
dockerized: bool
# Declared `int` for a long time while the handler sent the summary object, so the published
@@ -216,6 +275,7 @@ class StatusResponse:
memory_stats: MemoryStatusStat
processes_stats: List[ProcessStatusStat]
sanic_workers: dict
+ sync: SyncStatusResponse
version: str
# `str` here was wrong too; wrol_mode_enabled() returns a bool.
wrol_mode: bool
diff --git a/wrolpi/sync/__init__.py b/wrolpi/sync/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/wrolpi/sync/api.py b/wrolpi/sync/api.py
new file mode 100644
index 00000000..9e2fc371
--- /dev/null
+++ b/wrolpi/sync/api.py
@@ -0,0 +1,205 @@
+import asyncio
+import os
+import re
+from http import HTTPStatus
+
+from sanic import Request, Blueprint, response
+from sanic_ext import validate
+from sanic_ext.extensions.openapi import openapi
+
+from wrolpi import flags
+from wrolpi.api_utils import api_app, json_response, perpetual_signal
+from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled
+from wrolpi.db import get_db_session
+from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled
+from wrolpi.sync import backup, lib, schema
+from wrolpi.sync.config import get_sync_config
+from wrolpi.sync.status import get_backup_status, get_sync_status
+from wrolpi.vars import DOCKERIZED
+
+sync_bp = Blueprint('Sync', '/api/sync')
+
+logger = logger.getChild(__name__)
+
+# Requests a secondary answers while it syncs: enough for the UI to show the sync's progress, and to cancel it.
+SYNC_ALLOWED_REQUESTS = frozenset({
+ ('GET', '/api/echo'),
+ ('GET', '/api/events/feed'),
+ ('GET', '/api/settings'),
+ ('GET', '/api/status'),
+ ('GET', '/api/sync/info'),
+ ('GET', '/api/sync/status'),
+ ('POST', '/api/sync/cancel'),
+})
+
+
+@api_app.middleware('request')
+async def check_sync_api_lock(request: Request):
+ """A secondary refuses requests while it replaces its library with its primary's (see wrolpi.sync)."""
+ # The Event costs nothing to check; the Flag confirms it (the Event survives a perpetual process that died).
+ if not api_app.shared_ctx.sync_api_locked.is_set() or not flags.sync_running.is_set():
+ return
+ if request.method == 'OPTIONS' or (request.method, request.path.rstrip('/')) in SYNC_ALLOWED_REQUESTS:
+ return
+ # Answered here rather than raised: the UI polls while it waits, and each refusal would be logged as an error.
+ return json_response(dict(error=SyncInProgress.summary, code=SyncInProgress.code),
+ SyncInProgress.status_code)
+
+
+@perpetual_signal(sleep=5)
+async def perpetual_sync_worker():
+ # One loop, so a secondary's sync and its backup never run at once; the backup copies what the sync pulled.
+ await lib.sync_tick()
+ await backup.backup_tick()
+
+
+@sync_bp.get('/info')
+@openapi.definition(summary='This WROLPi, as a primary that secondaries sync from')
+async def get_info(_: Request):
+ def info():
+ with get_db_session() as session:
+ return lib.get_sync_info(session)
+
+ return json_response(await asyncio.to_thread(info))
+
+
+@sync_bp.get('/manifest')
+@openapi.definition(summary="One page of this WROLPi's files, for a secondary to compare with its own")
+@validate(query=schema.ManifestRequest)
+async def get_manifest(_: Request, query: schema.ManifestRequest):
+ def page():
+ with get_db_session() as session:
+ return lib.build_manifest_page(session, query.after_id, query.limit)
+
+ return json_response(await asyncio.to_thread(page))
+
+
+@sync_bp.get('/status')
+@openapi.definition(summary="The progress of this secondary's sync")
+async def get_status(_: Request):
+ return json_response(dict(sync=get_sync_status(), sync_running=bool(flags.sync_running.is_set()),
+ backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set())))
+
+
+@sync_bp.post('/run')
+@openapi.definition(summary='Sync from the primary now', body=schema.SyncRunRequest, validate=True)
+@wrol_mode_check
+async def post_run(_: Request, body: schema.SyncRunRequest):
+ if not is_secondary():
+ raise SecondaryRole('Only a secondary WROLPi syncs from a primary')
+ if flags.sync_running.is_set():
+ raise SyncInProgress()
+ if not get_sync_config().primary_url:
+ raise InvalidConfig('No primary is configured')
+ lib.request_sync(force=body.force)
+ return response.empty(HTTPStatus.ACCEPTED)
+
+
+@sync_bp.post('/cancel')
+@openapi.definition(summary='Stop the running sync')
+async def post_cancel(_: Request):
+ lib.request_sync_cancel()
+ return response.empty(HTTPStatus.ACCEPTED)
+
+
+@sync_bp.get('/test')
+@openapi.definition(summary='Check that the configured primary answers')
+async def get_test(_: Request):
+ url = get_sync_config().primary_url
+ if not url:
+ raise InvalidConfig('No primary is configured')
+ try:
+ async with lib.PrimaryClient(url) as client:
+ info = await asyncio.wait_for(client.get_info(), timeout=15)
+ except Exception as e:
+ logger.warning(f'Primary {url} did not answer', exc_info=e)
+ return json_response(dict(ok=False, error=str(e) or type(e).__name__))
+ ok = info.get('role') == 'primary'
+ return json_response(dict(ok=ok, info=info, error=None if ok else f'{url} is not a primary WROLPi'))
+
+
+@sync_bp.post('/directories')
+@openapi.definition(summary='List the folders inside a folder, to choose which are not copied',
+ body=schema.SyncDirectoriesRequest, validate=True)
+async def post_directories(_: Request, body: schema.SyncDirectoriesRequest):
+ if body.source == 'local':
+ return json_response(dict(directories=lib.list_local_directories(body.path)))
+ if not is_secondary() or not get_sync_config().primary_url:
+ raise InvalidConfig('Only a secondary with a primary can list its primary\'s folders')
+ try:
+ directories = await asyncio.wait_for(lib.list_primary_directories(body.path), timeout=15)
+ except InvalidConfig:
+ raise
+ except Exception as e:
+ logger.warning(f'Could not list the primary\'s folder {body.path!r}', exc_info=e)
+ return json_response(dict(error=str(e) or type(e).__name__), status=HTTPStatus.BAD_GATEWAY)
+ return json_response(dict(directories=directories))
+
+
+@sync_bp.put('/settings')
+@openapi.definition(summary="Set this WROLPi's role and its primary", body=schema.SyncSettingsRequest, validate=True)
+async def put_settings(_: Request, body: schema.SyncSettingsRequest):
+ if wrol_mode_enabled():
+ raise WROLModeEnabled()
+ changes = dict()
+ if body.role is not None:
+ changes['role'] = body.role
+ if body.sync_primary_url is not None:
+ try:
+ changes['primary_url'] = lib.normalize_primary_url(body.sync_primary_url) \
+ if body.sync_primary_url else None
+ except ValueError as e:
+ raise InvalidConfig(str(e))
+ for request_key, config_key in (('sync_window_start', 'window_start'), ('sync_window_end', 'window_end'),
+ ('backup_window_start', 'backup_window_start'),
+ ('backup_window_end', 'backup_window_end')):
+ value = getattr(body, request_key)
+ if value is None:
+ continue
+ if value and not re.match(r'^([01]\d|2[0-3]):[0-5]\d$', value):
+ raise InvalidConfig(f'{request_key} must be in HH:MM format')
+ changes[config_key] = value or None
+ for key in ('sync_excluded_directories', 'backup_excluded_directories'):
+ if (value := getattr(body, key)) is not None:
+ changes[key] = lib.normalize_excluded_directories(value)
+ if body.backup_directory is not None or body.backup_device_uuid is not None:
+ changes.update(_backup_drive_changes(body.backup_directory, body.backup_device_uuid))
+ if not changes:
+ raise InvalidConfig('No settings to change')
+
+ get_sync_config().update(changes)
+ if changes.get('role') == 'secondary':
+ # A secondary's library comes from its primary.
+ from wrolpi.downloader import download_manager
+ download_manager.disable()
+ return response.empty()
+
+
+def _backup_drive_changes(directory: str, uuid: str) -> dict:
+ if not directory and not uuid:
+ # No backup drive.
+ return dict(backup_directory=None, backup_device_uuid=None)
+ if DOCKERIZED:
+ raise InvalidConfig('Backup drives are not supported in Docker')
+ if reason := backup.check_backup_target(directory, uuid):
+ raise InvalidConfig(reason)
+ return dict(backup_directory=os.path.normpath(directory), backup_device_uuid=uuid)
+
+
+@sync_bp.post('/backup/run')
+@openapi.definition(summary='Copy the media directory to the backup drive now')
+@wrol_mode_check
+async def post_backup_run(_: Request):
+ if not get_sync_config().backup_directory:
+ raise InvalidConfig('No backup drive is configured')
+ if flags.backup_running.is_set():
+ raise InvalidConfig('A backup is already running')
+ backup.request_backup()
+ return response.empty(HTTPStatus.ACCEPTED)
+
+
+@sync_bp.post('/backup/cancel')
+@openapi.definition(summary='Stop the running backup')
+async def post_backup_cancel(_: Request):
+ backup.request_backup_cancel()
+ return response.empty(HTTPStatus.ACCEPTED)
diff --git a/wrolpi/sync/backup.py b/wrolpi/sync/backup.py
new file mode 100644
index 00000000..1b6a66aa
--- /dev/null
+++ b/wrolpi/sync/backup.py
@@ -0,0 +1,430 @@
+"""Copy this WROLPi's media directory to a second drive (its "backup drive").
+
+The backup drive mirrors the media directory, so if the media drive dies the backup drive can be mounted as the media
+directory and WROLPi starts from it:
+
+1. rsync copies every file that is new or changed (never `--delete`: a file deleted from the media directory stays on
+ the backup drive). The Tags Directory and the playlists directory are not copied; they are hard links that
+ WROLPi rebuilds from the configs when it starts.
+2. The database is not copied by rsync (a live database copied as files can be torn); SQLite's backup API writes a
+ consistent snapshot instead.
+
+A backup only reads the media directory, so the API stays usable while it runs. It runs in the sync window, or when
+the user asks. Nothing is written unless the configured drive is mounted where it was chosen: an unplugged drive
+leaves an empty directory on the system drive, which must never be filled.
+"""
+import asyncio
+import contextlib
+import os
+import pathlib
+import re
+import shutil
+import sqlite3
+import time
+from typing import List, Optional, Tuple
+
+from wrolpi import flags
+from wrolpi.common import logger, get_media_directory, get_wrolpi_config, wrol_mode_enabled
+from wrolpi.dates import now
+from wrolpi.db import get_db_file
+from wrolpi.events import Events
+from wrolpi.sync.config import get_sync_config
+from wrolpi.sync.lib import window_is_due, PROGRESS_INTERVAL_SECONDS
+from wrolpi.sync.status import get_backup_status, update_backup_status
+from wrolpi.vars import DOCKERIZED, PYTEST
+
+logger = logger.getChild(__name__)
+
+# Within the sync window, back up again this long after the last backup finished; each backup walks the whole library.
+BACKUP_REPEAT_SECONDS = 60 * 60
+# rsync keeps an interrupted file here (on the backup drive) so the next backup resumes it.
+PARTIAL_DIRECTORY = '.rsync-partial'
+# rsync exit codes: 23 = some files could not be read, 24 = some files vanished while copying.
+RSYNC_VANISHED = 24
+RSYNC_PARTIAL = 23
+# How often a running backup checks that its drive is still mounted.
+MOUNT_CHECK_SECONDS = 5.0
+PROC_MOUNTS = pathlib.Path('/proc/mounts')
+# Backup drives are mounted in this directory (the Controller mounts drives here).
+MOUNT_ROOT = pathlib.Path('/media')
+# Mount points (in MOUNT_ROOT) that are never a backup drive.
+RESERVED_MOUNT_NAMES = frozenset({'wrolpi_temp_onboarding'})
+
+# `--info=progress2` lines: " 1234567 45% 10.5MB/s 0:01:23 (xfr#12, to-chk=100/2000)"
+PROGRESS_LINE = re.compile(r'^\s*([\d,]+)\s+(\d+)%\s+\S+\s+\S+(?:\s+\(xfr#(\d+), (?:ir|to)-chk=(\d+)/(\d+)\))?')
+FILES_TRANSFERRED_LINE = re.compile(r'^Number of regular files transferred: ([\d,]+)')
+
+
+class BackupError(Exception):
+ pass
+
+
+class BackupCancelled(BackupError):
+ pass
+
+
+def _unescape_mount_path(path: str) -> str:
+ # /proc/mounts escapes spaces and the like as octal, e.g. "\040".
+ return re.sub(r'\\([0-7]{3})', lambda m: chr(int(m.group(1), 8)), path)
+
+
+def mount_source(mount_point: str) -> Optional[str]:
+ """The device mounted at `mount_point`, or None. A path mounted twice reports the last (visible) mount."""
+ try:
+ lines = PROC_MOUNTS.read_text().splitlines()
+ except OSError:
+ return None
+ source = None
+ for line in lines:
+ parts = line.split()
+ if len(parts) >= 2 and _unescape_mount_path(parts[1]) == mount_point:
+ source = parts[0]
+ return source
+
+
+def uuid_device(uuid: str) -> Optional[str]:
+ """The device with this filesystem UUID, or None."""
+ if not uuid or '/' in uuid:
+ return None
+ path = pathlib.Path('/dev/disk/by-uuid') / uuid
+ return os.path.realpath(path) if path.exists() else None
+
+
+def device_id(path) -> int:
+ return os.stat(path).st_dev
+
+
+def is_mount(path) -> bool:
+ return os.path.ismount(path)
+
+
+def find_mount_point(path: pathlib.Path) -> pathlib.Path:
+ """The mount point of the filesystem that holds `path` (an existing, resolved path)."""
+ while not is_mount(path) and path != path.parent:
+ path = path.parent
+ return path
+
+
+def check_backup_target(directory: Optional[str], uuid: Optional[str]) -> Optional[str]:
+ """Why `directory` must not be written as the backup, or None if it is on the drive with `uuid`, and that drive is
+ mounted, and is not the media drive or the system drive.
+
+ `directory` is the drive's mount point, or a folder on the drive. The folder may not exist yet (the backup
+ creates it); the drive is checked through its nearest existing folder, so a folder on an unplugged drive (which
+ would be on the system drive) is refused."""
+ if not directory or not uuid:
+ return 'No backup drive is configured'
+ path = pathlib.Path(directory)
+ if not path.is_absolute() or any(i in ('.', '..') for i in path.parts):
+ return f'{directory} is not a valid directory'
+ existing = path
+ while not existing.exists() and existing != existing.parent:
+ existing = existing.parent
+ try:
+ resolved_existing = existing.resolve(strict=True)
+ except (FileNotFoundError, RuntimeError):
+ return f'{directory} does not exist'
+ resolved = resolved_existing.joinpath(path.relative_to(existing))
+ mount_root = MOUNT_ROOT.resolve()
+ if mount_root not in resolved.parents:
+ return f'{directory} is not under {MOUNT_ROOT}'
+ media_directory = get_media_directory().resolve()
+ if resolved == media_directory or media_directory in resolved.parents or resolved in media_directory.parents:
+ return 'The backup drive cannot be the media directory'
+ mount_point = find_mount_point(resolved_existing)
+ if mount_root not in mount_point.parents:
+ # The folder is not on a drive mounted in MOUNT_ROOT; e.g. the drive is unplugged.
+ return f'No drive is mounted at {directory}'
+ if mount_point.parent == mount_root and mount_point.name in RESERVED_MOUNT_NAMES:
+ return f'{mount_point} cannot be a backup drive'
+ if device_id(mount_point) == device_id(media_directory):
+ return 'The backup drive is the media drive'
+ if device_id(mount_point) == device_id('/'):
+ return 'The backup drive is the system drive'
+ expected = uuid_device(uuid)
+ source = mount_source(str(mount_point))
+ if not expected or not source or os.path.realpath(source) != expected:
+ return f'The chosen drive is not mounted at {mount_point}'
+ if not os.access(resolved_existing, os.W_OK):
+ return f'{directory} is not writable'
+ return None
+
+
+def rsync_literal(path: str) -> str:
+ """`path` as an rsync pattern that matches only itself (rsync reads `* ? [ \\` as wildcards)."""
+ return re.sub(r'([*?\[\\])', r'\\\1', path)
+
+
+def build_rsync_command(source: pathlib.Path, target: pathlib.Path, playlists_destination: str = None,
+ excluded_directories: List[str] = None) -> List[str]:
+ """The rsync that copies `source` (the media directory) to `target` (the backup drive), never deleting."""
+ command = list()
+ # The backup should not slow the WROLPi down.
+ if ionice := shutil.which('ionice'):
+ command.extend([ionice, '-c3'])
+ if nice := shutil.which('nice'):
+ command.extend([nice, '-n', '19'])
+ command.extend([
+ shutil.which('rsync') or 'rsync',
+ '--archive',
+ '--partial', f'--partial-dir={PARTIAL_DIRECTORY}',
+ '--info=progress2,stats2', '--no-human-readable',
+ # The database is copied as a snapshot (see `snapshot_database`).
+ '--exclude=/config/wrolpi.db', '--exclude=/config/wrolpi.db-*', '--exclude=/config/.wrolpi.db.backup',
+ # Hard links rebuilt from the configs when WROLPi starts.
+ '--exclude=/tags/',
+ # Files still being written by a sync, and rsync's own partials.
+ '--exclude=.*.partial', f'--exclude={PARTIAL_DIRECTORY}/',
+ # Root-only directory on every ext4 drive.
+ '--exclude=/lost+found/',
+ ])
+ if playlists_destination and (playlists := playlists_destination.strip('/')):
+ command.append(f'--exclude=/{playlists}/')
+ # Folders the user chose not to back up.
+ for directory in excluded_directories or []:
+ if directory := directory.strip('/'):
+ command.append(f'--exclude=/{rsync_literal(directory)}/')
+ command.extend([f'{source}/', f'{target}/'])
+ return command
+
+
+def parse_progress(line: str) -> Optional[dict]:
+ if match := PROGRESS_LINE.match(line):
+ progress = dict(bytes_transferred=int(match.group(1).replace(',', '')), percent=int(match.group(2)))
+ if match.group(3):
+ progress['files_transferred'] = int(match.group(3))
+ progress['files_to_check'] = int(match.group(5))
+ return progress
+ if match := FILES_TRANSFERRED_LINE.match(line):
+ return dict(files_transferred=int(match.group(1).replace(',', '')))
+ return None
+
+
+def snapshot_database(target: pathlib.Path):
+ """Write a consistent copy of the live database to the backup drive's config directory."""
+ config_directory = target / 'config'
+ config_directory.mkdir(parents=True, exist_ok=True)
+ temporary = config_directory / '.wrolpi.db.backup'
+ temporary.unlink(missing_ok=True)
+ with contextlib.closing(sqlite3.connect(get_db_file(), timeout=60)) as source, \
+ contextlib.closing(sqlite3.connect(temporary)) as destination:
+ # One step: a read transaction, which writers in WAL mode do not wait for, and which never restarts.
+ source.backup(destination)
+ database = config_directory / 'wrolpi.db'
+ # A write-ahead log left by an older copy would be applied to the new database.
+ for suffix in ('-wal', '-shm'):
+ database.with_name(f'wrolpi.db{suffix}').unlink(missing_ok=True)
+ os.replace(temporary, database)
+
+
+def database_fingerprint(database: pathlib.Path) -> Optional[list]:
+ """The size and modification time of a database and of its write-ahead log, or None if it does not exist. A
+ commit changes one of them. An empty WAL (opened by a reader) holds nothing, so it is ignored."""
+ try:
+ stat = database.stat()
+ except FileNotFoundError:
+ return None
+ fingerprint = [stat.st_size, stat.st_mtime_ns]
+ try:
+ wal = database.with_name(f'{database.name}-wal').stat()
+ if wal.st_size:
+ fingerprint.extend([wal.st_size, wal.st_mtime_ns])
+ except FileNotFoundError:
+ pass
+ return fingerprint
+
+
+def snapshot_database_if_changed(target: pathlib.Path) -> bool:
+ """Write the snapshot (see `snapshot_database`) unless the live database and the drive's copy are both unchanged
+ since the last snapshot. Returns True if the snapshot was written.
+
+ The record is kept in memory, so the first backup after a restart always writes it."""
+ from wrolpi.api_utils import api_app
+ copy = target / 'config/wrolpi.db'
+ key = f'backup_database_snapshot:{copy}'
+ # Taken before the snapshot: a commit during the snapshot makes the next backup write it again.
+ source = database_fingerprint(get_db_file())
+ previous = api_app.shared_ctx.cache.get(key)
+ if previous and source and previous == [source, database_fingerprint(copy)]:
+ logger.info(f'The database has not changed; keeping the snapshot in {copy.parent}')
+ return False
+ snapshot_database(target)
+ api_app.shared_ctx.cache[key] = [source, database_fingerprint(copy)]
+ return True
+
+
+def backup_skip_reason() -> Optional[str]:
+ """Why a backup cannot start now, or None."""
+ config = get_sync_config()
+ if not config.backup_directory:
+ return 'No backup drive is configured'
+ if DOCKERIZED:
+ return 'Backup drives are not supported in Docker'
+ if wrol_mode_enabled():
+ return 'WROL Mode is enabled'
+ if not PYTEST and not flags.db_up.is_set():
+ return 'The database is not up'
+ if not shutil.which('rsync'):
+ return 'rsync is not installed'
+ if reason := check_backup_target(config.backup_directory, config.backup_device_uuid):
+ return reason
+ if flags.backup_running.is_set():
+ return 'A backup is already running'
+ return None
+
+
+def request_backup():
+ update_backup_status(requested=True)
+
+
+def request_backup_cancel():
+ from wrolpi.api_utils import api_app
+ if 'backup' not in api_app.shared_ctx.status:
+ return
+ status = get_backup_status()
+ if status['state'] == 'running' or status['requested']:
+ update_backup_status(cancel_requested=status['state'] == 'running', requested=False)
+
+
+def _check_backup_cancel():
+ if get_backup_status()['cancel_requested']:
+ raise BackupCancelled('The backup was cancelled')
+ if wrol_mode_enabled():
+ raise BackupCancelled('WROL Mode was enabled')
+
+
+async def _run_rsync(command: List[str], target: pathlib.Path, uuid: str) -> Tuple[int, List[str]]:
+ """Run rsync, reporting its progress. Stops it if the backup is cancelled or the drive disappears."""
+ logger.info(f'Running {" ".join(command)}')
+ process = await asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE,
+ stderr=asyncio.subprocess.PIPE)
+ errors: List[str] = list()
+
+ async def read_errors():
+ async for line in process.stderr:
+ errors.append(line.decode(errors='replace').rstrip())
+ del errors[:-20]
+
+ errors_task = asyncio.create_task(read_errors())
+ buffer = ''
+ last_report = last_mount_check = 0.0
+ progress = dict()
+ try:
+ while True:
+ try:
+ chunk = await asyncio.wait_for(process.stdout.read(65536), timeout=1)
+ except asyncio.TimeoutError:
+ chunk = None
+ if chunk == b'':
+ break
+ if chunk:
+ buffer += chunk.decode(errors='replace')
+ *lines, buffer = re.split(r'[\r\n]', buffer)
+ for line in lines:
+ if parsed := parse_progress(line):
+ progress.update(parsed)
+ if time.monotonic() - last_report >= PROGRESS_INTERVAL_SECONDS and progress:
+ last_report = time.monotonic()
+ update_backup_status(**progress)
+ if time.monotonic() - last_mount_check >= MOUNT_CHECK_SECONDS:
+ last_mount_check = time.monotonic()
+ if reason := check_backup_target(str(target), uuid):
+ raise BackupError(f'Stopped: {reason}')
+ _check_backup_cancel()
+ code = await process.wait()
+ except BaseException:
+ if process.returncode is None:
+ process.terminate()
+ try:
+ await asyncio.wait_for(process.wait(), timeout=30)
+ except asyncio.TimeoutError:
+ process.kill()
+ raise
+ finally:
+ await asyncio.gather(errors_task, return_exceptions=True)
+ if buffer and (parsed := parse_progress(buffer)):
+ progress.update(parsed)
+ if progress:
+ update_backup_status(**progress)
+ return code, errors
+
+
+async def run_backup() -> dict:
+ """Copy the media directory to the backup drive. Returns the final status."""
+ if reason := backup_skip_reason():
+ logger.info(f'Not backing up: {reason}')
+ return update_backup_status(last_skip_reason=reason)
+
+ config = get_sync_config()
+ target = pathlib.Path(config.backup_directory)
+ uuid = config.backup_device_uuid
+ started_at = now()
+ update_backup_status(state='running', phase='files', cancel_requested=False, started_at=started_at.isoformat(),
+ finished_at=None, last_error=None, last_skip_reason=None, directory=str(target),
+ bytes_transferred=0, percent=0, files_transferred=0, files_to_check=0)
+ logger.warning(f'Backing up to {target}')
+ warning = None
+ try:
+ with flags.backup_running:
+ # A folder on the drive is created once the drive has been checked (`backup_skip_reason`).
+ target.mkdir(parents=True, exist_ok=True)
+ command = build_rsync_command(get_media_directory(), target, get_wrolpi_config().playlists_destination,
+ config.backup_excluded_directories)
+ code, errors = await _run_rsync(command, target, uuid)
+ if code == RSYNC_PARTIAL:
+ # Unreadable files; everything else was copied.
+ warning = 'Some files could not be read: ' + ' '.join(errors[-3:])
+ logger.warning(warning)
+ elif code not in (0, RSYNC_VANISHED):
+ raise BackupError(f'rsync failed ({code}): ' + ' '.join(errors[-3:]))
+ _check_backup_cancel()
+
+ update_backup_status(phase='database')
+ if reason := check_backup_target(str(target), uuid):
+ raise BackupError(f'Stopped: {reason}')
+ await asyncio.to_thread(snapshot_database_if_changed, target)
+ except BackupCancelled as e:
+ logger.warning(f'Backup stopped: {e}')
+ return update_backup_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e),
+ cancel_requested=False)
+ except Exception as e:
+ logger.error('Backup failed', exc_info=e)
+ Events.send_backup_failed(f'Backup to {target} failed: {e}')
+ return update_backup_status(state='error', phase=None, finished_at=now().isoformat(), last_error=str(e),
+ cancel_requested=False)
+
+ finished_at = now()
+ status = get_backup_status()
+ if not warning:
+ get_sync_config().update({'backup_last_success_at': finished_at.isoformat()})
+ message = f'Backed up {status["files_transferred"]} files to {target}'
+ logger.warning(f'{message} in {(finished_at - started_at).total_seconds():.0f}s')
+ if status['files_transferred'] or warning:
+ Events.send_backup_completed(message + (f'; {warning}' if warning else ''))
+ return update_backup_status(state='idle', phase=None, finished_at=finished_at.isoformat(),
+ last_success_at=finished_at.isoformat() if not warning else status['last_success_at'],
+ last_error=warning, percent=100,
+ # A request to stop during the database snapshot (which runs to the end) is done.
+ cancel_requested=False)
+
+
+def backup_is_due(status: dict, local_now) -> bool:
+ """True if the backup window is open and the last backup finished long enough ago."""
+ config = get_sync_config()
+ return window_is_due(status, local_now, config.backup_window_start, config.backup_window_end,
+ BACKUP_REPEAT_SECONDS)
+
+
+async def backup_tick():
+ """Start a backup when one was requested, or when it is due."""
+ status = get_backup_status()
+ if not get_sync_config().backup_directory or wrol_mode_enabled():
+ if status['requested']:
+ reason = 'WROL Mode is enabled' if wrol_mode_enabled() else 'No backup drive is configured'
+ update_backup_status(requested=False, last_skip_reason=reason)
+ return
+ from wrolpi.downloader import DownloadManager
+ if not status['requested'] and not backup_is_due(status, DownloadManager._get_local_now()):
+ return
+ update_backup_status(requested=False)
+ await run_backup()
diff --git a/wrolpi/sync/config.py b/wrolpi/sync/config.py
new file mode 100644
index 00000000..da2c98f3
--- /dev/null
+++ b/wrolpi/sync/config.py
@@ -0,0 +1,110 @@
+"""sync.yaml: how this WROLPi syncs.
+
+These settings belong to this machine: they are never copied from a primary (a secondary's sync.yaml says it is a
+secondary; its primary's says otherwise), so they are kept apart from wrolpi.yaml, whose library settings are.
+"""
+from dataclasses import dataclass, field
+from typing import List, Optional
+
+from wrolpi.common import ConfigFile
+from wrolpi.sync.status import get_backup_status, update_backup_status
+
+
+@dataclass
+class SyncConfigValidator:
+ version: int = None
+ # 'primary' or 'secondary'; a secondary copies its library from `primary_url` (see wrolpi.sync.lib).
+ role: str = None
+ primary_url: str = None
+ # The daily window ('HH:MM') in which a secondary syncs; none means only "Sync Now".
+ window_start: str = None
+ window_end: str = None
+ # Folders (relative to the media directory) a secondary never downloads from its primary.
+ sync_excluded_directories: list[str] = field(default_factory=list)
+ # A second drive this WROLPi copies its media directory to (see wrolpi.sync.backup), and the UUID of the drive
+ # that must be mounted there.
+ backup_directory: str = None
+ backup_device_uuid: str = None
+ # The daily window ('HH:MM') in which the backup drive is written; none means only "Back Up Now".
+ backup_window_start: str = None
+ backup_window_end: str = None
+ # Folders (relative to the media directory) never copied to the backup drive.
+ backup_excluded_directories: list[str] = field(default_factory=list)
+ # When a backup last finished without errors (ISO 8601); kept so it is known after a restart.
+ backup_last_success_at: str = None
+
+
+class SyncConfig(ConfigFile):
+ file_name = 'sync.yaml'
+ default_config = dict(
+ version=0,
+ role='primary',
+ primary_url=None,
+ window_start=None,
+ window_end=None,
+ sync_excluded_directories=[],
+ backup_directory=None,
+ backup_device_uuid=None,
+ backup_window_start=None,
+ backup_window_end=None,
+ backup_excluded_directories=[],
+ backup_last_success_at=None,
+ )
+ validator = SyncConfigValidator
+
+ def import_config(self, file=None, send_events=False):
+ super().import_config(file, send_events)
+ # The backup's status lives in memory; a restart would otherwise forget when the last backup succeeded.
+ if self.backup_last_success_at and not get_backup_status()['last_success_at']:
+ update_backup_status(last_success_at=self.backup_last_success_at)
+
+ @property
+ def role(self) -> str:
+ return self._config.get('role') or 'primary'
+
+ @property
+ def primary_url(self) -> Optional[str]:
+ return self._config.get('primary_url')
+
+ @property
+ def window_start(self) -> Optional[str]:
+ return self._config.get('window_start')
+
+ @property
+ def window_end(self) -> Optional[str]:
+ return self._config.get('window_end')
+
+ @property
+ def sync_excluded_directories(self) -> List[str]:
+ return list(self._config.get('sync_excluded_directories') or [])
+
+ @property
+ def backup_directory(self) -> Optional[str]:
+ return self._config.get('backup_directory')
+
+ @property
+ def backup_device_uuid(self) -> Optional[str]:
+ return self._config.get('backup_device_uuid')
+
+ @property
+ def backup_window_start(self) -> Optional[str]:
+ return self._config.get('backup_window_start')
+
+ @property
+ def backup_window_end(self) -> Optional[str]:
+ return self._config.get('backup_window_end')
+
+ @property
+ def backup_excluded_directories(self) -> List[str]:
+ return list(self._config.get('backup_excluded_directories') or [])
+
+ @property
+ def backup_last_success_at(self) -> Optional[str]:
+ return self._config.get('backup_last_success_at')
+
+
+SYNC_CONFIG: SyncConfig = SyncConfig()
+
+
+def get_sync_config() -> SyncConfig:
+ return SYNC_CONFIG
diff --git a/wrolpi/sync/conftest.py b/wrolpi/sync/conftest.py
new file mode 100644
index 00000000..0f2f09f9
--- /dev/null
+++ b/wrolpi/sync/conftest.py
@@ -0,0 +1,158 @@
+import contextlib
+import email.utils
+import functools
+import http.server
+import json
+import os
+import pathlib
+import tempfile
+import threading
+from typing import Callable, List
+
+import pytest
+
+from wrolpi.common import get_wrolpi_config, set_test_media_directory
+from wrolpi.sync import lib
+from wrolpi.sync.config import get_sync_config
+
+
+class CaddyLikeHandler(http.server.SimpleHTTPRequestHandler):
+ """Serves a directory at /media/ the way the primary's Caddy does: Range requests, and JSON directory listings
+ when asked with `Accept: application/json`."""
+
+ def __init__(self, *args, server_state: dict, **kwargs):
+ self.server_state = server_state
+ super().__init__(*args, **kwargs)
+
+ def log_message(self, *args):
+ pass
+
+ def translate_path(self, path):
+ path = path.split('?', 1)[0]
+ if path.startswith('/media/'):
+ return super().translate_path(path[len('/media'):])
+ return '/nonexistent-wrolpi-sync-test-path'
+
+ def do_GET(self):
+ self.server_state['requests'].append((self.path, self.headers.get('Range')))
+ path = self.translate_path(self.path)
+ if os.path.isdir(path) and 'application/json' in (self.headers.get('Accept') or ''):
+ entries = list()
+ for name in sorted(os.listdir(path)):
+ full = os.path.join(path, name)
+ is_dir = os.path.isdir(full)
+ entries.append(dict(name=name + ('/' if is_dir else ''), size=os.path.getsize(full), is_dir=is_dir,
+ mod_time=email.utils.formatdate(os.path.getmtime(full))))
+ body = json.dumps(entries).encode()
+ self.send_response(200)
+ self.send_header('Content-Type', 'application/json')
+ self.send_header('Content-Length', str(len(body)))
+ self.end_headers()
+ self.wfile.write(body)
+ return
+
+ if self.server_state['omit_length'] and os.path.isfile(path):
+ # The body ends when the connection closes (HTTP/1.0), so the client cannot know its size.
+ self.send_response(200)
+ self.end_headers()
+ with open(path, 'rb') as fh:
+ self.wfile.write(fh.read())
+ return
+
+ range_ = self.headers.get('Range')
+ if range_ and self.server_state['honor_range'] and os.path.isfile(path):
+ start = int(range_.split('=', 1)[1].split('-', 1)[0])
+ size = os.path.getsize(path)
+ if start >= size:
+ self.send_response(416)
+ self.send_header('Content-Length', '0')
+ self.end_headers()
+ return
+ self.send_response(206)
+ self.send_header('Content-Length', str(size - start))
+ self.send_header('Content-Range', f'bytes {start}-{size - 1}/{size}')
+ self.end_headers()
+ with open(path, 'rb') as fh:
+ fh.seek(start)
+ self.wfile.write(fh.read())
+ return
+
+ super().do_GET()
+
+
+@pytest.fixture
+def primary_media_server() -> Callable[[pathlib.Path], tuple]:
+ """Start a server that serves a directory like the primary's Caddy. Returns (url, state); `state['requests']`
+ records (path, Range header) of each request, `state['honor_range'] = False` makes it ignore Range, and
+ `state['omit_length'] = True` makes it send files without a Content-Length."""
+ servers = list()
+
+ def start(directory: pathlib.Path):
+ state = dict(requests=list(), honor_range=True, omit_length=False)
+ handler = functools.partial(CaddyLikeHandler, directory=str(directory), server_state=state)
+ server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), handler)
+ threading.Thread(target=server.serve_forever, daemon=True).start()
+ servers.append(server)
+ return f'http://127.0.0.1:{server.server_address[1]}', state
+
+ yield start
+
+ for server in servers:
+ server.shutdown()
+ server.server_close()
+
+
+@pytest.fixture
+def secondary_directory(test_directory) -> pathlib.Path:
+ """A second, empty media directory. The test switches to it (`use_secondary`) after preparing the primary in
+ `test_directory`; both "machines" share the test database."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ path = pathlib.Path(tmp_dir).resolve()
+ (path / 'config').mkdir()
+ try:
+ yield path
+ finally:
+ set_test_media_directory(test_directory)
+
+
+@pytest.fixture
+def make_secondary(test_wrolpi_config):
+ """Make this WROLPi a secondary of `url`."""
+
+ def make(url: str = 'http://127.0.0.1:1'):
+ get_sync_config()._config.update(role='secondary', primary_url=url)
+
+ return make
+
+
+@pytest.fixture
+def fake_primary_api(monkeypatch):
+ """Answer the primary's sync API (info and manifest) from pages captured on the "primary" beforehand; media still
+ comes from the `primary_media_server`."""
+
+ def fake(pages: List[dict], media_directory: pathlib.Path, role: str = 'primary', version: str = None):
+ async def get_info(self):
+ from wrolpi.version import __version__
+ return dict(role=role, version=version or __version__, media_directory=str(media_directory),
+ max_file_group_id=pages[-1]['max_id'] if pages else 0)
+
+ async def iter_manifest(self, limit: int = lib.MANIFEST_PAGE_LIMIT):
+ for page in pages:
+ yield page
+
+ monkeypatch.setattr(lib.PrimaryClient, 'get_info', get_info)
+ monkeypatch.setattr(lib.PrimaryClient, 'iter_manifest', iter_manifest)
+
+ return fake
+
+
+def manifest_page(*files: tuple, next_after_id=None, max_id: int = 1) -> dict:
+ """A manifest page of one group holding `files`, each (path, size, mtime)."""
+ return dict(groups=[dict(id=1, paths=[dict(path=p, size=s, mtime=m) for p, s, m in files])],
+ next_after_id=next_after_id, max_id=max_id, limit=lib.MANIFEST_PAGE_LIMIT)
+
+
+@contextlib.contextmanager
+def media_directory(path: pathlib.Path):
+ set_test_media_directory(path)
+ yield path
diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py
new file mode 100644
index 00000000..e519e109
--- /dev/null
+++ b/wrolpi/sync/lib.py
@@ -0,0 +1,866 @@
+"""Copy a primary WROLPi's library to a secondary WROLPi.
+
+A secondary pulls; the primary only answers reads. The library is the media files plus the config YAMLs, because the
+database is rebuilt from both, so a sync is:
+
+1. Walk the primary's manifest (`/api/sync/manifest`): every file of every FileGroup, paged by FileGroup id.
+2. Download each file the secondary lacks, or has with a different size or modification time, from the primary's
+ `/media/` (Caddy). A file is written to a hidden partial (refresh never indexes hidden files), then renamed into
+ place with the primary's modification time, so a later comparison is exact and an interrupted download resumes.
+3. Refresh the downloaded files, a page at a time.
+4. Copy the synced configs (`SYNCED_CONFIG_FILE_NAMES`) and import them. Files come first so the configs never
+ refer to files the secondary does not have yet.
+
+Files deleted on the primary are never deleted on the secondary.
+
+While a sync runs the secondary's API refuses requests (`check_sync_api_lock`), so nobody edits a library that is
+being replaced.
+"""
+import asyncio
+import contextlib
+import json
+import os
+import pathlib
+import posixpath
+import re
+import shutil
+import ssl
+import time
+from dataclasses import dataclass
+from datetime import datetime
+from http import HTTPStatus
+from typing import AsyncGenerator, List, Optional
+from urllib.parse import quote
+
+import aiohttp
+import yaml
+from sqlalchemy import func, select
+from sqlalchemy.orm import Session
+
+from wrolpi import flags
+from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \
+ get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES
+from wrolpi.dates import now, is_within_daily_window
+from wrolpi.errors import InvalidConfig
+from wrolpi.events import Events
+from wrolpi.sync.config import get_sync_config
+from wrolpi.sync.status import get_sync_status, update_sync_status
+from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST
+from wrolpi.version import __version__
+
+logger = logger.getChild(__name__)
+
+MANIFEST_PAGE_LIMIT = 500
+# Files downloaded at once. Small files are dominated by round trips; four keeps a Pi's disk and CPU busy.
+DOWNLOAD_CONCURRENCY = 4
+DOWNLOAD_CHUNK_SIZE = 1024 * 1024
+# Never fill the secondary's drive; leave this much free.
+MIN_FREE_BYTES = 1024 ** 3
+# exFAT stores modification times in 2-second steps.
+MTIME_TOLERANCE_SECONDS = 2.0
+# How often a running sync reports its progress.
+PROGRESS_INTERVAL_SECONDS = 2.0
+# Within the sync window, sync again this long after the last sync finished (an idle sync only compares sizes).
+SYNC_WINDOW_REPEAT_SECONDS = 15 * 60
+REFRESH_TIMEOUT_SECONDS = 60 * 60
+# Configs and directory listings are small; never read more than this from the primary.
+MAX_CONFIG_BYTES = 64 * 1024 * 1024
+# Enough for any real library; keeps a pasted list from growing without bound.
+MAX_EXCLUDED_DIRECTORIES = 500
+
+# Top-level directories never written by a sync: the configs and database, and the Tags Directory (hard links that
+# the secondary rebuilds from tags.yaml).
+PROTECTED_DIRECTORIES = frozenset({'config', 'tags'})
+# Top-level folders never offered for exclusion: the protected ones, and the root-only folder of every ext4 drive.
+UNLISTED_DIRECTORIES = PROTECTED_DIRECTORIES | {'lost+found'}
+# Files that are still being downloaded on the primary: aria2 and yt-dlp partials, and yt-dlp format files.
+INCOMPLETE_FILE = re.compile(r'(\.part|\.aria2|\.ytdl|\.temp)$|\.f\d{2,4}\.[^.]+$')
+
+# wrolpi.yaml keys that describe the library rather than this machine (hotspot, timezone, etc. stay local; the role
+# and other sync settings are in sync.yaml, which is never synced).
+SYNCED_WROLPI_CONFIG_KEYS = (
+ 'archive_destination',
+ 'ignore_outdated_zims',
+ 'ignored_directories',
+ 'map_default_location',
+ 'map_destination',
+ 'playlists_destination',
+ 'repos_destination',
+ 'save_ffprobe_json',
+ 'tags_directory',
+ 'videos_destination',
+ 'zims_destination',
+)
+
+
+class SyncError(Exception):
+ pass
+
+
+class SyncDiskFull(SyncError):
+ pass
+
+
+class SyncCancelled(SyncError):
+ pass
+
+
+@dataclass(frozen=True)
+class ManifestFile:
+ path: str # Relative to the media directory, '/' separated.
+ size: int
+ mtime: float
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Primary: the manifest.
+
+
+def build_manifest_page(session: Session, after_id: int = 0, limit: int = MANIFEST_PAGE_LIMIT) -> dict:
+ """One page of the primary's library: the files of the FileGroups after `after_id`, in id order.
+
+ Sizes and modification times come from the files themselves, not the database, which does not record them for
+ every file of a group."""
+ from wrolpi.files.models import FileGroup
+
+ media_directory = get_media_directory().resolve()
+ rows = session.execute(
+ select(FileGroup.id, FileGroup.directory, FileGroup.files)
+ .where(FileGroup.id > after_id)
+ .order_by(FileGroup.id)
+ .limit(limit)
+ ).all()
+ max_id = session.execute(select(func.max(FileGroup.id))).scalar() or 0
+
+ groups = list()
+ for id_, directory, files in rows:
+ paths = list()
+ for entry in files or []:
+ if not isinstance(entry, dict) or not entry.get('path'):
+ continue
+ path = pathlib.Path(entry['path'])
+ if not path.is_absolute():
+ # Files are stored relative to their FileGroup's directory; older rows stored absolute paths.
+ path = pathlib.Path(directory) / path
+ if manifest_file := _manifest_file(media_directory, path):
+ paths.append(manifest_file)
+ groups.append(dict(id=id_, paths=[dict(path=i.path, size=i.size, mtime=i.mtime) for i in paths]))
+
+ next_after_id = rows[-1][0] if len(rows) == limit else None
+ return dict(groups=groups, next_after_id=next_after_id, max_id=max_id, limit=limit)
+
+
+def _is_protected(name: str) -> bool:
+ """True if the top-level folder `name` is a protected one. Case is ignored: on a drive that ignores case (exFAT,
+ NTFS, macOS) 'Config' is the real config folder."""
+ return name.casefold() in PROTECTED_DIRECTORIES
+
+
+def _is_unlisted(name: str) -> bool:
+ return name.casefold() in UNLISTED_DIRECTORIES
+
+
+def _manifest_file(media_directory: pathlib.Path, path: pathlib.Path) -> Optional[ManifestFile]:
+ try:
+ relative = path.relative_to(media_directory)
+ except ValueError:
+ return None
+ parts = relative.parts
+ if not parts or any(i.startswith('.') for i in parts) or _is_protected(parts[0]):
+ return None
+ if INCOMPLETE_FILE.search(path.name):
+ return None
+ try:
+ stat = path.stat()
+ except (FileNotFoundError, NotADirectoryError):
+ return None
+ if not path.is_file():
+ return None
+ return ManifestFile(relative.as_posix(), stat.st_size, stat.st_mtime)
+
+
+def get_sync_info(session: Session) -> dict:
+ from wrolpi.files.models import FileGroup
+ return dict(
+ version=__version__.strip(),
+ role=get_sync_config().role,
+ media_directory=str(get_media_directory()),
+ max_file_group_id=session.execute(select(func.max(FileGroup.id))).scalar() or 0,
+ )
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Secondary: comparing and writing files.
+
+
+def resolve_destination(media_directory: pathlib.Path, path: str) -> Optional[pathlib.Path]:
+ """The secondary's path for a manifest path, or None if the path must not be written: outside the media directory,
+ hidden, protected, or reached through a symlink."""
+ if not path or path.startswith('/') or '\\' in path:
+ return None
+ parts = pathlib.PurePosixPath(path).parts
+ if not parts or any(i in ('.', '..') or i.startswith('.') for i in parts):
+ return None
+ if _is_protected(parts[0]):
+ return None
+ media_directory = media_directory.resolve()
+ destination = media_directory.joinpath(*parts)
+ # A symlink anywhere on the way (or the file itself) resolves elsewhere.
+ if pathlib.Path(os.path.realpath(destination)) != destination:
+ return None
+ return destination
+
+
+def normalize_excluded_directories(values: List[str]) -> List[str]:
+ """Excluded folders as '/'-separated paths relative to the media directory, without duplicates or folders that
+ are already inside another excluded folder."""
+ if len(values) > MAX_EXCLUDED_DIRECTORIES:
+ raise InvalidConfig(f'No more than {MAX_EXCLUDED_DIRECTORIES} folders can be excluded')
+ directories = set()
+ for value in values:
+ if not isinstance(value, str) or '\\' in value or '\x00' in value:
+ raise InvalidConfig(f'Invalid folder: {value!r}')
+ stripped = value.strip().strip('/')
+ # Reject '..' before normalizing, which would quietly resolve it.
+ parts = [i for i in stripped.split('/') if i] if stripped else []
+ directory = posixpath.normpath(stripped) if stripped else ''
+ if not parts or any(i.startswith('.') for i in parts):
+ raise InvalidConfig(f'Invalid folder: {value!r}')
+ if _is_protected(parts[0]):
+ raise InvalidConfig(f'{parts[0]} is never copied')
+ directories.add(directory)
+ return sorted(i for i in directories if not any(is_excluded(i, {j}) for j in directories if j != i))
+
+
+def is_excluded(path: str, excluded_directories) -> bool:
+ """True if `path` (relative, '/'-separated) is an excluded folder or inside one."""
+ return any(path == i or path.startswith(f'{i}/') for i in excluded_directories)
+
+
+def list_local_directories(path: str) -> List[dict]:
+ """The folders directly inside `path` (relative to the media directory) that a user may exclude."""
+ media_directory = get_media_directory().resolve()
+ path = (path or '').strip('/')
+ directory = resolve_destination(media_directory, path) if path else media_directory
+ if directory is None or not directory.is_dir():
+ raise InvalidConfig(f'Not a folder: {path!r}')
+ directories = list()
+ with os.scandir(directory) as entries:
+ for entry in entries:
+ if entry.name.startswith('.') or not entry.is_dir(follow_symlinks=False):
+ continue
+ if not path and _is_unlisted(entry.name):
+ continue
+ directories.append(dict(name=entry.name, path=f'{path}/{entry.name}' if path else entry.name))
+ return sorted(directories, key=lambda i: i['name'].lower())
+
+
+async def list_primary_directories(path: str) -> List[dict]:
+ """The folders directly inside `path` in the primary's media directory (Caddy's browse listing)."""
+ path = (path or '').strip('/')
+ if path and resolve_destination(get_media_directory(), path) is None:
+ raise InvalidConfig(f'Not a folder: {path!r}')
+ async with PrimaryClient(get_sync_config().primary_url) as client:
+ entries = await client.list_directory(path)
+ directories = list()
+ for entry in entries:
+ name = (entry.get('name') or '').rstrip('/')
+ if not entry.get('is_dir') or not name or name.startswith('.') or '/' in name:
+ continue
+ if not path and _is_unlisted(name):
+ continue
+ directories.append(dict(name=name, path=f'{path}/{name}' if path else name))
+ return sorted(directories, key=lambda i: i['name'].lower())
+
+
+def needs_download(destination: pathlib.Path, file: ManifestFile) -> bool:
+ try:
+ stat = destination.stat()
+ except FileNotFoundError:
+ return True
+ return stat.st_size != file.size or abs(stat.st_mtime - file.mtime) > MTIME_TOLERANCE_SECONDS
+
+
+def partial_path(destination: pathlib.Path) -> pathlib.Path:
+ """Hidden, so a refresh never indexes a file that is still downloading."""
+ return destination.with_name(f'.{destination.name}.partial')
+
+
+def _cpu_lacks_aes() -> bool:
+ """True when the CPU has no AES instructions (a Raspberry Pi 4). ChaCha20 is then about twice as fast."""
+ try:
+ cpuinfo = pathlib.Path('/proc/cpuinfo').read_text()
+ except OSError:
+ return False
+ features = [i for i in cpuinfo.splitlines() if i.lower().startswith(('features', 'flags'))]
+ return bool(features) and not any(' aes' in f' {i.split(":", 1)[-1]} ' for i in features)
+
+
+def make_sync_ssl_context() -> ssl.SSLContext:
+ """WROLPi serves HTTPS with its own certificate authority; on a trusted LAN (see the Bookshelf threat model) the
+ primary is not verified. Pinning it would load the primary's `/ca.crt` here with `load_verify_locations`."""
+ context = ssl.create_default_context()
+ context.check_hostname = False
+ context.verify_mode = ssl.CERT_NONE
+ if _cpu_lacks_aes():
+ # Python cannot order TLS 1.3 ciphers, and TLS 1.3 picks AES-GCM, which a Pi 4 decrypts at half the speed.
+ context.maximum_version = ssl.TLSVersion.TLSv1_2
+ context.set_ciphers('ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE+AESGCM')
+ return context
+
+
+def normalize_primary_url(url: str) -> str:
+ """'https://host[:port]' with no path; raises ValueError otherwise."""
+ from urllib.parse import urlparse
+ url = (url or '').strip().rstrip('/')
+ parsed = urlparse(url)
+ if parsed.scheme not in ('http', 'https') or not parsed.hostname:
+ raise ValueError('The primary must be an http:// or https:// address')
+ if parsed.username or parsed.password:
+ # The address is logged and shown in the sync's status.
+ raise ValueError('The primary address must not contain a user name or password')
+ if parsed.path or parsed.query or parsed.fragment:
+ raise ValueError('The primary address must not have a path')
+ return url
+
+
+async def _read_limited(response: aiohttp.ClientResponse, path: str) -> bytes:
+ data = bytearray()
+ async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE):
+ data.extend(chunk)
+ if len(data) > MAX_CONFIG_BYTES:
+ raise SyncError(f'{path} is larger than {MAX_CONFIG_BYTES} bytes')
+ return bytes(data)
+
+
+class PrimaryClient:
+ """Reads from a primary WROLPi: its sync API, and its media directory through Caddy."""
+
+ def __init__(self, url: str):
+ self.url = normalize_primary_url(url)
+ self.session: Optional[aiohttp.ClientSession] = None
+
+ async def __aenter__(self):
+ ssl_context = make_sync_ssl_context() if self.url.startswith('https') else False
+ connector = aiohttp.TCPConnector(ssl=ssl_context, limit=DOWNLOAD_CONCURRENCY * 2)
+ # A large file takes minutes; only a stalled read is an error.
+ timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=120)
+ self.session = aiohttp.ClientSession(connector=connector, timeout=timeout, headers=DEFAULT_HTTP_HEADERS)
+ return self
+
+ async def __aexit__(self, *exc):
+ await self.session.close()
+
+ def media_url(self, path: str) -> str:
+ return f'{self.url}/media/{quote(path)}'
+
+ async def get_json(self, path: str, params: dict = None, headers: dict = None):
+ async with self.session.get(f'{self.url}{path}', params=params, headers=headers) as response:
+ if response.status != HTTPStatus.OK:
+ raise SyncError(f'{path} returned HTTP {response.status}')
+ return json.loads(await _read_limited(response, path))
+
+ async def get_info(self) -> dict:
+ return await self.get_json('/api/sync/info')
+
+ async def iter_manifest(self, limit: int = MANIFEST_PAGE_LIMIT) -> AsyncGenerator[dict, None]:
+ after_id = 0
+ while True:
+ page = await self.get_json('/api/sync/manifest', params=dict(after_id=after_id, limit=limit))
+ yield page
+ if page.get('next_after_id') is None:
+ return
+ after_id = page['next_after_id']
+
+ async def get_bytes(self, path: str) -> Optional[bytes]:
+ """A file in the primary's media directory, or None if it does not exist."""
+ async with self.session.get(self.media_url(path)) as response:
+ if response.status == HTTPStatus.NOT_FOUND:
+ return None
+ if response.status != HTTPStatus.OK:
+ raise SyncError(f'{path} returned HTTP {response.status}')
+ return await _read_limited(response, path)
+
+ async def list_directory(self, path: str) -> List[dict]:
+ """The entries of a directory in the primary's media directory (Caddy's browse listing), [] if missing."""
+ url = self.media_url(path.rstrip('/') + '/')
+ async with self.session.get(url, headers={'Accept': 'application/json'}) as response:
+ if response.status == HTTPStatus.NOT_FOUND:
+ return []
+ if response.status != HTTPStatus.OK:
+ raise SyncError(f'{path} returned HTTP {response.status}')
+ return json.loads(await _read_limited(response, path))
+
+ @contextlib.asynccontextmanager
+ async def open_media(self, path: str, offset: int = 0):
+ # Sizes and ranges count the file's bytes, so the file must not be compressed in transit.
+ headers = {'Accept-Encoding': 'identity'}
+ if offset:
+ headers['Range'] = f'bytes={offset}-'
+ async with self.session.get(self.media_url(path), headers=headers) as response:
+ yield response
+
+
+async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path,
+ on_bytes=None) -> int:
+ """Download one file to `destination` with the primary's modification time. Returns the bytes received.
+
+ A partial file left by an earlier attempt is resumed."""
+ partial = partial_path(destination)
+ destination.parent.mkdir(parents=True, exist_ok=True)
+
+ offset = partial.stat().st_size if partial.is_file() else 0
+ if offset > file.size:
+ offset = 0
+ remaining = file.size - offset
+ if shutil.disk_usage(destination.parent).free - remaining < MIN_FREE_BYTES:
+ raise SyncDiskFull(f'Not enough free space for {file.path}')
+
+ received = 0
+ if remaining > 0 or not partial.is_file():
+ async with client.open_media(file.path, offset) as response:
+ if response.status == HTTPStatus.PARTIAL_CONTENT and offset:
+ mode = 'ab'
+ elif response.status == HTTPStatus.OK:
+ mode, offset = 'wb', 0
+ elif response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE:
+ # The partial does not match the primary's file; start over next time.
+ partial.unlink(missing_ok=True)
+ raise SyncError(f'{file.path} changed on the primary')
+ else:
+ raise SyncError(f'{file.path} returned HTTP {response.status}')
+
+ if response.content_length is not None and response.content_length != file.size - offset:
+ # The file changed on the primary after the manifest was read; the next sync gets it.
+ raise SyncError(f'{file.path} changed on the primary')
+
+ with open(partial, mode) as fh:
+ async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE):
+ if offset + received + len(chunk) > file.size:
+ # Never write past the size the free-space check allowed for.
+ fh.close()
+ partial.unlink(missing_ok=True)
+ raise SyncError(f'{file.path} is larger than the primary said')
+ await asyncio.to_thread(fh.write, chunk)
+ received += len(chunk)
+ if on_bytes:
+ on_bytes(len(chunk))
+
+ if (size := partial.stat().st_size) != file.size:
+ raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}')
+ os.utime(partial, (file.mtime, file.mtime))
+ os.replace(partial, destination)
+ return received
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Secondary: configs.
+
+
+def _import_tags():
+ from wrolpi.tags import import_tags_config
+ import_tags_config()
+
+
+def _import_downloads():
+ from wrolpi.downloader import get_download_manager_config
+ get_download_manager_config().import_config()
+
+
+def _import_channels():
+ from modules.videos.lib import import_channels_config
+ import_channels_config()
+
+
+def _import_domains():
+ from modules.archive.lib import import_domains_config
+ import_domains_config()
+
+
+def _import_playlists():
+ from wrolpi.collections.config import playlists_config
+ from wrolpi.collections.sync import sync_playlists_directory
+ playlists_config.import_config()
+ sync_playlists_directory.activate_switch()
+
+
+def _import_zims():
+ from modules.zim.lib import import_zims_config
+ import_zims_config()
+
+
+def _import_plain(file_name: str):
+ def import_():
+ get_config_by_file_name(file_name).import_config()
+
+ return import_
+
+
+def _import_inventories():
+ from modules.inventory.common import import_inventories_config
+ import_inventories_config()
+
+
+# In the order of `import_all_db_configs`: Collections refer to Tags, and link to Downloads.
+SYNCED_CONFIG_IMPORTS = (
+ ('tags.yaml', _import_tags),
+ ('download_manager.yaml', _import_downloads),
+ ('channels.yaml', _import_channels),
+ ('domains.yaml', _import_domains),
+ ('playlists.yaml', _import_playlists),
+ ('zims.yaml', _import_zims),
+ ('map_pins.yaml', _import_plain('map_pins.yaml')),
+ ('bookmarks.yaml', _import_plain('bookmarks.yaml')),
+ ('videos_downloader.yaml', _import_plain('videos_downloader.yaml')),
+ ('archives_downloader.yaml', _import_plain('archives_downloader.yaml')),
+)
+INVENTORY_DIRECTORY = 'config/inventory'
+assert {i for i, _ in SYNCED_CONFIG_IMPORTS} | {'inventory/catalog.yaml'} == SYNCED_CONFIG_FILE_NAMES
+
+
+def _parse_config(data: bytes, name: str) -> dict:
+ config = yaml.safe_load(data)
+ if not isinstance(config, dict):
+ raise SyncError(f'{name} from the primary is not a config')
+ return config
+
+
+def write_config_bytes(file: pathlib.Path, data: bytes, backup: pathlib.Path = None) -> bool:
+ """Replace `file` with the primary's bytes (keeping its version). Returns False if they are already the same.
+
+ The replaced file is copied to `backup` first (today's config backup), unless today's backup exists: like a save,
+ the first copy of the day is the one kept, so a sync can be undone with a config backup import."""
+ if file.is_file() and file.read_bytes() == data:
+ return False
+ if backup and file.is_file() and not backup.exists():
+ backup.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy2(file, backup)
+ file.parent.mkdir(parents=True, exist_ok=True)
+ temporary = file.with_name(f'.{file.name}.sync')
+ temporary.write_bytes(data)
+ os.replace(temporary, file)
+ return True
+
+
+async def sync_configs(client: PrimaryClient, primary_media_directory: str, reimport: bool) -> List[str]:
+ """Copy the primary's synced configs, then import those that changed (all of them when `reimport`, because newly
+ downloaded files may let a config import entries it skipped before). Returns the imported configs."""
+ imported = list()
+ for file_name, import_ in SYNCED_CONFIG_IMPORTS:
+ data = await client.get_bytes(f'config/{file_name}')
+ if data is None:
+ continue
+ config = get_config_by_file_name(file_name)
+ _parse_config(data, file_name)
+ changed = write_config_bytes(config.get_file(), data, config._get_backup_filename())
+ if changed or reimport:
+ logger.info(f'Importing {file_name} from the primary')
+ await asyncio.to_thread(import_)
+ imported.append(file_name)
+
+ if await _sync_inventory_directory(client) or reimport:
+ logger.info('Importing inventories from the primary')
+ await asyncio.to_thread(_import_inventories)
+ imported.append('inventory')
+
+ await _merge_wrolpi_config(client, primary_media_directory)
+ return imported
+
+
+async def _sync_inventory_directory(client: PrimaryClient) -> bool:
+ from modules.inventory.common import get_inventory_configs
+ inventories = get_inventory_configs()
+ directory = inventories.get_directory()
+ changed = False
+ for entry in await client.list_directory(INVENTORY_DIRECTORY):
+ name = entry.get('name') or ''
+ if entry.get('is_dir') or not name.endswith('.yaml') or '/' in name or name.startswith('.'):
+ continue
+ data = await client.get_bytes(f'{INVENTORY_DIRECTORY}/{name}')
+ if data is None:
+ continue
+ _parse_config(data, name)
+ backup = inventories._get_backup_filename(name.removesuffix('.yaml'))
+ changed = write_config_bytes(directory / name, data, backup) or changed
+ return changed
+
+
+def merge_wrolpi_config(primary_config: dict, primary_media_directory: str) -> dict:
+ """The library settings of the primary's wrolpi.yaml that differ from this WROLPi's."""
+ wrolpi_config = get_wrolpi_config()
+ changes = dict()
+ for key in SYNCED_WROLPI_CONFIG_KEYS:
+ if key not in primary_config:
+ continue
+ value = primary_config[key]
+ if key == 'ignored_directories':
+ value = _relative_directories(value or [], primary_media_directory)
+ if wrolpi_config.dict().get(key) != value:
+ changes[key] = value
+ return changes
+
+
+def _relative_directories(directories: List[str], primary_media_directory: str) -> List[str]:
+ relative = list()
+ for directory in directories:
+ path = pathlib.PurePosixPath(directory)
+ if path.is_absolute():
+ try:
+ path = path.relative_to(primary_media_directory)
+ except ValueError:
+ # Outside the primary's media directory; meaningless here.
+ continue
+ relative.append(str(path))
+ return relative
+
+
+async def _merge_wrolpi_config(client: PrimaryClient, primary_media_directory: str):
+ data = await client.get_bytes('config/wrolpi.yaml')
+ if data is None:
+ return
+ if changes := merge_wrolpi_config(_parse_config(data, 'wrolpi.yaml'), primary_media_directory):
+ logger.info(f'Applying library settings from the primary: {sorted(changes)}')
+ get_wrolpi_config().update(changes)
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# Secondary: the sync.
+
+
+def sync_skip_reason() -> Optional[str]:
+ """Why a sync cannot start now, or None."""
+ config = get_sync_config()
+ if not is_secondary():
+ return 'This WROLPi is not a secondary'
+ if wrol_mode_enabled():
+ return 'WROL Mode is enabled'
+ if not config.primary_url:
+ return 'No primary is configured'
+ if not PYTEST:
+ if not flags.db_up.is_set():
+ return 'The database is not up'
+ if not flags.refresh_complete.is_set():
+ return 'The first refresh has not finished'
+ if get_wrolpi_config().require_media_mounted and not flags.media_mounted.is_set():
+ return 'The media directory is not mounted'
+ if flags.sync_running.is_set():
+ return 'A sync is already running'
+ return None
+
+
+def request_sync(force: bool = False):
+ update_sync_status(requested=True, force=force)
+
+
+def request_sync_cancel():
+ """Stop a running sync between pages, and forget a requested one."""
+ from wrolpi.api_utils import api_app
+ if 'sync' not in api_app.shared_ctx.status:
+ return
+ status = get_sync_status()
+ if status['state'] == 'running' or status['requested']:
+ update_sync_status(cancel_requested=status['state'] == 'running', requested=False)
+
+
+def _check_cancel():
+ if get_sync_status()['cancel_requested']:
+ raise SyncCancelled('The sync was cancelled')
+ if wrol_mode_enabled():
+ raise SyncCancelled('WROL Mode was enabled')
+
+
+@contextlib.contextmanager
+def lock_api():
+ """Refuse API requests (see `check_sync_api_lock`) while the library is replaced."""
+ from wrolpi.api_utils import api_app
+ with flags.sync_running:
+ api_app.shared_ctx.sync_api_locked.set()
+ try:
+ yield
+ finally:
+ api_app.shared_ctx.sync_api_locked.clear()
+
+
+async def run_sync(force: bool = False) -> dict:
+ """Copy the primary's library to this secondary. Returns the final status."""
+ if reason := sync_skip_reason():
+ logger.info(f'Not syncing: {reason}')
+ return update_sync_status(last_skip_reason=reason, force=False)
+
+ url = get_sync_config().primary_url
+ started_at = now()
+ update_sync_status(
+ state='running', phase='files', force=False, cancel_requested=False, started_at=started_at.isoformat(),
+ finished_at=None, last_error=None, last_skip_reason=None, primary_url=url, primary_version=None,
+ groups_seen=0, max_id=0, files_checked=0, files_to_download=0, files_downloaded=0, bytes_downloaded=0,
+ files_failed=0, files_excluded=0, configs_imported=[], disk_full=False,
+ )
+ logger.warning(f'Syncing from the primary {url}')
+ try:
+ with lock_api():
+ async with PrimaryClient(url) as client:
+ info = await client.get_info()
+ if info.get('role') != 'primary':
+ raise SyncError(f'{url} is not a primary WROLPi')
+ if (info.get('version') or '').strip() != __version__.strip():
+ logger.warning(f'The primary runs WROLPi {info.get("version")}, this is {__version__.strip()}')
+ update_sync_status(primary_version=(info.get('version') or '').strip() or None)
+
+ downloaded = await _sync_files(client)
+ _check_cancel()
+
+ update_sync_status(phase='configs')
+ primary_media_directory = info.get('media_directory') or ''
+ imported = await sync_configs(client, primary_media_directory, reimport=bool(downloaded) or force)
+ except SyncCancelled as e:
+ logger.warning(f'Sync stopped: {e}')
+ return update_sync_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e),
+ cancel_requested=False)
+ except Exception as e:
+ logger.error('Sync failed', exc_info=e)
+ Events.send_sync_failed(f'Sync from the primary failed: {e}')
+ return update_sync_status(state='error', phase=None, finished_at=now().isoformat(), last_error=str(e),
+ cancel_requested=False)
+
+ status = get_sync_status()
+ finished_at = now()
+ failed = status['files_failed']
+ message = f'Synced {len(downloaded)} files from the primary'
+ if failed:
+ message += f'; {failed} failed'
+ if status['disk_full']:
+ message += '; the drive is full'
+ logger.warning(f'{message} in {(finished_at - started_at).total_seconds():.0f}s')
+ if downloaded or failed or imported:
+ Events.send_sync_completed(message)
+ last_success_at = finished_at.isoformat() if not failed and not status['disk_full'] else status['last_success_at']
+ return update_sync_status(state='idle', phase=None, finished_at=finished_at.isoformat(),
+ last_success_at=last_success_at, configs_imported=imported,
+ last_error=f'{failed} files failed' if failed else None)
+
+
+async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]:
+ """Download the files this secondary lacks, and refresh them a page at a time. Returns the files downloaded."""
+ media_directory = get_media_directory()
+ downloaded = list()
+ groups_seen = files_checked = files_to_download = files_failed = files_excluded = 0
+ excluded_directories = set(get_sync_config().sync_excluded_directories)
+ bytes_downloaded = 0
+ disk_full = False
+
+ last_report = 0.0
+
+ def report(force: bool = False):
+ nonlocal last_report
+ if force or time.monotonic() - last_report >= PROGRESS_INTERVAL_SECONDS:
+ last_report = time.monotonic()
+ update_sync_status(groups_seen=groups_seen, files_checked=files_checked,
+ files_to_download=files_to_download, files_downloaded=len(downloaded),
+ bytes_downloaded=bytes_downloaded, files_failed=files_failed,
+ files_excluded=files_excluded, disk_full=disk_full)
+
+ def count_bytes(n: int):
+ nonlocal bytes_downloaded
+ bytes_downloaded += n
+ report()
+
+ async for page in client.iter_manifest():
+ _check_cancel()
+ wanted = list()
+ for group in page['groups']:
+ groups_seen += 1
+ for entry in group['paths']:
+ files_checked += 1
+ file = ManifestFile(entry['path'], int(entry['size']), float(entry['mtime']))
+ if excluded_directories and is_excluded(file.path, excluded_directories):
+ files_excluded += 1
+ continue
+ destination = resolve_destination(media_directory, file.path)
+ if destination is None:
+ logger.warning(f'Refusing to sync {repr(file.path)}')
+ files_failed += 1
+ continue
+ if needs_download(destination, file):
+ wanted.append((file, destination))
+ files_to_download += len(wanted)
+ update_sync_status(max_id=page.get('max_id') or 0)
+ report(force=True)
+
+ page_downloaded = list()
+ if wanted and not disk_full:
+ semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY)
+
+ async def download(file_: ManifestFile, destination_: pathlib.Path):
+ nonlocal disk_full
+ async with semaphore:
+ if disk_full:
+ return None
+ try:
+ await download_file(client, file_, destination_, count_bytes)
+ downloaded.append(destination_)
+ return destination_
+ except SyncDiskFull as e:
+ disk_full = True
+ logger.error(str(e))
+ except Exception as e:
+ logger.error(f'Failed to sync {repr(file_.path)}: {e}')
+ return False
+
+ results = await asyncio.gather(*(download(f, d) for f, d in wanted))
+ page_downloaded = [i for i in results if i]
+ files_failed += sum(1 for i in results if i is False)
+
+ update_sync_status(max_id=page.get('max_id') or 0)
+ report(force=True)
+
+ if page_downloaded:
+ await _refresh(page_downloaded)
+
+ return downloaded
+
+
+async def _refresh(paths: List[pathlib.Path]):
+ from wrolpi.files.worker import file_worker, FileWorkerJobFailed
+ job_id = file_worker.queue_refresh(paths, send_events=False)
+ try:
+ await file_worker.wait_for_job(job_id, timeout=REFRESH_TIMEOUT_SECONDS)
+ except (FileWorkerJobFailed, TimeoutError) as e:
+ # The files are in place; a later refresh indexes them.
+ logger.error(f'Failed to refresh synced files: {e}')
+
+
+def window_is_due(status: dict, local_now: datetime, start: Optional[str], end: Optional[str],
+ repeat_seconds: int) -> bool:
+ """True if the daily window from `start` to `end` is open and the last run (described by `status`) finished
+ `repeat_seconds` ago. No window: never (only on request)."""
+ if not start or not end:
+ return False
+ if not is_within_daily_window(start, end, local_now.time()):
+ return False
+ if not status.get('finished_at'):
+ return True
+ finished_at = datetime.fromisoformat(status['finished_at'])
+ return (local_now - finished_at).total_seconds() >= repeat_seconds
+
+
+def sync_is_due(status: dict, local_now: datetime) -> bool:
+ """True if the sync window is open and the last sync finished long enough ago."""
+ config = get_sync_config()
+ return window_is_due(status, local_now, config.window_start, config.window_end, SYNC_WINDOW_REPEAT_SECONDS)
+
+
+async def sync_tick():
+ """Start a sync when one was requested, or when it is due."""
+ if not is_secondary():
+ return
+ status = get_sync_status()
+ if wrol_mode_enabled():
+ if status['requested']:
+ update_sync_status(requested=False, last_skip_reason='WROL Mode is enabled')
+ return
+ from wrolpi.downloader import DownloadManager
+ if not status['requested'] and not sync_is_due(status, DownloadManager._get_local_now()):
+ return
+ force = status['force']
+ update_sync_status(requested=False)
+ await run_sync(force=force)
diff --git a/wrolpi/sync/schema.py b/wrolpi/sync/schema.py
new file mode 100644
index 00000000..1332bc0a
--- /dev/null
+++ b/wrolpi/sync/schema.py
@@ -0,0 +1,56 @@
+from dataclasses import dataclass
+from typing import List, Optional
+
+from wrolpi.errors import ValidationError
+
+
+@dataclass
+class ManifestRequest:
+ after_id: int = 0
+ limit: int = 500
+
+ def __post_init__(self):
+ self.after_id = int(self.after_id)
+ self.limit = int(self.limit)
+ if self.after_id < 0:
+ raise ValidationError('after_id must not be negative')
+ if not 1 <= self.limit <= 5000:
+ raise ValidationError('limit must be between 1 and 5000')
+
+
+@dataclass
+class SyncRunRequest:
+ # Import every synced config even if the primary's copy did not change.
+ force: bool = False
+
+
+@dataclass
+class SyncDirectoriesRequest:
+ # The folder whose sub-folders are listed, relative to the media directory ('' is the media directory).
+ path: str = ''
+ # 'local' lists this WROLPi's media directory, 'primary' lists its primary's.
+ source: str = 'local'
+
+ def __post_init__(self):
+ if self.source not in ('local', 'primary'):
+ raise ValidationError("source must be 'local' or 'primary'")
+
+
+@dataclass
+class SyncSettingsRequest:
+ role: Optional[str] = None
+ sync_primary_url: Optional[str] = None
+ sync_window_start: Optional[str] = None
+ sync_window_end: Optional[str] = None
+ # The mount point of the backup drive, and the UUID of the drive that must be mounted there ('' clears both).
+ backup_directory: Optional[str] = None
+ backup_device_uuid: Optional[str] = None
+ backup_window_start: Optional[str] = None
+ backup_window_end: Optional[str] = None
+ # Folders, relative to the media directory, never copied ([] clears).
+ sync_excluded_directories: Optional[List[str]] = None
+ backup_excluded_directories: Optional[List[str]] = None
+
+ def __post_init__(self):
+ if self.role is not None and self.role not in ('primary', 'secondary'):
+ raise ValidationError('role must be "primary" or "secondary"')
diff --git a/wrolpi/sync/status.py b/wrolpi/sync/status.py
new file mode 100644
index 00000000..845b8e62
--- /dev/null
+++ b/wrolpi/sync/status.py
@@ -0,0 +1,81 @@
+"""The shared progress records of a secondary WROLPi's sync, `api_app.shared_ctx.status['sync']`, and of a backup to
+a second drive, `api_app.shared_ctx.status['backup']`.
+
+They live in a manager dict, so every read returns a copy: change them only with `update_sync_status` and
+`update_backup_status`.
+"""
+
+
+def default_sync_status() -> dict:
+ return dict(
+ state='idle',
+ phase=None,
+ requested=False,
+ force=False,
+ cancel_requested=False,
+ started_at=None,
+ finished_at=None,
+ last_success_at=None,
+ last_error=None,
+ last_skip_reason=None,
+ primary_url=None,
+ primary_version=None,
+ groups_seen=0,
+ max_id=0,
+ files_checked=0,
+ files_to_download=0,
+ files_downloaded=0,
+ bytes_downloaded=0,
+ files_failed=0,
+ files_excluded=0,
+ configs_imported=[],
+ disk_full=False,
+ )
+
+
+def get_sync_status() -> dict:
+ from wrolpi.api_utils import api_app
+ return dict(api_app.shared_ctx.status.get('sync') or default_sync_status())
+
+
+def update_sync_status(**changes) -> dict:
+ """Merge `changes` into the shared sync status. Only the perpetual process and the sync endpoints write it, and
+ the endpoints only set the request/cancel keys, so a lost update can only delay a request by one tick."""
+ from wrolpi.api_utils import api_app
+ status = get_sync_status()
+ status.update(changes)
+ api_app.shared_ctx.status['sync'] = status
+ return status
+
+
+def default_backup_status() -> dict:
+ return dict(
+ state='idle',
+ phase=None,
+ requested=False,
+ cancel_requested=False,
+ started_at=None,
+ finished_at=None,
+ last_success_at=None,
+ last_error=None,
+ last_skip_reason=None,
+ directory=None,
+ bytes_transferred=0,
+ percent=0,
+ files_transferred=0,
+ files_to_check=0,
+ )
+
+
+def get_backup_status() -> dict:
+ from wrolpi.api_utils import api_app
+ return dict(api_app.shared_ctx.status.get('backup') or default_backup_status())
+
+
+def update_backup_status(**changes) -> dict:
+ """Merge `changes` into the shared backup status (see `update_sync_status`)."""
+ from wrolpi.api_utils import api_app
+ status = get_backup_status()
+ status.update(changes)
+ api_app.shared_ctx.status['backup'] = status
+ return status
diff --git a/wrolpi/sync/test/__init__.py b/wrolpi/sync/test/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/wrolpi/sync/test/test_api.py b/wrolpi/sync/test/test_api.py
new file mode 100644
index 00000000..f895ef42
--- /dev/null
+++ b/wrolpi/sync/test/test_api.py
@@ -0,0 +1,216 @@
+import json
+from http import HTTPStatus
+
+import pytest
+import yaml
+
+from wrolpi import flags
+from wrolpi.api_utils import api_app
+from wrolpi.common import get_wrolpi_config, enable_wrol_mode
+from wrolpi.downloader import download_manager
+from wrolpi.sync import lib
+from wrolpi.sync.config import get_sync_config
+from wrolpi.sync.status import get_sync_status, update_sync_status
+
+
+@pytest.mark.asyncio
+async def test_info_and_manifest(async_client, test_session, test_wrolpi_config, make_files_structure,
+ refresh_files):
+ make_files_structure({'docs/a.txt': 'a', 'docs/b.txt': 'b'})
+ await refresh_files()
+
+ request, response = await async_client.get('/api/sync/info')
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['role'] == 'primary'
+ assert response.json['max_file_group_id'] > 0
+
+ request, response = await async_client.get('/api/sync/manifest?after_id=0&limit=1')
+ assert response.status_code == HTTPStatus.OK
+ assert len(response.json['groups']) == 1
+ after_id = response.json['next_after_id']
+ assert after_id
+
+ request, response = await async_client.get(f'/api/sync/manifest?after_id={after_id}&limit=1')
+ paths = [p['path'] for g in response.json['groups'] for p in g['paths']]
+ assert len(paths) == 1 and paths[0].startswith('docs/')
+
+ request, response = await async_client.get('/api/sync/manifest?limit=0')
+ assert response.status_code == HTTPStatus.BAD_REQUEST
+
+
+@pytest.mark.asyncio
+async def test_run(async_client, test_wrolpi_config, make_secondary):
+ request, response = await async_client.post('/api/sync/run', content='{}')
+ assert response.status_code == HTTPStatus.FORBIDDEN, 'a primary does not sync'
+
+ make_secondary('https://primary.example')
+ request, response = await async_client.post('/api/sync/run', content=json.dumps(dict(force=True)))
+ assert response.status_code == HTTPStatus.ACCEPTED
+ assert get_sync_status()['requested'] is True
+ assert get_sync_status()['force'] is True
+
+ request, response = await async_client.get('/api/sync/status')
+ assert response.json['sync']['requested'] is True
+
+
+@pytest.mark.asyncio
+async def test_settings(async_client, test_directory, test_wrolpi_config, test_download_manager, await_switches):
+ body = dict(role='secondary', sync_primary_url=' https://10.0.0.5:8443/ ', sync_window_start='01:00',
+ sync_window_end='05:00')
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ config = get_sync_config()
+ assert config.role == 'secondary'
+ assert config.primary_url == 'https://10.0.0.5:8443'
+ assert (config.window_start, config.window_end) == ('01:00', '05:00')
+ # Saved to sync.yaml, not wrolpi.yaml.
+ await await_switches()
+ saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text())
+ assert (saved['role'], saved['primary_url']) == ('secondary', 'https://10.0.0.5:8443')
+ assert 'role' not in get_wrolpi_config().dict()
+ assert download_manager.is_disabled, 'a secondary does not download'
+
+ request, response = await async_client.get('/api/settings')
+ assert response.json['role'] == 'secondary'
+ assert response.json['sync_primary_url'] == 'https://10.0.0.5:8443'
+
+ request, response = await async_client.post('/api/download/enable')
+ assert response.status_code == HTTPStatus.FORBIDDEN
+
+ # The backup drive has its own window.
+ body = dict(backup_window_start='02:00', backup_window_end='04:00')
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ assert (config.backup_window_start, config.backup_window_end) == ('02:00', '04:00')
+ assert (config.window_start, config.window_end) == ('01:00', '05:00')
+ request, response = await async_client.get('/api/settings')
+ assert (response.json['backup_window_start'], response.json['backup_window_end']) == ('02:00', '04:00')
+
+ # Clearing the window leaves only "Sync now".
+ body = dict(sync_window_start='', sync_window_end='')
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ assert config.window_start is None
+
+ for body in (dict(role='tertiary'), dict(sync_primary_url='ftp://primary'),
+ dict(sync_primary_url='https://user:password@primary'),
+ dict(sync_primary_url='https://primary/media'), dict(sync_window_start='25:00'),
+ dict(backup_window_end='5pm'), dict()):
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.BAD_REQUEST, body
+
+
+@pytest.mark.asyncio
+async def test_settings_excluded_directories(async_client, test_directory, test_wrolpi_config, await_switches):
+ """Each panel has its own excluded folders, normalized and saved to sync.yaml."""
+ body = dict(sync_excluded_directories=['/Movies/', 'Movies/deep'], backup_excluded_directories=['videos/Movies'])
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ config = get_sync_config()
+ assert config.sync_excluded_directories == ['Movies']
+ assert config.backup_excluded_directories == ['videos/Movies']
+ await await_switches()
+ saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text())
+ assert saved['sync_excluded_directories'] == ['Movies']
+ assert saved['backup_excluded_directories'] == ['videos/Movies']
+
+ request, response = await async_client.get('/api/settings')
+ assert response.json['sync_excluded_directories'] == ['Movies']
+ assert response.json['backup_excluded_directories'] == ['videos/Movies']
+
+ # Changing one list leaves the other alone; [] clears.
+ request, response = await async_client.put('/api/sync/settings',
+ content=json.dumps(dict(backup_excluded_directories=[])))
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ assert config.backup_excluded_directories == []
+ assert config.sync_excluded_directories == ['Movies']
+
+ for value in (['../etc'], ['config'], ['.hidden'], [''], 'Movies'):
+ body = dict(sync_excluded_directories=value)
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.BAD_REQUEST, value
+ assert config.sync_excluded_directories == ['Movies']
+
+
+@pytest.mark.asyncio
+async def test_directories(async_client, test_directory, test_wrolpi_config):
+ for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'):
+ (test_directory / i).mkdir(parents=True, exist_ok=True)
+
+ request, response = await async_client.post('/api/sync/directories', content=json.dumps(dict(path='')))
+ assert response.status_code == HTTPStatus.OK
+ assert response.json['directories'] == [dict(name='Movies', path='Movies'), dict(name='videos', path='videos')]
+
+ body = dict(path='videos', source='local')
+ request, response = await async_client.post('/api/sync/directories', content=json.dumps(body))
+ assert response.json['directories'] == [dict(name='channel', path='videos/channel')]
+
+ for body in (dict(path='../'), dict(path='missing'), dict(source='elsewhere'),
+ dict(source='primary')): # A primary has no primary.
+ request, response = await async_client.post('/api/sync/directories', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.BAD_REQUEST, body
+
+
+@pytest.mark.asyncio
+async def test_directories_primary_unreachable(async_client, test_wrolpi_config, make_secondary):
+ """A primary that does not answer is reported, not raised."""
+ make_secondary('https://127.0.0.1:1')
+ body = dict(path='', source='primary')
+ request, response = await async_client.post('/api/sync/directories', content=json.dumps(body))
+ assert response.status_code == HTTPStatus.BAD_GATEWAY
+ assert response.json['error']
+
+
+@pytest.mark.asyncio
+async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_lock, make_secondary):
+ """While a secondary syncs, its API refuses everything but the sync's own progress."""
+ make_secondary()
+ with lib.lock_api():
+ request, response = await async_client.get('/api/files/search?search_str=foo')
+ assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+ assert response.json['code'] == 'SYNCING'
+ request, response = await async_client.post('/api/tag', content=json.dumps(dict(name='one')))
+ assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+
+ for path in ('/api/status', '/api/sync/status', '/api/settings', '/api/echo'):
+ request, response = await async_client.get(path)
+ assert response.status_code == HTTPStatus.OK, path
+ request, response = await async_client.post('/api/sync/cancel')
+ assert response.status_code == HTTPStatus.ACCEPTED
+ # Nothing may change the sync while it runs, except stopping it.
+ request, response = await async_client.put('/api/sync/settings', content=json.dumps(dict(role='primary')))
+ assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+ request, response = await async_client.post('/api/sync/run', content='{}')
+ assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+
+ request, response = await async_client.get('/api/status')
+ assert response.json['flags']['sync_running'] is True
+
+ request, response = await async_client.get('/api/settings')
+ assert response.status_code == HTTPStatus.OK
+ assert not api_app.shared_ctx.sync_api_locked.is_set()
+ assert not flags.sync_running.is_set()
+
+
+@pytest.mark.asyncio
+async def test_stale_lock_event_does_not_lock(async_client, test_wrolpi_config, flags_lock):
+ """The Event outlives a perpetual process that died mid-sync; the cleared Flag unlocks the API."""
+ api_app.shared_ctx.sync_api_locked.set()
+ try:
+ request, response = await async_client.get('/api/settings')
+ assert response.status_code == HTTPStatus.OK
+ finally:
+ api_app.shared_ctx.sync_api_locked.clear()
+
+
+@pytest.mark.asyncio
+async def test_wrol_mode_cancels_sync(async_client, test_wrolpi_config, test_download_manager, make_secondary):
+ """WROL Mode means the secondary is in use: a running sync stops, and none start."""
+ make_secondary()
+ update_sync_status(state='running')
+ enable_wrol_mode()
+ assert get_sync_status()['cancel_requested'] is True
+ assert lib.sync_skip_reason() == 'WROL Mode is enabled'
+
+ request, response = await async_client.post('/api/sync/run', content='{}')
+ assert response.status_code == HTTPStatus.FORBIDDEN
diff --git a/wrolpi/sync/test/test_backup.py b/wrolpi/sync/test/test_backup.py
new file mode 100644
index 00000000..ba75143a
--- /dev/null
+++ b/wrolpi/sync/test/test_backup.py
@@ -0,0 +1,467 @@
+import contextlib
+import os
+import pathlib
+import re
+import shutil
+import sqlite3
+import subprocess
+import tempfile
+from datetime import datetime, timezone
+
+import pytest
+import yaml
+
+from wrolpi.common import get_wrolpi_config
+from wrolpi.sync import backup
+from wrolpi.sync.config import get_sync_config
+from wrolpi.sync.status import get_backup_status, update_backup_status
+
+
+def _rsync_supports_progress2() -> bool:
+ """rsync 3.1+ (Raspberry Pi OS, Debian); macOS ships openrsync, which has no `--info=progress2`."""
+ if not shutil.which('rsync'):
+ return False
+ output = subprocess.run(['rsync', '--version'], capture_output=True, text=True).stdout
+ match = re.search(r'rsync\s+version\s+(\d+)\.(\d+)', output)
+ return bool(match) and (int(match.group(1)), int(match.group(2))) >= (3, 1)
+
+
+requires_rsync = pytest.mark.skipif(not _rsync_supports_progress2(), reason='rsync 3.1 or newer is not installed')
+
+
+@pytest.fixture
+def backup_drive(test_directory, test_wrolpi_config, monkeypatch):
+ """A temporary directory that the target checks accept as the mounted backup drive `uuid-1`."""
+ with tempfile.TemporaryDirectory() as tmp_dir:
+ drive = pathlib.Path(tmp_dir).resolve()
+ monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: None)
+ get_sync_config()._config.update(backup_directory=str(drive), backup_device_uuid='uuid-1')
+ yield drive
+
+
+def test_build_rsync_command(monkeypatch):
+ monkeypatch.setattr(backup.shutil, 'which', lambda name: f'/usr/bin/{name}')
+ command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), 'playlists')
+ assert command[:5] == ['/usr/bin/ionice', '-c3', '/usr/bin/nice', '-n', '19']
+ assert command[-2:] == ['/media/wrolpi/', '/media/backup/'], 'trailing slashes copy the contents, not the folder'
+ assert not any(i.startswith('--delete') for i in command), 'a backup never deletes'
+ assert '-H' not in command and '--hard-links' not in command
+ for exclude in ('/config/wrolpi.db', '/config/wrolpi.db-*', '/tags/', '/playlists/', '.*.partial',
+ '.rsync-partial/', '/lost+found/'):
+ assert f'--exclude={exclude}' in command, exclude
+
+ # ionice and nice are optional.
+ monkeypatch.setattr(backup.shutil, 'which', lambda name: '/usr/bin/rsync' if name == 'rsync' else None)
+ command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), None)
+ assert command[0] == '/usr/bin/rsync'
+ assert not any(i.startswith('--exclude=/playlists') for i in command)
+
+
+def test_build_rsync_command_excluded_directories(monkeypatch):
+ """Excluded folders are anchored at the media directory and matched literally."""
+ monkeypatch.setattr(backup.shutil, 'which', lambda name: '/usr/bin/rsync' if name == 'rsync' else None)
+ command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), None,
+ ['Movies', 'videos/Movies [HD]', 'a*b?'])
+ assert '--exclude=/Movies/' in command
+ assert '--exclude=/videos/Movies \\[HD]/' in command
+ assert '--exclude=/a\\*b\\?/' in command
+ assert command[-2:] == ['/media/wrolpi/', '/media/backup/'], 'excludes come before the paths'
+
+
+@pytest.mark.parametrize('line, expected', [
+ (' 32,768 0% 0.00kB/s 0:00:00 (xfr#1, ir-chk=1018/1023)',
+ dict(bytes_transferred=32768, percent=0, files_transferred=1, files_to_check=1023)),
+ (' 1234567890 45% 10.50MB/s 0:01:23 (xfr#12, to-chk=100/2000)',
+ dict(bytes_transferred=1234567890, percent=45, files_transferred=12, files_to_check=2000)),
+ (' 1234567890 45% 10.50MB/s 0:01:23', dict(bytes_transferred=1234567890, percent=45)),
+ ('Number of regular files transferred: 1,234', dict(files_transferred=1234)),
+ ('sending incremental file list', None),
+ ('', None),
+])
+def test_parse_progress(line, expected):
+ assert backup.parse_progress(line) == expected
+
+
+@pytest.fixture
+def fake_mounts(test_directory, monkeypatch, tmp_path):
+ """Pretend `/backup` is a mounted drive with UUID `uuid-1` (device /dev/sdb1). The mount root is a
+ temporary directory standing in for /media, so folders can be made on the "drive"."""
+ mount_root = (tmp_path / 'media').resolve()
+ drive = mount_root / 'backup'
+ drive.mkdir(parents=True)
+ media = test_directory.resolve()
+ monkeypatch.setattr(backup, 'MOUNT_ROOT', mount_root)
+
+ state = dict(mounted=True, device=2, uuid_device='/dev/sdb1', source='/dev/sdb1', nested=None)
+ monkeypatch.setattr(backup, 'is_mount', lambda path: (path == drive and state['mounted'])
+ or path == state['nested'] or path == pathlib.Path('/'))
+ monkeypatch.setattr(backup, 'device_id', lambda path: {drive: state['device'], media: 1}.get(
+ pathlib.Path(path), 3 if pathlib.Path(path) == state['nested'] else 0))
+ monkeypatch.setattr(backup, 'uuid_device', lambda uuid: state['uuid_device'] if uuid == 'uuid-1' else None)
+ monkeypatch.setattr(backup, 'mount_source', lambda path: {str(drive): state['source']}.get(
+ path, '/dev/sdz1' if path == str(state['nested']) else None))
+ state['drive'] = drive
+ state['root'] = mount_root
+ return state
+
+
+def test_check_backup_target(test_directory, fake_mounts):
+ drive, root = str(fake_mounts['drive']), fake_mounts['root']
+ assert backup.check_backup_target(drive, 'uuid-1') is None
+
+ assert backup.check_backup_target(None, None) == 'No backup drive is configured'
+ assert backup.check_backup_target('backup', 'uuid-1') == 'backup is not a valid directory'
+ assert backup.check_backup_target(f'{drive}/../backup', 'uuid-1') == f'{drive}/../backup is not a valid directory'
+ assert backup.check_backup_target('/opt/backup', 'uuid-1') == f'/opt/backup is not under {root}'
+ (root / 'wrolpi_temp_onboarding').mkdir()
+ fake_mounts['nested'] = root / 'wrolpi_temp_onboarding'
+ assert backup.check_backup_target(str(root / 'wrolpi_temp_onboarding'), 'uuid-1') \
+ == f'{root}/wrolpi_temp_onboarding cannot be a backup drive'
+ fake_mounts['nested'] = None
+
+ fake_mounts['mounted'] = False
+ assert backup.check_backup_target(drive, 'uuid-1') == f'No drive is mounted at {drive}', \
+ 'an unplugged drive leaves a directory on the system drive'
+ fake_mounts['mounted'] = True
+
+ fake_mounts['device'] = 1
+ assert backup.check_backup_target(drive, 'uuid-1') == 'The backup drive is the media drive'
+ fake_mounts['device'] = 0
+ assert backup.check_backup_target(drive, 'uuid-1') == 'The backup drive is the system drive'
+ fake_mounts['device'] = 2
+
+ fake_mounts['source'] = '/dev/sdc1'
+ assert backup.check_backup_target(drive, 'uuid-1') \
+ == f'The chosen drive is not mounted at {drive}', 'another drive mounted at the same place'
+ fake_mounts['source'] = '/dev/sdb1'
+ assert backup.check_backup_target(drive, 'uuid-2') \
+ == f'The chosen drive is not mounted at {drive}', 'the drive is not plugged in'
+
+
+def test_check_backup_target_folder(test_directory, fake_mounts):
+ """The backup may go to a folder on the chosen drive; the drive is checked through the folder's mount point."""
+ drive = fake_mounts['drive']
+ (drive / 'wrolpi').mkdir()
+ assert backup.check_backup_target(str(drive / 'wrolpi'), 'uuid-1') is None
+ assert backup.check_backup_target(str(drive / 'not' / 'made' / 'yet'), 'uuid-1') is None, \
+ 'the backup creates the folder'
+
+ # The folder is checked through the drive: wrong drive, unplugged drive.
+ assert backup.check_backup_target(str(drive / 'wrolpi'), 'uuid-2') \
+ == f'The chosen drive is not mounted at {drive}'
+ fake_mounts['mounted'] = False
+ assert backup.check_backup_target(str(drive / 'wrolpi'), 'uuid-1') == f'No drive is mounted at {drive}/wrolpi', \
+ 'a folder left on the system drive by an unplugged drive is not written'
+ fake_mounts['mounted'] = True
+
+ # Another drive mounted inside the chosen drive is not the chosen drive.
+ (drive / 'other').mkdir()
+ fake_mounts['nested'] = drive / 'other'
+ assert backup.check_backup_target(str(drive / 'other' / 'wrolpi'), 'uuid-1') \
+ == f'The chosen drive is not mounted at {drive}/other'
+
+
+def test_mount_source(monkeypatch, tmp_path):
+ mounts = tmp_path / 'mounts'
+ mounts.write_text('/dev/sda1 /media/wrolpi ext4 rw 0 0\n'
+ '/dev/sdb1 /media/my\\040backup ext4 rw 0 0\n'
+ '/dev/sdc1 /media/stacked ext4 rw 0 0\n'
+ '/dev/sdd1 /media/stacked ext4 rw 0 0\n')
+ monkeypatch.setattr(backup, 'PROC_MOUNTS', mounts)
+ assert backup.mount_source('/media/my backup') == '/dev/sdb1'
+ assert backup.mount_source('/media/stacked') == '/dev/sdd1', 'the last mount is the visible one'
+ assert backup.mount_source('/media/missing') is None
+
+
+def test_snapshot_database(test_directory, tmp_path):
+ """The snapshot is a complete, consistent database, even with writes still in the live database's WAL."""
+ (test_directory / 'config').mkdir(exist_ok=True)
+ live = sqlite3.connect(test_directory / 'config/wrolpi.db')
+ live.execute('PRAGMA journal_mode=WAL')
+ live.execute('CREATE TABLE t (v TEXT)')
+ live.execute("INSERT INTO t VALUES ('in the WAL')")
+ live.commit()
+ assert (test_directory / 'config/wrolpi.db-wal').stat().st_size > 0
+
+ drive = tmp_path / 'drive'
+ (drive / 'config').mkdir(parents=True)
+ # A stale WAL from an older copy must not be applied to the new snapshot.
+ (drive / 'config/wrolpi.db-wal').write_bytes(b'stale')
+ try:
+ backup.snapshot_database(drive)
+ finally:
+ live.close()
+
+ assert not (drive / 'config/wrolpi.db-wal').exists()
+ assert not (drive / 'config/.wrolpi.db.backup').exists()
+ with sqlite3.connect(drive / 'config/wrolpi.db') as copy:
+ assert copy.execute('SELECT v FROM t').fetchall() == [('in the WAL',)]
+
+
+def test_snapshot_database_only_when_changed(async_client, test_directory, tmp_path):
+ """The 2.5 GB snapshot is written again only when the live database, or the drive's copy, has changed."""
+ (test_directory / 'config').mkdir(exist_ok=True)
+ database = test_directory / 'config/wrolpi.db'
+ with contextlib.closing(sqlite3.connect(database)) as live:
+ live.execute('PRAGMA journal_mode=WAL')
+ live.execute('CREATE TABLE t (v TEXT)')
+ live.commit()
+ drive = tmp_path / 'drive'
+ copy = drive / 'config/wrolpi.db'
+
+ assert backup.snapshot_database_if_changed(drive) is True
+ written = copy.stat().st_mtime_ns
+ assert backup.snapshot_database_if_changed(drive) is False
+ assert copy.stat().st_mtime_ns == written
+
+ # Reading opens an empty WAL; nothing changed.
+ with contextlib.closing(sqlite3.connect(database)) as reader:
+ reader.execute('SELECT * FROM t').fetchall()
+ assert backup.snapshot_database_if_changed(drive) is False
+
+ # A write still in the WAL is a change.
+ with contextlib.closing(sqlite3.connect(database)) as live:
+ live.execute("INSERT INTO t VALUES ('new')")
+ live.commit()
+ assert (test_directory / 'config/wrolpi.db-wal').stat().st_size > 0
+ assert backup.snapshot_database_if_changed(drive) is True
+ with contextlib.closing(sqlite3.connect(copy)) as db:
+ assert db.execute('SELECT v FROM t').fetchall() == [('new',)]
+
+ # The live database changed when the WAL was checkpointed on close.
+ backup.snapshot_database_if_changed(drive)
+ assert backup.snapshot_database_if_changed(drive) is False
+
+ # The drive's copy is missing, or replaced by something else.
+ copy.unlink()
+ assert backup.snapshot_database_if_changed(drive) is True
+ copy.write_bytes(b'not the snapshot')
+ assert backup.snapshot_database_if_changed(drive) is True
+ # Another drive has no snapshot yet.
+ assert backup.snapshot_database_if_changed(tmp_path / 'other') is True
+
+
+@requires_rsync
+@pytest.mark.asyncio
+async def test_run_backup(async_client, test_directory, backup_drive, events_fixture):
+ """Files and configs are copied with their modification times; the database is a snapshot; hard-link directories
+ are not copied; nothing on the backup drive is deleted; a second backup copies nothing."""
+ files = {
+ 'videos/channel/video.mp4': b'video',
+ 'videos/channel/video.info.json': b'{}',
+ 'config/tags.yaml': b'tags: {}\n',
+ 'tags/one/video.mp4': b'video',
+ 'playlists/0001_video.mp4': b'video',
+ 'videos/channel/.video.mp4.partial': b'vid',
+ }
+ for path, content in files.items():
+ (test_directory / path).parent.mkdir(parents=True, exist_ok=True)
+ (test_directory / path).write_bytes(content)
+ os.utime(test_directory / path, (1_700_000_000, 1_700_000_000))
+ with sqlite3.connect(test_directory / 'config/wrolpi.db') as db:
+ db.execute('CREATE TABLE t (v TEXT)')
+ db.execute("INSERT INTO t VALUES ('row')")
+ # Only on the backup drive: never deleted.
+ (backup_drive / 'old.txt').write_text('deleted from the media directory long ago')
+
+ status = await backup.run_backup()
+
+ assert status['state'] == 'idle', status['last_error']
+ assert status['last_success_at']
+ for path in ('videos/channel/video.mp4', 'videos/channel/video.info.json', 'config/tags.yaml'):
+ assert (backup_drive / path).read_bytes() == files[path]
+ assert (backup_drive / path).stat().st_mtime == 1_700_000_000
+ assert not (backup_drive / 'tags').exists()
+ assert not (backup_drive / 'playlists').exists()
+ assert not (backup_drive / 'videos/channel/.video.mp4.partial').exists()
+ assert (backup_drive / 'old.txt').exists()
+ with sqlite3.connect(backup_drive / 'config/wrolpi.db') as copy:
+ assert copy.execute('SELECT v FROM t').fetchall() == [('row',)]
+ assert status['files_transferred'] == 3
+ assert status['cancel_requested'] is False
+ events_fixture.assert_has_event('backup_completed')
+
+ status = await backup.run_backup()
+ assert status['state'] == 'idle', status['last_error']
+ assert status['files_transferred'] == 0
+
+
+@requires_rsync
+@pytest.mark.asyncio
+async def test_run_backup_excluded_directories(async_client, test_directory, backup_drive):
+ """Excluded folders are not copied, even when their names hold rsync wildcards; what the drive already has in
+ them is kept."""
+ files = ['Movies/a.mp4', 'Movies [HD]/b.mp4', 'Movies2/c.mp4', 'videos/Movies/d.mp4', 'Mo*/e.mp4']
+ for path in files:
+ (test_directory / path).parent.mkdir(parents=True, exist_ok=True)
+ (test_directory / path).write_bytes(b'file')
+ (test_directory / 'config').mkdir(exist_ok=True)
+ with sqlite3.connect(test_directory / 'config/wrolpi.db') as db:
+ db.execute('CREATE TABLE t (v TEXT)')
+ (backup_drive / 'Movies').mkdir()
+ (backup_drive / 'Movies/old.mp4').write_bytes(b'old')
+ get_sync_config()._config['backup_excluded_directories'] = ['Movies', 'Movies [HD]', 'Mo*']
+
+ status = await backup.run_backup()
+
+ assert status['state'] == 'idle', status['last_error']
+ assert not (backup_drive / 'Movies/a.mp4').exists()
+ assert not (backup_drive / 'Movies [HD]').exists()
+ assert not (backup_drive / 'Mo*').exists()
+ assert (backup_drive / 'Movies2/c.mp4').read_bytes() == b'file', 'only the folder itself is excluded'
+ assert (backup_drive / 'videos/Movies/d.mp4').read_bytes() == b'file', 'excludes are anchored'
+ assert (backup_drive / 'Movies/old.mp4').read_bytes() == b'old', 'nothing is deleted'
+
+
+@requires_rsync
+@pytest.mark.asyncio
+async def test_run_backup_to_folder(async_client, test_directory, backup_drive):
+ """A backup to a folder on the drive creates the folder and mirrors the media directory inside it."""
+ (test_directory / 'videos').mkdir()
+ (test_directory / 'config').mkdir(exist_ok=True)
+ (test_directory / 'videos/video.mp4').write_bytes(b'video')
+ with sqlite3.connect(test_directory / 'config/wrolpi.db') as db:
+ db.execute('CREATE TABLE t (v TEXT)')
+ get_sync_config()._config.update(backup_directory=str(backup_drive / 'wrolpi' / 'library'))
+
+ status = await backup.run_backup()
+
+ assert status['state'] == 'idle', status['last_error']
+ assert (backup_drive / 'wrolpi/library/videos/video.mp4').read_bytes() == b'video'
+ assert (backup_drive / 'wrolpi/library/config/wrolpi.db').is_file()
+ assert not (backup_drive / 'videos').exists()
+
+
+@requires_rsync
+@pytest.mark.asyncio
+async def test_run_backup_stops_when_drive_disappears(async_client, test_directory, backup_drive, monkeypatch):
+ """A drive that is unplugged during a backup is not written to again: rsync is stopped."""
+ (test_directory / 'a.txt').write_text('a')
+ monkeypatch.setattr(backup, 'MOUNT_CHECK_SECONDS', 0)
+ monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: 'No drive is mounted at /media/x')
+ monkeypatch.setattr(backup, 'backup_skip_reason', lambda: None)
+
+ status = await backup.run_backup()
+
+ assert status['state'] == 'error'
+ assert 'No drive is mounted' in status['last_error']
+
+
+@pytest.mark.asyncio
+async def test_run_backup_skips(async_client, test_wrolpi_config):
+ status = await backup.run_backup()
+ assert status['last_skip_reason'] == 'No backup drive is configured'
+ assert status['started_at'] is None
+
+ get_sync_config()._config.update(backup_directory='/media/backup', backup_device_uuid='uuid-1')
+ get_wrolpi_config()._config.update(wrol_mode=True)
+ status = await backup.run_backup()
+ assert status['last_skip_reason'] == 'WROL Mode is enabled'
+
+
+@pytest.mark.asyncio
+async def test_backup_tick(async_client, test_wrolpi_config, monkeypatch):
+ calls = list()
+
+ async def fake_run_backup():
+ calls.append(True)
+
+ monkeypatch.setattr(backup, 'run_backup', fake_run_backup)
+
+ backup.request_backup()
+ await backup.backup_tick()
+ assert calls == [], 'no backup drive'
+ assert get_backup_status()['last_skip_reason'] == 'No backup drive is configured'
+
+ get_sync_config()._config.update(backup_directory='/media/backup', backup_device_uuid='uuid-1')
+ await backup.backup_tick()
+ assert calls == [], 'nothing requested and no window'
+
+ backup.request_backup()
+ await backup.backup_tick()
+ assert calls == [True]
+ assert get_backup_status()['requested'] is False
+
+ get_wrolpi_config()._config.update(wrol_mode=True)
+ backup.request_backup()
+ await backup.backup_tick()
+ assert calls == [True]
+ assert get_backup_status()['last_skip_reason'] == 'WROL Mode is enabled'
+
+
+def test_backup_is_due_hourly_in_window(test_wrolpi_config):
+ """Within the backup window, a backup repeats an hour after the last one finished."""
+ get_sync_config()._config.update(backup_window_start='01:00', backup_window_end='05:00')
+ local_now = datetime(2026, 11, 5, 3, 0, tzinfo=timezone.utc)
+ never = dict(finished_at=None)
+ recent = dict(finished_at=local_now.replace(hour=2, minute=30).isoformat())
+ earlier = dict(finished_at=local_now.replace(hour=1, minute=55).isoformat())
+ assert backup.backup_is_due(never, local_now)
+ assert not backup.backup_is_due(recent, local_now)
+ assert backup.backup_is_due(earlier, local_now)
+ assert not backup.backup_is_due(never, local_now.replace(hour=6)), 'outside the window'
+
+
+def test_backup_and_sync_windows_are_separate(test_wrolpi_config):
+ """The sync window (a secondary's sync) does not start backups, and the backup window does not start syncs."""
+ from wrolpi.sync.lib import sync_is_due
+ local_now = datetime(2026, 11, 5, 3, 0, tzinfo=timezone.utc)
+ never = dict(finished_at=None)
+
+ get_sync_config()._config.update(window_start='01:00', window_end='05:00')
+ assert sync_is_due(never, local_now)
+ assert not backup.backup_is_due(never, local_now)
+
+ get_sync_config()._config.update(window_start=None, window_end=None,
+ backup_window_start='02:00', backup_window_end='04:00')
+ assert not sync_is_due(never, local_now)
+ assert backup.backup_is_due(never, local_now)
+
+
+def test_request_backup_cancel(async_client):
+ update_backup_status(state='running')
+ backup.request_backup_cancel()
+ assert get_backup_status()['cancel_requested'] is True
+
+
+@pytest.mark.asyncio
+async def test_last_success_is_kept(async_client, test_directory, test_wrolpi_config, monkeypatch, await_switches):
+ """When the last backup succeeded is saved in sync.yaml, and known again after a restart (its status is in
+ memory)."""
+ async def fake_rsync(command, target, uuid):
+ return 0, []
+
+ monkeypatch.setattr(backup, 'backup_skip_reason', lambda: None)
+ monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: None)
+ monkeypatch.setattr(backup, '_run_rsync', fake_rsync)
+ monkeypatch.setattr(backup, 'snapshot_database', lambda target: None)
+ drive = test_directory / 'drive'
+ get_sync_config()._config.update(backup_directory=str(drive), backup_device_uuid='uuid-1')
+
+ status = await backup.run_backup()
+ await await_switches()
+
+ assert status['last_success_at']
+ saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text())
+ assert saved['backup_last_success_at'] == status['last_success_at']
+
+ # A restart: the status is new, the config is imported.
+ update_backup_status(last_success_at=None)
+ get_sync_config().import_config()
+ assert get_backup_status()['last_success_at'] == status['last_success_at']
+
+
+@pytest.mark.asyncio
+async def test_failed_backup_keeps_last_success(async_client, test_directory, test_wrolpi_config, monkeypatch):
+ async def failing_rsync(command, target, uuid):
+ return 12, ['rsync: connection unexpectedly closed']
+
+ monkeypatch.setattr(backup, 'backup_skip_reason', lambda: None)
+ monkeypatch.setattr(backup, '_run_rsync', failing_rsync)
+ get_sync_config()._config.update(backup_directory=str(test_directory / 'drive'), backup_device_uuid='uuid-1',
+ backup_last_success_at='2026-11-01T02:00:00+00:00')
+
+ status = await backup.run_backup()
+
+ assert status['state'] == 'error'
+ assert get_sync_config().backup_last_success_at == '2026-11-01T02:00:00+00:00'
diff --git a/wrolpi/sync/test/test_engine.py b/wrolpi/sync/test/test_engine.py
new file mode 100644
index 00000000..218d2294
--- /dev/null
+++ b/wrolpi/sync/test/test_engine.py
@@ -0,0 +1,531 @@
+import os
+import pathlib
+from collections import namedtuple
+
+import pytest
+import yaml
+
+from wrolpi.common import get_wrolpi_config, set_test_media_directory
+from wrolpi.db import get_db_session
+from wrolpi.errors import InvalidConfig
+from wrolpi.files.models import FileGroup
+from wrolpi.sync import lib
+from wrolpi.sync.config import get_sync_config
+from wrolpi.sync.conftest import manifest_page
+from wrolpi.sync.status import get_sync_status, update_sync_status
+from wrolpi.tags import get_tags_config, Tag, TagFile
+
+
+@pytest.mark.parametrize('path, allowed', [
+ ('videos/channel/video.mp4', True),
+ ('a.txt', True),
+ ('../escape.txt', False),
+ ('videos/../../escape.txt', False),
+ ('/etc/passwd', False),
+ ('videos/.hidden/video.mp4', False),
+ ('.hidden.txt', False),
+ ('config/wrolpi.db', False),
+ ('config/tags.yaml', False),
+ ('tags/one/video.mp4', False),
+ # A drive that ignores case (exFAT, NTFS, macOS) would write these into the real config and tags folders.
+ ('Config/wrolpi.db', False),
+ ('CONFIG/tags.yaml', False),
+ ('Tags/one/video.mp4', False),
+ ('videos\\video.mp4', False),
+ ('', False),
+])
+def test_resolve_destination(test_directory, path, allowed):
+ destination = lib.resolve_destination(test_directory, path)
+ if allowed:
+ assert destination == test_directory / path
+ else:
+ assert destination is None
+
+
+def test_resolve_destination_refuses_symlinks(test_directory):
+ """A sync never writes through a symlink; it could point outside the media directory."""
+ outside = test_directory / 'outside'
+ outside.mkdir()
+ (test_directory / 'videos').symlink_to(outside)
+ assert lib.resolve_destination(test_directory, 'videos/video.mp4') is None
+
+ (test_directory / 'docs').mkdir()
+ (test_directory / 'docs/link.txt').symlink_to(outside / 'target.txt')
+ assert lib.resolve_destination(test_directory, 'docs/link.txt') is None
+
+
+def test_needs_download(test_directory):
+ path = test_directory / 'a.txt'
+ file = lib.ManifestFile('a.txt', 5, 1_700_000_000.0)
+ assert lib.needs_download(path, file), 'a missing file is downloaded'
+
+ path.write_text('12345')
+ os.utime(path, (file.mtime, file.mtime))
+ assert not lib.needs_download(path, file)
+
+ os.utime(path, (file.mtime + 1.5, file.mtime + 1.5))
+ assert not lib.needs_download(path, file), 'within the 2 second tolerance of exFAT'
+
+ os.utime(path, (file.mtime + 3, file.mtime + 3))
+ assert lib.needs_download(path, file), 'a different modification time is downloaded'
+
+ os.utime(path, (file.mtime, file.mtime))
+ path.write_text('1234')
+ os.utime(path, (file.mtime, file.mtime))
+ assert lib.needs_download(path, file), 'a different size is downloaded'
+
+
+Primary = namedtuple('Primary', 'directory url state')
+
+
+@pytest.fixture
+def primary(test_directory, secondary_directory, primary_media_server) -> Primary:
+ """A primary whose media is `test_directory`, served like Caddy."""
+ url, state = primary_media_server(test_directory)
+ return Primary(test_directory, url, state)
+
+
+def write_primary_file(primary: Primary, path: str, content: bytes, mtime: float = 1_700_000_000.0) \
+ -> lib.ManifestFile:
+ file = primary.directory / path
+ file.parent.mkdir(parents=True, exist_ok=True)
+ file.write_bytes(content)
+ os.utime(file, (mtime, mtime))
+ return lib.ManifestFile(path, len(content), mtime)
+
+
+@pytest.mark.asyncio
+async def test_download_file(primary, secondary_directory):
+ """A downloaded file has the primary's bytes and modification time, and no partial is left."""
+ file = write_primary_file(primary, 'videos/channel/video name.mp4', b'video bytes' * 1000)
+ destination = secondary_directory / file.path
+
+ async with lib.PrimaryClient(primary.url) as client:
+ received = await lib.download_file(client, file, destination)
+
+ assert received == file.size
+ assert destination.read_bytes() == (primary.directory / file.path).read_bytes()
+ assert destination.stat().st_mtime == file.mtime
+ assert not lib.partial_path(destination).exists()
+ assert not lib.needs_download(destination, file)
+
+
+@pytest.mark.asyncio
+async def test_download_file_resumes(primary, secondary_directory):
+ """A partial left by an interrupted download is resumed with a Range request."""
+ content = bytes(range(256)) * 400
+ file = write_primary_file(primary, 'big.bin', content)
+ destination = secondary_directory / file.path
+ lib.partial_path(destination).write_bytes(content[:30_000])
+
+ async with lib.PrimaryClient(primary.url) as client:
+ received = await lib.download_file(client, file, destination)
+
+ assert received == len(content) - 30_000
+ assert ('/media/big.bin', 'bytes=30000-') in primary.state['requests']
+ assert destination.read_bytes() == content
+
+
+@pytest.mark.asyncio
+async def test_download_file_restarts_without_range(primary, secondary_directory):
+ """A server that ignores Range sends the whole file; the partial is replaced, not appended to."""
+ primary.state['honor_range'] = False
+ content = b'abcdefghij' * 100
+ file = write_primary_file(primary, 'a.bin', content)
+ destination = secondary_directory / file.path
+ lib.partial_path(destination).write_bytes(content[:500])
+
+ async with lib.PrimaryClient(primary.url) as client:
+ await lib.download_file(client, file, destination)
+
+ assert destination.read_bytes() == content
+
+
+@pytest.mark.asyncio
+async def test_download_file_changed_on_primary(primary, secondary_directory):
+ """A file whose size changed after the manifest was read is not written; the next sync gets it."""
+ file = write_primary_file(primary, 'a.txt', b'old')
+ (primary.directory / 'a.txt').write_bytes(b'new and longer')
+ destination = secondary_directory / file.path
+
+ async with lib.PrimaryClient(primary.url) as client:
+ with pytest.raises(lib.SyncError):
+ await lib.download_file(client, file, destination)
+
+ assert not destination.exists()
+
+
+@pytest.mark.asyncio
+async def test_download_file_larger_than_manifest(primary, secondary_directory):
+ """A primary that sends more bytes than its manifest said (no Content-Length to catch it) is not written past
+ the manifest's size, which is what the free-space check allowed for."""
+ primary.state['omit_length'] = True
+ write_primary_file(primary, 'a.bin', b'x' * 5 * lib.DOWNLOAD_CHUNK_SIZE)
+ file = lib.ManifestFile('a.bin', 3000, 1_700_000_000.0)
+ destination = secondary_directory / file.path
+
+ async with lib.PrimaryClient(primary.url) as client:
+ with pytest.raises(lib.SyncError, match='larger than the primary said'):
+ await lib.download_file(client, file, destination)
+
+ assert not destination.exists()
+ assert not lib.partial_path(destination).exists()
+
+
+@pytest.mark.asyncio
+async def test_download_file_disk_full(primary, secondary_directory, monkeypatch):
+ """A file that would leave less than the minimum free space is not downloaded; its partial is kept."""
+ file = write_primary_file(primary, 'a.bin', b'x' * 1000)
+ destination = secondary_directory / file.path
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ lib.partial_path(destination).write_bytes(b'x' * 10)
+
+ usage = namedtuple('usage', 'total used free')
+ monkeypatch.setattr(lib.shutil, 'disk_usage', lambda _: usage(10, 10, lib.MIN_FREE_BYTES))
+
+ async with lib.PrimaryClient(primary.url) as client:
+ with pytest.raises(lib.SyncDiskFull):
+ await lib.download_file(client, file, destination)
+
+ assert lib.partial_path(destination).read_bytes() == b'x' * 10
+ assert primary.state['requests'] == []
+
+
+def capture_manifest() -> list:
+ pages = list()
+ after_id = 0
+ with get_db_session() as session:
+ while True:
+ page = lib.build_manifest_page(session, after_id, limit=2)
+ pages.append(page)
+ if page['next_after_id'] is None:
+ return pages
+ after_id = page['next_after_id']
+
+
+@pytest.mark.asyncio
+async def test_run_sync(async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api,
+ refresh_files, tag_factory, test_tags_config, events_fixture, await_switches):
+ """A secondary gets the primary's files, then its tags.yaml, and its FileGroups are tagged like the primary's."""
+ write_primary_file(primary, 'docs/notes.txt', b'some notes', mtime=1_700_000_001.0)
+ write_primary_file(primary, 'docs/more notes.txt', b'more notes', mtime=1_700_000_002.0)
+ write_primary_file(primary, 'videos/channel/video.mp4', b'not really a video', mtime=1_700_000_003.0)
+ write_primary_file(primary, 'videos/channel/video.info.json', b'{}', mtime=1_700_000_004.0)
+ await refresh_files()
+
+ tag = await tag_factory('one')
+ notes = test_session.query(FileGroup).filter_by(primary_path=primary.directory / 'docs/notes.txt').one()
+ notes.add_tag(test_session, tag.id)
+ test_session.commit()
+ get_tags_config().dump_config()
+ await await_switches()
+ primary_tags = (primary.directory / 'config/tags.yaml').read_bytes()
+ assert b'docs/notes.txt' in primary_tags
+
+ pages = capture_manifest()
+ assert len(pages) >= 2, 'the manifest is paged'
+ fake_primary_api(pages, primary.directory)
+
+ # The secondary is another machine with its own (empty) database.
+ test_session.query(TagFile).delete()
+ test_session.query(Tag).delete()
+ test_session.query(FileGroup).delete()
+ test_session.commit()
+
+ # This WROLPi becomes the secondary; it has its own (empty) tags.yaml.
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+ (secondary_directory / 'config/tags.yaml').write_text('tags: {}\ntag_files: []\nversion: 1\n')
+
+ status = await lib.run_sync()
+
+ assert status['state'] == 'idle', status['last_error']
+ assert status['files_failed'] == 0
+ assert status['files_downloaded'] == 4
+ assert status['last_success_at']
+ for path in ('docs/notes.txt', 'docs/more notes.txt', 'videos/channel/video.mp4',
+ 'videos/channel/video.info.json'):
+ copy = secondary_directory / path
+ assert copy.read_bytes() == (primary.directory / path).read_bytes()
+ assert copy.stat().st_mtime == (primary.directory / path).stat().st_mtime
+
+ # The secondary's own tags.yaml is kept in today's backup.
+ backups = list((secondary_directory / 'config/backup').glob('tags-*.yaml'))
+ assert len(backups) == 1 and b'docs/notes.txt' not in backups[0].read_bytes()
+
+ # The primary's tags.yaml is copied byte for byte, and imported.
+ assert (secondary_directory / 'config/tags.yaml').read_bytes() == primary_tags
+ assert 'tags.yaml' in status['configs_imported']
+ with get_db_session() as session:
+ copy = session.query(FileGroup).filter_by(primary_path=secondary_directory / 'docs/notes.txt').one()
+ assert copy.tag_names == ['one']
+ # Files of a group arrive together, and are grouped again on the secondary.
+ video = session.query(FileGroup).filter_by(directory=secondary_directory / 'videos/channel').one()
+ assert {pathlib.Path(i['path']).name for i in video.files} == {'video.mp4', 'video.info.json'}
+
+ events_fixture.assert_has_event('sync_completed')
+
+ # Nothing changed, so nothing is downloaded or imported again.
+ primary.state['requests'].clear()
+ status = await lib.run_sync()
+ assert status['files_to_download'] == 0
+ assert status['configs_imported'] == []
+ assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')]
+
+
+@pytest.mark.asyncio
+async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary,
+ fake_primary_api):
+ """A manifest cannot make a secondary write outside its library, or over its configs and database."""
+ good = write_primary_file(primary, 'good.txt', b'good')
+ write_primary_file(primary, 'config/wrolpi.db', b'not a database')
+ pages = [manifest_page(
+ (good.path, good.size, good.mtime),
+ ('config/wrolpi.db', 14, 1_700_000_000.0),
+ ('../escape.txt', 4, 1_700_000_000.0),
+ ('tags/one/good.txt', 4, 1_700_000_000.0),
+ )]
+ fake_primary_api(pages, primary.directory)
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+
+ status = await lib.run_sync()
+
+ assert status['files_failed'] == 3
+ assert (secondary_directory / 'good.txt').read_bytes() == b'good'
+ assert not (secondary_directory / 'config/wrolpi.db').exists()
+ assert not (secondary_directory.parent / 'escape.txt').exists()
+ assert not (secondary_directory / 'tags').exists()
+
+
+@pytest.mark.asyncio
+async def test_run_sync_excluded_directories(async_client, primary, secondary_directory, make_secondary,
+ fake_primary_api):
+ """A secondary never downloads files inside its excluded folders, and keeps what it already has in them."""
+ wanted = [write_primary_file(primary, i, b'wanted') for i in ('Movies2/a.mp4', 'docs/Movies/b.txt', 'c.txt')]
+ excluded = [write_primary_file(primary, i, b'excluded') for i in ('Movies/a.mp4', 'Movies/deep/b.mp4',
+ 'Movies [HD]/c.mp4')]
+ pages = [manifest_page(*((i.path, i.size, i.mtime) for i in wanted + excluded))]
+ fake_primary_api(pages, primary.directory)
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+ get_sync_config()._config['sync_excluded_directories'] = ['Movies', 'Movies [HD]']
+ # Copied before the folder was excluded.
+ (secondary_directory / 'Movies').mkdir()
+ (secondary_directory / 'Movies/old.mp4').write_bytes(b'old')
+
+ status = await lib.run_sync()
+
+ assert status['last_error'] is None
+ assert status['files_excluded'] == 3
+ assert status['files_downloaded'] == 3
+ for file in wanted:
+ assert (secondary_directory / file.path).read_bytes() == b'wanted'
+ for file in excluded:
+ assert not (secondary_directory / file.path).exists()
+ assert (secondary_directory / 'Movies/old.mp4').read_bytes() == b'old', 'nothing is deleted'
+
+
+@pytest.mark.parametrize('values, expected', [
+ ([], []),
+ (['Movies'], ['Movies']),
+ (['/Movies/', ' Movies ', 'Movies//'], ['Movies']),
+ (['videos/Movies', 'videos'], ['videos']),
+ (['Movies', 'Movies2', 'Movies/deep'], ['Movies', 'Movies2']),
+ (['a//b'], ['a/b']),
+ (['Movies [HD]'], ['Movies [HD]']),
+])
+def test_normalize_excluded_directories(values, expected):
+ assert lib.normalize_excluded_directories(values) == expected
+
+
+@pytest.mark.parametrize('value', ['', '/', '..', 'a/../b', '../a', 'a/./b', '.hidden', 'a/.hidden', 'config',
+ 'config/x', 'tags', 'Config', 'TAGS/x', 'a\\b', None])
+def test_normalize_excluded_directories_rejects(value):
+ with pytest.raises(InvalidConfig):
+ lib.normalize_excluded_directories([value])
+
+
+def test_is_excluded():
+ assert lib.is_excluded('Movies/a.mp4', {'Movies'})
+ assert lib.is_excluded('Movies', {'Movies'})
+ assert not lib.is_excluded('Movies2/a.mp4', {'Movies'})
+ assert not lib.is_excluded('docs/Movies/a.mp4', {'Movies'})
+ assert lib.is_excluded('docs/Movies/a.mp4', {'docs/Movies'})
+
+
+def test_list_local_directories(test_directory):
+ for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'zims', 'lost+found', 'Tags'):
+ try:
+ (test_directory / i).mkdir(parents=True)
+ except FileExistsError:
+ pass # 'Tags' is 'tags' on a filesystem that ignores case.
+ (test_directory / 'file.txt').write_text('a file')
+ (test_directory / 'link').symlink_to(test_directory / 'Movies')
+
+ assert lib.list_local_directories('') == [
+ dict(name='Movies', path='Movies'), dict(name='videos', path='videos'), dict(name='zims', path='zims')]
+ assert lib.list_local_directories('/videos/') == [dict(name='channel', path='videos/channel')]
+ for path in ('../', 'videos/../..', 'file.txt', 'missing', 'link'):
+ with pytest.raises(InvalidConfig):
+ lib.list_local_directories(path)
+
+
+@pytest.mark.asyncio
+async def test_list_primary_directories(primary, secondary_directory, make_secondary):
+ for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'):
+ (primary.directory / i).mkdir(parents=True)
+ (primary.directory / 'file.txt').write_text('a file')
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+
+ assert await lib.list_primary_directories('') == [
+ dict(name='Movies', path='Movies'), dict(name='videos', path='videos')]
+ assert await lib.list_primary_directories('videos') == [dict(name='channel', path='videos/channel')]
+ with pytest.raises(InvalidConfig):
+ await lib.list_primary_directories('../etc')
+
+
+@pytest.mark.asyncio
+async def test_run_sync_only_on_a_secondary(async_client, test_wrolpi_config, fake_primary_api):
+ status = await lib.run_sync()
+ assert status['state'] == 'idle'
+ assert status['last_skip_reason'] == 'This WROLPi is not a secondary'
+ assert status['started_at'] is None
+
+
+@pytest.mark.asyncio
+async def test_run_sync_refuses_a_secondary_primary(async_client, primary, secondary_directory, make_secondary,
+ fake_primary_api):
+ """Secondaries do not chain; a secondary's library may be stale."""
+ fake_primary_api([manifest_page()], primary.directory, role='secondary')
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+
+ status = await lib.run_sync()
+ assert status['state'] == 'error'
+ assert 'is not a primary' in status['last_error']
+
+
+@pytest.mark.asyncio
+async def test_run_sync_cancel(async_client, primary, secondary_directory, make_secondary, fake_primary_api):
+ """A cancelled sync stops between pages; the files of finished pages stay."""
+ first = write_primary_file(primary, 'first.txt', b'first')
+ second = write_primary_file(primary, 'second.txt', b'second')
+ pages = [manifest_page((first.path, first.size, first.mtime), next_after_id=1, max_id=2),
+ manifest_page((second.path, second.size, second.mtime), max_id=2)]
+ fake_primary_api(pages, primary.directory)
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+
+ original = lib._refresh
+
+ async def refresh_then_cancel(paths):
+ await original(paths)
+ lib.request_sync_cancel()
+
+ lib._refresh = refresh_then_cancel
+ try:
+ status = await lib.run_sync()
+ finally:
+ lib._refresh = original
+
+ assert status['last_error'] == 'The sync was cancelled'
+ assert status['state'] == 'idle'
+ assert (secondary_directory / 'first.txt').exists()
+ assert not (secondary_directory / 'second.txt').exists()
+
+
+@pytest.mark.asyncio
+async def test_sync_configs_wrolpi_yaml(async_client, primary, secondary_directory, make_secondary):
+ """Only the library settings of the primary's wrolpi.yaml are applied; the secondary keeps its own role, hotspot,
+ and the like."""
+ primary_config = dict(
+ videos_destination='my videos/%(channel_name)s',
+ ignored_directories=['config', 'tags', str(primary.directory / 'private'), '/elsewhere/directory'],
+ hotspot_ssid='primary hotspot',
+ role='primary',
+ nav_color='red',
+ version=12,
+ )
+ (primary.directory / 'config').mkdir(exist_ok=True)
+ (primary.directory / 'config/wrolpi.yaml').write_text(yaml.dump(primary_config))
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+
+ async with lib.PrimaryClient(primary.url) as client:
+ await lib.sync_configs(client, str(primary.directory), reimport=False)
+
+ config = get_wrolpi_config()
+ assert config.videos_destination == 'my videos/%(channel_name)s'
+ assert config.ignored_directories == ['config', 'tags', 'private']
+ assert get_sync_config().role == 'secondary', 'the role is in sync.yaml, which is never synced'
+ assert config.hotspot_ssid == 'WROLPi'
+ assert config.nav_color == 'violet'
+
+
+@pytest.mark.asyncio
+async def test_sync_configs_inventories(async_client, primary, secondary_directory, make_secondary):
+ """Inventories are a directory of configs; each one is copied."""
+ from modules.inventory.common import get_inventory_configs
+
+ inventory_directory = primary.directory / 'config/inventory'
+ inventory_directory.mkdir(parents=True)
+ pantry = dict(version=3, slug='pantry', name='Pantry', type='food', fields=[], items=[])
+ (inventory_directory / 'pantry.yaml').write_text(yaml.dump(pantry))
+ (inventory_directory / '.hidden.yaml').write_text('ignored')
+ set_test_media_directory(secondary_directory)
+ make_secondary(primary.url)
+
+ async with lib.PrimaryClient(primary.url) as client:
+ changed = await lib._sync_inventory_directory(client)
+
+ assert changed
+ copy = get_inventory_configs().get_directory() / 'pantry.yaml'
+ assert yaml.safe_load(copy.read_text())['name'] == 'Pantry'
+ assert not (copy.parent / '.hidden.yaml').exists()
+
+
+@pytest.mark.asyncio
+async def test_sync_tick(async_client, test_wrolpi_config, make_secondary, monkeypatch):
+ """A requested sync runs; WROL Mode drops the request."""
+ calls = list()
+
+ async def fake_run_sync(force=False):
+ calls.append(force)
+
+ monkeypatch.setattr(lib, 'run_sync', fake_run_sync)
+
+ await lib.sync_tick()
+ assert calls == [], 'a primary never syncs'
+
+ make_secondary()
+ await lib.sync_tick()
+ assert calls == [], 'nothing requested, no window'
+
+ lib.request_sync(force=True)
+ await lib.sync_tick()
+ assert calls == [True]
+ assert get_sync_status()['requested'] is False
+
+ get_wrolpi_config()._config['wrol_mode'] = True
+ lib.request_sync()
+ await lib.sync_tick()
+ assert calls == [True]
+ assert get_sync_status()['requested'] is False
+ assert get_sync_status()['last_skip_reason'] == 'WROL Mode is enabled'
+
+
+def test_request_sync_cancel(async_client):
+ update_sync_status(state='running')
+ lib.request_sync_cancel()
+ assert get_sync_status()['cancel_requested'] is True
+
+ update_sync_status(state='idle', cancel_requested=False, requested=True)
+ lib.request_sync_cancel()
+ assert get_sync_status()['requested'] is False
+ assert get_sync_status()['cancel_requested'] is False
+
+
+def test_partial_path():
+ assert lib.partial_path(pathlib.Path('/a/b/video.mp4')) == pathlib.Path('/a/b/.video.mp4.partial')
diff --git a/wrolpi/sync/test/test_manifest.py b/wrolpi/sync/test/test_manifest.py
new file mode 100644
index 00000000..7416dadc
--- /dev/null
+++ b/wrolpi/sync/test/test_manifest.py
@@ -0,0 +1,67 @@
+import os
+
+import pytest
+
+from wrolpi.files.models import FileGroup
+from wrolpi.sync import lib
+
+
+@pytest.mark.asyncio
+async def test_manifest_pages(async_client, test_session, test_directory, make_files_structure, refresh_files):
+ """The manifest lists every file of every FileGroup, a page of FileGroups at a time, in id order."""
+ make_files_structure({f'docs/{i}.txt': f'file {i}' for i in range(7)})
+ await refresh_files()
+
+ pages = list()
+ after_id = 0
+ while after_id is not None:
+ page = lib.build_manifest_page(test_session, after_id, limit=3)
+ pages.append(page)
+ after_id = page['next_after_id']
+
+ assert [len(i['groups']) for i in pages] == [3, 3, 1]
+ ids = [g['id'] for page in pages for g in page['groups']]
+ assert ids == sorted(ids)
+ assert pages[0]['max_id'] == max(ids)
+
+ files = {p['path']: p for page in pages for g in page['groups'] for p in g['paths']}
+ assert set(files) == {f'docs/{i}.txt' for i in range(7)}
+ stat = (test_directory / 'docs/3.txt').stat()
+ assert files['docs/3.txt']['size'] == stat.st_size
+ assert files['docs/3.txt']['mtime'] == stat.st_mtime
+
+
+def test_manifest_skips(test_session, test_directory):
+ """Files that are still downloading, hidden, missing, or outside the library are not listed."""
+ directory = test_directory / 'videos'
+ directory.mkdir()
+ for name in ('video.mp4', 'video.mp4.part', 'video.f137.mp4', 'video.mp4.aria2', '.video.mp4.partial',
+ 'video.en.vtt'):
+ (directory / name).write_text(name)
+ outside = test_directory.parent / f'{test_directory.name}-outside.txt'
+ outside.write_text('outside')
+ try:
+ group = FileGroup.from_paths(test_session, directory / 'video.mp4', directory / 'video.en.vtt')
+ group.files = [
+ *group.files,
+ dict(path='video.mp4.part'), dict(path='video.f137.mp4'), dict(path='video.mp4.aria2'),
+ dict(path='.video.mp4.partial'), dict(path='deleted.mp4'),
+ # Older rows stored absolute paths.
+ dict(path=str(directory / 'video.en.vtt')),
+ dict(path=str(outside)),
+ {},
+ ]
+ test_session.commit()
+
+ page = lib.build_manifest_page(test_session)
+ finally:
+ os.remove(outside)
+
+ paths = [p['path'] for g in page['groups'] for p in g['paths']]
+ assert sorted(set(paths)) == ['videos/video.en.vtt', 'videos/video.mp4']
+ assert page['next_after_id'] is None
+
+
+def test_manifest_empty(test_session, test_directory):
+ assert lib.build_manifest_page(test_session) == dict(groups=[], next_after_id=None, max_id=0,
+ limit=lib.MANIFEST_PAGE_LIMIT)
diff --git a/wrolpi/sync/test/test_role.py b/wrolpi/sync/test/test_role.py
new file mode 100644
index 00000000..2fc36922
--- /dev/null
+++ b/wrolpi/sync/test/test_role.py
@@ -0,0 +1,105 @@
+"""A secondary WROLPi never saves the configs it copies from its primary, and never downloads."""
+from datetime import datetime, time, timezone
+
+import pytest
+import yaml
+
+from wrolpi.common import get_wrolpi_config, SYNCED_CONFIG_FILE_NAMES, get_all_configs
+from wrolpi.config_version import IMPORT, SAVE, get_config_version, set_config_version
+from wrolpi.dates import is_within_daily_window
+from wrolpi.db import get_db_session
+from wrolpi.downloader import download_manager
+from wrolpi.sync import lib
+from wrolpi.sync.config import get_sync_config
+from wrolpi.tags import get_tags_config
+
+
+def test_synced_configs_exist(test_channels_config):
+ """Every synced config name is a real config (a typo would sync nothing, and save on a secondary)."""
+ assert SYNCED_CONFIG_FILE_NAMES - {'inventory/catalog.yaml'} <= set(get_all_configs())
+ assert 'repos.yaml' not in SYNCED_CONFIG_FILE_NAMES, 'git clones are not synced'
+ assert 'wrolpi.yaml' not in SYNCED_CONFIG_FILE_NAMES, 'wrolpi.yaml is merged, it holds local settings'
+ assert 'sync.yaml' not in SYNCED_CONFIG_FILE_NAMES, "a secondary's role is its own"
+ assert 'sync.yaml' in get_all_configs()
+
+
+@pytest.mark.asyncio
+async def test_secondary_does_not_save_synced_configs(async_client, test_session, test_tags_config, tag_factory,
+ make_secondary, await_switches):
+ await tag_factory('one')
+ get_tags_config().dump_config()
+ await await_switches()
+ primary_copy = test_tags_config.read_text()
+ assert 'one' in primary_copy
+
+ make_secondary()
+ await tag_factory('two')
+ get_tags_config().dump_config()
+ await await_switches()
+ assert test_tags_config.read_text() == primary_copy, 'a secondary must not save tags.yaml'
+
+ # wrolpi.yaml holds the secondary's own settings, and is saved.
+ get_wrolpi_config().update(dict(nav_color='red'))
+ await await_switches()
+ assert yaml.safe_load(get_wrolpi_config().get_file().read_text())['nav_color'] == 'red'
+
+
+@pytest.mark.asyncio
+async def test_secondary_does_not_save_inventories(async_client, test_inventory_configs, make_secondary,
+ await_switches):
+ from modules.inventory.common import get_inventory_configs
+ config = get_inventory_configs()
+ slug = config.create_inventory('Pantry', 'food')['slug']
+ pantry = config.get_file(slug)
+ before = pantry.read_text()
+
+ make_secondary()
+ config.replace(slug, dict(name='Changed'))
+ await await_switches()
+ assert pantry.read_text() == before, 'a secondary must not save inventories'
+
+
+def test_secondary_imports_synced_configs(async_client, test_tags_config, make_secondary):
+ """A secondary's database never outranks the primary's copy of a config: every import applies the file."""
+ with get_db_session(commit=True) as session:
+ set_config_version(session, 'tags.yaml', 10)
+ config = get_tags_config()
+ assert config.version_decision(file_version=3)[0] == SAVE
+
+ make_secondary()
+ assert config.version_decision(file_version=3)[0] == IMPORT
+
+
+def test_secondary_never_downloads(async_client, test_wrolpi_config, make_secondary):
+ assert download_manager.blocked_download_reason() != 'secondary role'
+ make_secondary()
+ assert download_manager.blocked_download_reason() == 'secondary role'
+
+
+@pytest.mark.parametrize('start, end, current, expected', [
+ ('01:00', '05:00', time(0, 59), False),
+ ('01:00', '05:00', time(1, 0), True),
+ ('01:00', '05:00', time(4, 59), True),
+ ('01:00', '05:00', time(5, 0), False),
+ ('22:00', '06:00', time(23, 0), True),
+ ('22:00', '06:00', time(3, 0), True),
+ ('22:00', '06:00', time(12, 0), False),
+])
+def test_is_within_daily_window(start, end, current, expected):
+ assert is_within_daily_window(start, end, current) is expected
+
+
+def test_sync_is_due(test_wrolpi_config):
+ config = get_sync_config()
+ local_now = datetime(2026, 11, 5, 2, 0, tzinfo=timezone.utc)
+ never = dict(finished_at=None)
+ assert not lib.sync_is_due(never, local_now), 'no window is "Sync now" only'
+
+ config._config.update(window_start='01:00', window_end='05:00')
+ assert lib.sync_is_due(never, local_now)
+ assert not lib.sync_is_due(never, local_now.replace(hour=6)), 'outside the window'
+
+ recent = dict(finished_at=local_now.replace(hour=1, minute=50).isoformat())
+ assert not lib.sync_is_due(recent, local_now), 'synced 10 minutes ago'
+ earlier = dict(finished_at=local_now.replace(hour=1, minute=40).isoformat())
+ assert lib.sync_is_due(earlier, local_now), 'synced 20 minutes ago'
diff --git a/wrolpi/test/test_api_fixtures.py b/wrolpi/test/test_api_fixtures.py
index b68c47fd..d76504d1 100644
--- a/wrolpi/test/test_api_fixtures.py
+++ b/wrolpi/test/test_api_fixtures.py
@@ -218,6 +218,28 @@ async def test_downloads_summary_fixture_matches_the_download_manager(async_clie
)
+@pytest.mark.parametrize('fixture_name, seed_name, response_name', [
+ ('syncStatusFixture', 'default_sync_status', 'SyncStatusResponse'),
+ ('backupStatusFixture', 'default_backup_status', 'BackupStatusResponse'),
+])
+def test_sync_status_fixture_matches_the_seed(fixture_name, seed_name, response_name):
+ """statusFixture's `sync` and `backup` are seeded by wrolpi.sync.status and declared in wrolpi.schema; the
+ fixture, the seed and the declaration must name the same keys."""
+ from dataclasses import fields
+ from wrolpi import schema
+ from wrolpi.sync import status
+
+ seeded = set(getattr(status, seed_name)())
+ declared = {i.name for i in fields(getattr(schema, response_name))}
+ assert seeded == declared, f'seed and {response_name} differ: {sorted(seeded ^ declared)}'
+
+ source = TEST_FIXTURES_JS.read_text()
+ block = re.search(rf'^export const {fixture_name} = .*?\n(.*?)\n \.\.\.overrides', source, re.S | re.M)
+ assert block, f'{fixture_name} is no longer in the expected form'
+ fixture = set(re.findall(r'^ ([a-z0-9_]+):', block.group(1), re.M))
+ assert seeded == fixture, f'{fixture_name} differs from the seed: {sorted(seeded ^ fixture)}'
+
+
@pytest.mark.asyncio
async def test_which_fixtures_the_spec_can_vouch_for(async_client):
"""Name the API-shaped fixtures this file does not cover, and why.
@@ -249,9 +271,10 @@ async def test_which_fixtures_the_spec_can_vouch_for(async_client):
source = TEST_FIXTURES_JS.read_text()
api_fixtures = set(re.findall(r'^export const ([a-zA-Z]+Fixture) =', source, re.M))
- # domainsFixture is a list built from domainFixture, and the *Context* fixtures are held to
+ # domainsFixture is a list built from domainFixture, syncStatusFixture and backupStatusFixture are
+ # statusFixture's `sync` and `backup` (test_sync_status_fixture_matches_the_seed holds them), and the *Context* fixtures are held to
# their React contexts by the frontend's own test-fixtures.test.js.
- api_fixtures -= {'domainsFixture'}
+ api_fixtures -= {'domainsFixture', 'syncStatusFixture', 'backupStatusFixture'}
api_fixtures = {n for n in api_fixtures if 'Context' not in n}
assert api_fixtures == set(checked) | set(unchecked), (