mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
A secondary syncs by rsync: the Controller runs it on its schedule, and both pages follow it (#995)
* A secondary syncs by rsync: the Controller runs it on its schedule, and both pages follow it The third part of replacing the sync. - The Controller syncs within the daily window (every 15 minutes after the last sync finished, in WROLPi's timezone), as the API did. The API's loop no longer runs the old engine, and its /api/sync/run and /api/sync/cancel ask the Controller. - The Sync page and the overlay that covers the app read the sync's status from the Controller, which is the only one that can answer while the API is stopped. The overlay is also shown when the settings cannot be read, as happens while the API is stopped. Sync Now and Stop go to the Controller. - The Controller's fallback page, where Caddy sends the browser while the API is down, shows the sync's progress and a Stop button. - New on the Sync page: how long files the primary deletes are kept (30 days by default, or never), and a note that an excluded folder's files are still listed on a secondary. The old engine's code is removed in the next part. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Remove SyncRunRequest, unused since a sync is asked of the Controller Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * In Docker, a sync's progress moves while a large file is copied Docker's logs hold a line until it ends, and rsync's progress for one large file is a single line rewritten with carriage returns, so the page showed 0% until the file was done (hours, for a large map). The rsync helper's output is now read by attaching to it, before it is started so that nothing is missed. Found by a sync between two Docker WROLPis. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * A sync is recovered before the schedule can start another, and an empty days value is refused Review found: - The schedule and the recovery of an interrupted sync started side by side when the Controller started. The state an interrupted sync leaves is due, so inside the window a new sync could start while the recovery restarted the API and wrote over the new sync's status. Every tick of the schedule now recovers first, and only a status of "running" with no sync holding the lock counts as interrupted. A sync records its outcome before it frees the lock, so a tick between the two cannot mistake it for an interrupted one. - The Controller may start before the drive is mounted. The empty record it read then was kept and saved over the real one, losing the last successful sync. The state file is now read again until it can be. - A cleared number of days was saved as 0, which empties the trash at the next sync. It is now refused. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
6772762ce9
commit
8cfe36b3ea
13 changed files with 708 additions and 128 deletions
|
|
@ -482,19 +482,6 @@ export async function getSyncDirectories(path, source) {
|
|||
throw new Error(message);
|
||||
}
|
||||
|
||||
export async function runSync(force = false) {
|
||||
const response = await apiPost(`${API_URI}/sync/run`, {force});
|
||||
if (!response.ok) {
|
||||
const message = await getErrorMessage(response, 'Could not start the sync.');
|
||||
toast({type: 'error', title: 'Sync Failed', description: message, time: 5000});
|
||||
}
|
||||
return response;
|
||||
}
|
||||
|
||||
export async function cancelSync() {
|
||||
return await apiPost(`${API_URI}/sync/cancel`);
|
||||
}
|
||||
|
||||
export async function runBackup() {
|
||||
const response = await apiPost(`${API_URI}/sync/backup/run`);
|
||||
if (!response.ok) {
|
||||
|
|
|
|||
|
|
@ -279,3 +279,18 @@ export async function healthCheck() {
|
|||
export async function getControllerInfo() {
|
||||
return controllerFetch('/info');
|
||||
}
|
||||
|
||||
// --- Sync (a secondary WROLPi copying its primary; the Controller runs it, because the API is stopped) ---
|
||||
|
||||
export async function getSyncStatus() {
|
||||
return controllerFetch('/sync/status');
|
||||
}
|
||||
|
||||
/** Start a sync now. Throws with the reason a sync cannot start. */
|
||||
export async function runSync() {
|
||||
return controllerFetch('/sync/run', {method: 'POST'});
|
||||
}
|
||||
|
||||
export async function cancelSync() {
|
||||
return controllerFetch('/sync/cancel', {method: 'POST'});
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,15 +2,13 @@ import React from "react";
|
|||
import {SettingsContext, StatusContext} from "../../contexts/contexts";
|
||||
import {
|
||||
cancelBackup,
|
||||
cancelSync,
|
||||
getSyncDirectories,
|
||||
PrimaryUnreachableError,
|
||||
runBackup,
|
||||
runSync,
|
||||
saveSyncSettings,
|
||||
testSync
|
||||
} from "../../api";
|
||||
import {getDisks} from "../../api/controller";
|
||||
import {cancelSync, getDisks, getSyncStatus, runSync} from "../../api/controller";
|
||||
import {humanFileSize, isoDatetimeToAgoPopup, useTitle} from "../Common";
|
||||
import {Link} from "react-router";
|
||||
import {useDockerized, useSyncRole, useWROLMode} from "../../hooks/customHooks";
|
||||
|
|
@ -34,18 +32,67 @@ const roleOptions = [
|
|||
{value: 'secondary', label: 'Secondary'},
|
||||
];
|
||||
|
||||
/** The sync's progress: how many of the primary's FileGroups have been compared. */
|
||||
function syncPercent(sync) {
|
||||
if (!sync || !sync.max_id) {
|
||||
return 0;
|
||||
// What each step of a sync is called (see controller/lib/sync.py).
|
||||
const SYNC_PHASES = {
|
||||
checks: 'Checking the primary',
|
||||
snapshot: 'The primary is copying its database',
|
||||
files: 'Copying files',
|
||||
database: 'Copying the database',
|
||||
starting: 'Starting WROLPi',
|
||||
purge: 'Emptying the trash',
|
||||
};
|
||||
|
||||
export function syncProgressLabel(sync) {
|
||||
const phase = SYNC_PHASES[sync?.phase] || 'Syncing';
|
||||
if (sync?.phase === 'files' || sync?.phase === 'database') {
|
||||
return `${phase}: ${humanFileSize(sync.bytes_transferred || 0)}`;
|
||||
}
|
||||
// FileGroup ids are not contiguous; `groups_seen / max_id` underestimates, it never overshoots.
|
||||
return Math.min(100, Math.round(100 * sync.groups_seen / sync.max_id));
|
||||
return phase;
|
||||
}
|
||||
|
||||
function syncProgressLabel(sync) {
|
||||
const downloaded = `${sync.files_downloaded} / ${sync.files_to_download} files`;
|
||||
return `${sync.phase === 'configs' ? 'Importing configs' : downloaded} (${humanFileSize(sync.bytes_downloaded)})`;
|
||||
// How often the sync's status is asked for, while one runs, and otherwise.
|
||||
const SYNC_POLL_RUNNING_MS = 2000;
|
||||
const SYNC_POLL_IDLE_MS = 10000;
|
||||
|
||||
/**
|
||||
* The sync's status, from the Controller: a secondary's API is stopped while it syncs, so the Controller is the
|
||||
* only one that can say how the sync is going. Not asked for at all on a primary.
|
||||
*/
|
||||
export function useSyncStatus() {
|
||||
const role = useSyncRole();
|
||||
const [sync, setSync] = React.useState(null);
|
||||
// The settings (and the role) cannot be read while the API is stopped; a role that is not known yet may be a
|
||||
// secondary in the middle of a sync.
|
||||
const enabled = role !== 'primary';
|
||||
const running = sync?.state === 'running';
|
||||
|
||||
const fetchSync = React.useCallback(async () => {
|
||||
try {
|
||||
setSync(await getSyncStatus());
|
||||
} catch (e) {
|
||||
// The Controller is restarting; ask again later.
|
||||
}
|
||||
}, []);
|
||||
|
||||
React.useEffect(() => {
|
||||
if (!enabled) {
|
||||
setSync(null);
|
||||
return;
|
||||
}
|
||||
fetchSync();
|
||||
const interval = setInterval(fetchSync, running ? SYNC_POLL_RUNNING_MS : SYNC_POLL_IDLE_MS);
|
||||
return () => clearInterval(interval);
|
||||
}, [enabled, running, fetchSync]);
|
||||
|
||||
return {sync, running, fetchSync};
|
||||
}
|
||||
|
||||
async function stopSync() {
|
||||
try {
|
||||
await cancelSync();
|
||||
} catch (e) {
|
||||
toast({type: 'error', title: 'Could not stop the sync', description: e.message, time: 5000});
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
|
@ -73,38 +120,38 @@ export function SecondaryRoleNotice({primaryUrl, content}) {
|
|||
</Message>
|
||||
}
|
||||
|
||||
// A sync that was stopped on purpose is not a problem.
|
||||
const STOPPED = ['The sync was cancelled', 'WROL Mode was enabled'];
|
||||
|
||||
/** The progress and outcome of a secondary's sync. */
|
||||
export function SyncStatus({sync, running}) {
|
||||
if (!sync) {
|
||||
return null;
|
||||
}
|
||||
const stopped = STOPPED.includes(sync.last_error);
|
||||
return <div>
|
||||
{running && <Progress percent={syncPercent(sync)} label={syncProgressLabel(sync)}/>}
|
||||
{running && <Progress percent={sync.percent || 0} label={syncProgressLabel(sync)}/>}
|
||||
<p>
|
||||
<b>Last successful sync:</b> {sync.last_success_at ? isoDatetimeToAgoPopup(sync.last_success_at) : 'Never'}
|
||||
</p>
|
||||
{!running && sync.finished_at && <p>
|
||||
<b>Last sync:</b> {sync.files_downloaded} files downloaded
|
||||
({humanFileSize(sync.bytes_downloaded)}), {sync.files_failed} failed
|
||||
{sync.files_excluded > 0 && <>, {sync.files_excluded} excluded</>}
|
||||
<b>Last sync:</b> {sync.files_transferred} files copied ({humanFileSize(sync.bytes_transferred || 0)}),
|
||||
{' '}{sync.files_deleted} moved to the trash
|
||||
</p>}
|
||||
{sync.disk_full && <Message kind='error' icon='hdd' title='The drive is full'>
|
||||
Some files were not synced because the drive is nearly full.
|
||||
</Message>}
|
||||
{sync.last_error && !running && <Message kind='error' icon='warning' title='Sync problem'>
|
||||
{sync.last_error}
|
||||
</Message>}
|
||||
{sync.last_skip_reason && !running && <p style={{color: 'var(--muted)'}}>
|
||||
Not synced: {sync.last_skip_reason}
|
||||
{sync.trash_bytes > 0 && <p style={{color: 'var(--muted)'}}>
|
||||
The trash holds {humanFileSize(sync.trash_bytes)} of files the primary deleted or replaced.
|
||||
</p>}
|
||||
{!running && (sync.warnings || []).map(warning => <Message key={warning} kind='warning' icon='warning'
|
||||
title='Sync warning'>{warning}</Message>)}
|
||||
{sync.last_error && !running && (stopped
|
||||
? <p style={{color: 'var(--muted)'}}>{sync.last_error}.</p>
|
||||
: <Message kind='error' icon='warning' title='Sync problem'>{sync.last_error}</Message>)}
|
||||
</div>
|
||||
}
|
||||
|
||||
/** Covers the app while a secondary syncs; its API refuses requests until the sync finishes. */
|
||||
/** Covers the app while a secondary syncs; its API is stopped until the sync finishes. */
|
||||
export function SyncingOverlay() {
|
||||
const {status} = React.useContext(StatusContext);
|
||||
const role = useSyncRole();
|
||||
const running = role === 'secondary' && status?.flags?.sync_running === true;
|
||||
const {sync, running} = useSyncStatus();
|
||||
|
||||
if (!running) {
|
||||
return null;
|
||||
|
|
@ -114,7 +161,7 @@ export function SyncingOverlay() {
|
|||
position: 'fixed', inset: 0, zIndex: 1000, display: 'flex', alignItems: 'center',
|
||||
justifyContent: 'center', padding: 16, background: 'rgba(0, 0, 0, 0.6)',
|
||||
}}>
|
||||
<SyncingPanel sync={status.sync} onCancel={cancelSync}/>
|
||||
<SyncingPanel sync={sync} onCancel={stopSync}/>
|
||||
</div>
|
||||
}
|
||||
|
||||
|
|
@ -126,7 +173,7 @@ export function SyncingPanel({sync, onCancel}) {
|
|||
This WROLPi is copying the library of {sync?.primary_url || 'its primary'}. It is unavailable
|
||||
until the sync finishes.
|
||||
</p>
|
||||
<Progress percent={syncPercent(sync)} label={sync ? syncProgressLabel(sync) : null}/>
|
||||
<Progress percent={sync?.percent || 0} label={syncProgressLabel(sync)}/>
|
||||
<Button color='red' onClick={onCancel} style={{marginTop: '1em'}}>Stop Sync</Button>
|
||||
</Panel>
|
||||
}
|
||||
|
|
@ -226,7 +273,7 @@ export function ExcludeFolderModal({open, onClose, value, onSubmit, source}) {
|
|||
}
|
||||
|
||||
/** The folders a sync or a backup does not copy, and the button that chooses them. */
|
||||
export function ExcludedDirectories({value, onChange, source, disabled}) {
|
||||
export function ExcludedDirectories({value, onChange, source, disabled, note}) {
|
||||
const [open, setOpen] = React.useState(false);
|
||||
const excluded = value || [];
|
||||
return <div style={{margin: '0.5em 0'}}>
|
||||
|
|
@ -243,6 +290,7 @@ export function ExcludedDirectories({value, onChange, source, disabled}) {
|
|||
<Button icon='folder open' onClick={() => setOpen(true)} disabled={disabled}>Choose Folders</Button>
|
||||
<p style={{color: 'var(--muted)', fontSize: '0.9em'}}>
|
||||
Files already copied into an excluded folder are kept; delete them yourself to free space.
|
||||
{note && <> {note}</>}
|
||||
</p>
|
||||
<ExcludeFolderModal open={open} onClose={() => setOpen(false)} value={excluded} source={source}
|
||||
onSubmit={chosen => {
|
||||
|
|
@ -254,7 +302,7 @@ export function ExcludedDirectories({value, onChange, source, disabled}) {
|
|||
|
||||
export function SyncSection() {
|
||||
const {settings, fetchSettings} = React.useContext(SettingsContext);
|
||||
const {status} = React.useContext(StatusContext);
|
||||
const {sync, running, fetchSync} = useSyncStatus();
|
||||
const wrolMode = useWROLMode();
|
||||
const [state, setState] = React.useState({});
|
||||
const [testResult, setTestResult] = React.useState(null);
|
||||
|
|
@ -267,6 +315,7 @@ export function SyncSection() {
|
|||
const windowStart = settings?.sync_window_start;
|
||||
const windowEnd = settings?.sync_window_end;
|
||||
const excludedKey = JSON.stringify(settings?.sync_excluded_directories || []);
|
||||
const deleteAfterDays = settings?.sync_delete_after_days;
|
||||
React.useEffect(() => {
|
||||
if (loaded) {
|
||||
setState({
|
||||
|
|
@ -275,9 +324,11 @@ export function SyncSection() {
|
|||
sync_window_start: windowStart || '',
|
||||
sync_window_end: windowEnd || '',
|
||||
sync_excluded_directories: JSON.parse(excludedKey),
|
||||
// null: never delete.
|
||||
sync_delete_after_days: deleteAfterDays === undefined ? 30 : deleteAfterDays,
|
||||
});
|
||||
}
|
||||
}, [loaded, role, primaryUrl, windowStart, windowEnd, excludedKey]);
|
||||
}, [loaded, role, primaryUrl, windowStart, windowEnd, excludedKey, deleteAfterDays]);
|
||||
|
||||
if (!settings) {
|
||||
return null;
|
||||
|
|
@ -285,12 +336,24 @@ export function SyncSection() {
|
|||
|
||||
const secondary = state.role === 'secondary';
|
||||
const saved = secondary && settings.role === 'secondary' && settings.sync_primary_url;
|
||||
const running = status?.flags?.sync_running === true;
|
||||
const neverDelete = state.sync_delete_after_days === null;
|
||||
|
||||
const save = async () => {
|
||||
// An empty box is not "delete at once": that would empty the trash at the next sync.
|
||||
const days = String(state.sync_delete_after_days ?? '').trim();
|
||||
if (secondary && !neverDelete && !/^\d+$/.test(days)) {
|
||||
toast({
|
||||
type: 'error', title: 'Sync settings were not saved', time: 5000,
|
||||
description: 'Enter how many days to keep deleted files, or choose to never delete them.',
|
||||
});
|
||||
return;
|
||||
}
|
||||
setSaving(true);
|
||||
try {
|
||||
const response = await saveSyncSettings(state);
|
||||
// The API takes -1 for "never".
|
||||
const response = await saveSyncSettings({
|
||||
...state, sync_delete_after_days: neverDelete ? -1 : Number(days),
|
||||
});
|
||||
if (response?.ok) {
|
||||
toast({type: 'success', title: 'Sync settings saved', time: 3000});
|
||||
setTestResult(null);
|
||||
|
|
@ -306,17 +369,20 @@ export function SyncSection() {
|
|||
};
|
||||
|
||||
const syncNow = async () => {
|
||||
const response = await runSync();
|
||||
if (response?.ok) {
|
||||
toast({type: 'info', title: 'Sync requested', description: 'The sync starts in a few seconds.', time: 3000});
|
||||
try {
|
||||
await runSync();
|
||||
toast({type: 'info', title: 'Sync started', time: 3000});
|
||||
await fetchSync();
|
||||
} catch (e) {
|
||||
toast({type: 'error', title: 'The sync did not start', description: e.message, time: 5000});
|
||||
}
|
||||
};
|
||||
|
||||
return <Panel>
|
||||
<Header as='h3' icon='copy'>Sync</Header>
|
||||
<p>
|
||||
A <b>secondary</b> WROLPi keeps a copy of a <b>primary</b> WROLPi's library: its files, tags,
|
||||
channels, domains, playlists, and other library settings. Files deleted on the primary are kept.
|
||||
A <b>secondary</b> WROLPi keeps a copy of a <b>primary</b> WROLPi's library: all of its files (maps
|
||||
and repositories too), its database, tags, channels, playlists, and other library settings.
|
||||
</p>
|
||||
<div style={{margin: '0.5em 0', maxWidth: '20em'}}>
|
||||
<label><b>This WROLPi is</b></label>
|
||||
|
|
@ -326,8 +392,9 @@ export function SyncSection() {
|
|||
|
||||
{secondary && <>
|
||||
<Message kind='warning' icon='warning' title='A secondary is overwritten'>
|
||||
Every sync replaces this WROLPi's library with the primary's; changes made here are lost.
|
||||
It does not download, and it is unavailable while it syncs.
|
||||
Every sync replaces this WROLPi's library and database with the primary's; changes made here are
|
||||
lost. It does not download, and it is unavailable while it syncs. Both WROLPis must run the same
|
||||
version.
|
||||
</Message>
|
||||
<div style={{margin: '0.5em 0', maxWidth: '30em'}}>
|
||||
<label><b>Primary address</b></label>
|
||||
|
|
@ -353,11 +420,32 @@ export function SyncSection() {
|
|||
Within the window, this secondary syncs every 15 minutes. Leave both empty to sync only with
|
||||
"Sync Now".
|
||||
</p>
|
||||
<div style={{margin: '0.5em 0'}}>
|
||||
<label><b>Files the primary deletes</b></label>
|
||||
<div style={{display: 'flex', gap: '1em', alignItems: 'center', flexWrap: 'wrap'}}>
|
||||
<span style={{display: 'flex', gap: '0.5em', alignItems: 'center'}}>
|
||||
Delete after
|
||||
<TextInput type='number' min={0} max={3650} style={{width: '6em'}} aria-label='Days'
|
||||
value={neverDelete ? '' : state.sync_delete_after_days}
|
||||
disabled={wrolMode || neverDelete}
|
||||
onChange={e => setState({...state, sync_delete_after_days: e.currentTarget.value})}/>
|
||||
days
|
||||
</span>
|
||||
<Checkbox checked={neverDelete} disabled={wrolMode} label='Never delete them'
|
||||
onChange={() => setState({...state, sync_delete_after_days: neverDelete ? 30 : null})}/>
|
||||
</div>
|
||||
<p style={{color: 'var(--muted)', fontSize: '0.9em'}}>
|
||||
A file the primary deletes, or the old version of one it changes, waits in a trash folder
|
||||
(<code>.sync-trash</code>) for this long. It is removed sooner if the drive is nearly full.
|
||||
</p>
|
||||
</div>
|
||||
</>}
|
||||
|
||||
{/* The primary's folders can only be listed once this WROLPi knows its primary. */}
|
||||
{secondary && saved && <ExcludedDirectories value={state.sync_excluded_directories} source='primary'
|
||||
disabled={wrolMode}
|
||||
note="This WROLPi's database is the primary's, so the files of an
|
||||
excluded folder are still listed here; they do not open."
|
||||
onChange={value => setState({
|
||||
...state,
|
||||
sync_excluded_directories: value
|
||||
|
|
@ -379,7 +467,7 @@ export function SyncSection() {
|
|||
WROL Mode is enabled; this WROLPi does not sync.
|
||||
</Message>}
|
||||
|
||||
{settings.role === 'secondary' && <SyncStatus sync={status?.sync} running={running}/>}
|
||||
{settings.role === 'secondary' && <SyncStatus sync={sync} running={running}/>}
|
||||
</Panel>
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,21 +1,24 @@
|
|||
import React from 'react';
|
||||
import {fireEvent, screen, waitFor} from '@testing-library/react';
|
||||
import {renderWithProviders} from '../../test-utils';
|
||||
import {backupStatusFixture, settingsFixture, statusFixture, syncStatusFixture} from '../../test-fixtures';
|
||||
import {backupStatusFixture, controllerSyncStatusFixture, settingsFixture, statusFixture} from '../../test-fixtures';
|
||||
import {
|
||||
BackupDriveSection, backupDriveOptions, ExcludedDirectories, isExcludedPath, SecondaryRoleMessage, simplifyExcluded,
|
||||
splitBackupDirectory, SyncingOverlay, SyncSection,
|
||||
} from './Sync';
|
||||
|
||||
jest.mock('../ui/toast', () => ({...jest.requireActual('../ui/toast'), toast: jest.fn()}));
|
||||
|
||||
jest.mock('../../api/controller', () => ({
|
||||
getDisks: jest.fn(),
|
||||
getSyncStatus: jest.fn(),
|
||||
runSync: jest.fn(),
|
||||
cancelSync: jest.fn(),
|
||||
}));
|
||||
|
||||
jest.mock('../../api', () => ({
|
||||
cancelBackup: jest.fn(),
|
||||
runBackup: jest.fn(),
|
||||
cancelSync: jest.fn(),
|
||||
runSync: jest.fn(),
|
||||
saveSyncSettings: jest.fn(),
|
||||
testSync: jest.fn(),
|
||||
getSyncDirectories: jest.fn(),
|
||||
|
|
@ -24,14 +27,16 @@ jest.mock('../../api', () => ({
|
|||
}));
|
||||
|
||||
import {
|
||||
cancelBackup, cancelSync, getSyncDirectories, PrimaryUnreachableError, runBackup, runSync, saveSyncSettings, testSync,
|
||||
cancelBackup, getSyncDirectories, PrimaryUnreachableError, runBackup, saveSyncSettings, testSync,
|
||||
} from '../../api';
|
||||
import {getDisks} from '../../api/controller';
|
||||
import {cancelSync, getDisks, getSyncStatus, runSync} from '../../api/controller';
|
||||
import {toast} from '../ui/toast';
|
||||
|
||||
beforeEach(() => {
|
||||
// react-scripts resets mock implementations before each test.
|
||||
cancelSync.mockResolvedValue({ok: true});
|
||||
runSync.mockResolvedValue({ok: true});
|
||||
cancelSync.mockResolvedValue({success: true});
|
||||
runSync.mockResolvedValue({success: true});
|
||||
getSyncStatus.mockResolvedValue(controllerSyncStatusFixture());
|
||||
saveSyncSettings.mockResolvedValue({ok: true});
|
||||
testSync.mockResolvedValue({ok: true, info: {version: '0.30.5-beta'}});
|
||||
runBackup.mockResolvedValue({ok: true});
|
||||
|
|
@ -76,29 +81,39 @@ describe('SecondaryRoleMessage', () => {
|
|||
});
|
||||
|
||||
describe('SyncingOverlay', () => {
|
||||
const syncing = (sync = {}) => ({
|
||||
status: statusFixture({
|
||||
flags: {sync_running: true},
|
||||
sync: syncStatusFixture({state: 'running', phase: 'files', groups_seen: 50, max_id: 100, ...sync}),
|
||||
}),
|
||||
const running = (sync = {}) => controllerSyncStatusFixture({
|
||||
state: 'running', phase: 'files', percent: 40, bytes_transferred: 5 * 1024 ** 3,
|
||||
primary_url: 'https://192.168.0.10', ...sync,
|
||||
});
|
||||
|
||||
it('covers the app while a secondary syncs', () => {
|
||||
renderWithProviders(<SyncingOverlay/>, {settings: secondary(), status: syncing()});
|
||||
expect(screen.getByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
|
||||
it('covers the app while a secondary syncs, from the Controller (the API is stopped)', async () => {
|
||||
getSyncStatus.mockResolvedValue(running());
|
||||
renderWithProviders(<SyncingOverlay/>, {settings: secondary()});
|
||||
expect(await screen.findByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
|
||||
expect(screen.getByText('Syncing from the primary')).toBeInTheDocument();
|
||||
expect(screen.getByText(/Copying files: 5/)).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByText('Stop Sync'));
|
||||
expect(cancelSync).toHaveBeenCalled();
|
||||
await waitFor(() => expect(cancelSync).toHaveBeenCalled());
|
||||
});
|
||||
|
||||
it('is absent when not syncing', () => {
|
||||
it('is shown when the settings cannot be read, as while the API is stopped', async () => {
|
||||
getSyncStatus.mockResolvedValue(running());
|
||||
renderWithProviders(<SyncingOverlay/>, {settings: {settings: null}});
|
||||
expect(await screen.findByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('is absent when not syncing', async () => {
|
||||
renderWithProviders(<SyncingOverlay/>, {settings: secondary()});
|
||||
await waitFor(() => expect(getSyncStatus).toHaveBeenCalled());
|
||||
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('is absent on a primary', () => {
|
||||
renderWithProviders(<SyncingOverlay/>, {status: syncing()});
|
||||
it('is absent on a primary, which never asks', async () => {
|
||||
getSyncStatus.mockResolvedValue(running());
|
||||
renderWithProviders(<SyncingOverlay/>);
|
||||
await new Promise(resolve => setTimeout(resolve, 50));
|
||||
expect(getSyncStatus).not.toHaveBeenCalled();
|
||||
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
|
||||
});
|
||||
});
|
||||
|
|
@ -115,11 +130,16 @@ describe('SyncSection', () => {
|
|||
});
|
||||
|
||||
it('a secondary shows its primary, window, and sync controls', async () => {
|
||||
const status = {status: statusFixture({sync: syncStatusFixture({last_error: 'The primary did not answer'})})};
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_window_start: '01:00'}), status});
|
||||
getSyncStatus.mockResolvedValue(controllerSyncStatusFixture({
|
||||
finished_at: '2026-11-19T03:00:00+00:00', last_error: 'The primary did not answer',
|
||||
files_transferred: 12, files_deleted: 3, trash_bytes: 2048,
|
||||
}));
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_window_start: '01:00'})});
|
||||
expect(screen.getByDisplayValue('https://192.168.0.10')).toBeInTheDocument();
|
||||
expect(screen.getByDisplayValue('01:00')).toBeInTheDocument();
|
||||
expect(screen.getByText('The primary did not answer')).toBeInTheDocument();
|
||||
expect(await screen.findByText('The primary did not answer')).toBeInTheDocument();
|
||||
expect(screen.getByText(/12 files copied/)).toBeInTheDocument();
|
||||
expect(screen.getByText(/3 moved to the trash/)).toBeInTheDocument();
|
||||
|
||||
fireEvent.click(screen.getByText('Sync Now'));
|
||||
await waitFor(() => expect(runSync).toHaveBeenCalled());
|
||||
|
|
@ -127,10 +147,55 @@ describe('SyncSection', () => {
|
|||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
|
||||
role: 'secondary', sync_primary_url: 'https://192.168.0.10', sync_window_start: '01:00',
|
||||
sync_window_end: '', sync_excluded_directories: [],
|
||||
sync_window_end: '', sync_excluded_directories: [], sync_delete_after_days: 30,
|
||||
}));
|
||||
});
|
||||
|
||||
it('says why a sync did not start', async () => {
|
||||
runSync.mockRejectedValue(new Error('A sync is already running'));
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary()});
|
||||
fireEvent.click(screen.getByText('Sync Now'));
|
||||
await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: 'error', title: 'The sync did not start', description: 'A sync is already running'})));
|
||||
});
|
||||
|
||||
it('a stopped sync is not a problem', async () => {
|
||||
getSyncStatus.mockResolvedValue(controllerSyncStatusFixture({last_error: 'The sync was cancelled'}));
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary()});
|
||||
expect(await screen.findByText('The sync was cancelled.')).toBeInTheDocument();
|
||||
expect(screen.queryByText('Sync problem')).not.toBeInTheDocument();
|
||||
});
|
||||
|
||||
it('saves how long deleted files are kept, or never to delete them', async () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_delete_after_days: 7})});
|
||||
expect(screen.getByLabelText('Days')).toHaveValue(7);
|
||||
|
||||
fireEvent.click(screen.getByLabelText('Never delete them'));
|
||||
expect(screen.getByLabelText('Days')).toBeDisabled();
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({sync_delete_after_days: -1})));
|
||||
});
|
||||
|
||||
it('an empty number of days is not saved as "delete at once"', async () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_delete_after_days: 30})});
|
||||
fireEvent.change(screen.getByLabelText('Days'), {target: {value: ''}});
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({
|
||||
type: 'error', title: 'Sync settings were not saved'})));
|
||||
expect(saveSyncSettings).not.toHaveBeenCalled();
|
||||
|
||||
fireEvent.change(screen.getByLabelText('Days'), {target: {value: '0'}});
|
||||
fireEvent.click(screen.getByText('Save'));
|
||||
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
|
||||
expect.objectContaining({sync_delete_after_days: 0})));
|
||||
});
|
||||
|
||||
it('a secondary that never deletes shows it', () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({sync_delete_after_days: null})});
|
||||
expect(screen.getByLabelText('Never delete them')).toBeChecked();
|
||||
});
|
||||
|
||||
it('WROL Mode stops syncing', () => {
|
||||
renderWithProviders(<SyncSection/>, {settings: secondary({wrol_mode: true})});
|
||||
expect(screen.getByText('Syncing is stopped')).toBeInTheDocument();
|
||||
|
|
|
|||
|
|
@ -123,6 +123,25 @@ export const syncStatusFixture = (overrides = {}) => ({
|
|||
...overrides,
|
||||
});
|
||||
|
||||
/** GET /controller/api/sync/status: a secondary's sync that has never run (controller/lib/sync.py). */
|
||||
export const controllerSyncStatusFixture = (overrides = {}) => ({
|
||||
state: 'idle',
|
||||
phase: null,
|
||||
started_at: null,
|
||||
finished_at: null,
|
||||
last_success_at: null,
|
||||
last_error: null,
|
||||
primary_url: null,
|
||||
primary_version: null,
|
||||
percent: 0,
|
||||
bytes_transferred: 0,
|
||||
files_transferred: 0,
|
||||
files_deleted: 0,
|
||||
trash_bytes: 0,
|
||||
warnings: [],
|
||||
...overrides,
|
||||
});
|
||||
|
||||
/** The `backup` of GET /api/status: a backup drive that has never been written. */
|
||||
export const backupStatusFixture = (overrides = {}) => ({
|
||||
state: 'idle',
|
||||
|
|
|
|||
|
|
@ -29,10 +29,13 @@ import subprocess
|
|||
import threading
|
||||
import time
|
||||
import uuid
|
||||
import zoneinfo
|
||||
from pathlib import Path
|
||||
from typing import Callable, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import yaml
|
||||
|
||||
|
|
@ -58,6 +61,10 @@ STATE_FILE = "config/sync_state.yaml"
|
|||
DEFAULT_DELETE_AFTER_DAYS = 30
|
||||
# Never fill the secondary's drive; leave this much free.
|
||||
MIN_FREE_BYTES = 1024 ** 3
|
||||
# Within the daily sync window, a sync starts this long after the last one finished.
|
||||
SYNC_REPEAT_SECONDS = 15 * 60
|
||||
# How often the schedule is looked at.
|
||||
SCHEDULE_INTERVAL_SECONDS = 60
|
||||
SNAPSHOT_TIMEOUT_SECONDS = 60 * 60
|
||||
API_START_TIMEOUT_SECONDS = 10 * 60
|
||||
# A sync runs unattended with the API stopped: a primary that loses power or its link must end the run, not hold it.
|
||||
|
|
@ -118,6 +125,9 @@ def default_status() -> dict:
|
|||
|
||||
|
||||
_status = None
|
||||
# True once the state file was read or written: until then the drive may not be mounted, and what is in memory must
|
||||
# not be mistaken for (or saved over) what the last sync recorded.
|
||||
_status_from_file = False
|
||||
_status_lock = threading.Lock()
|
||||
_run_lock = threading.Lock()
|
||||
_cancel = threading.Event()
|
||||
|
|
@ -132,16 +142,19 @@ def _state_file() -> Path:
|
|||
|
||||
|
||||
def get_status() -> dict:
|
||||
"""The progress and outcome of the sync. Read from the state file the first time, so a restarted Controller
|
||||
still knows when the last sync succeeded."""
|
||||
global _status
|
||||
"""The progress and outcome of the sync. Read from the state file, so a restarted Controller still knows when
|
||||
the last sync succeeded; read again until that works, as the Controller may start before the drive is mounted."""
|
||||
global _status, _status_from_file
|
||||
with _status_lock:
|
||||
if _status is None:
|
||||
_status = default_status()
|
||||
if not _status_from_file:
|
||||
try:
|
||||
saved = yaml.safe_load(checked_path(_state_file()).read_text())
|
||||
if isinstance(saved, dict):
|
||||
_status = default_status()
|
||||
_status.update({k: v for k, v in saved.items() if k in _status})
|
||||
_status_from_file = True
|
||||
except (OSError, yaml.YAMLError, SyncError):
|
||||
pass
|
||||
return dict(_status)
|
||||
|
|
@ -154,8 +167,10 @@ def update_status(save: bool = False, **changes) -> dict:
|
|||
_status.update(changes)
|
||||
status = dict(_status)
|
||||
if save:
|
||||
global _status_from_file
|
||||
try:
|
||||
write_file(_state_file(), yaml.safe_dump(status))
|
||||
_status_from_file = True
|
||||
except (OSError, SyncError) as e:
|
||||
logger.warning("Could not save the sync state: %s", e)
|
||||
return status
|
||||
|
|
@ -374,11 +389,15 @@ class DockerRunner:
|
|||
def rsync(self, arguments: list, on_line: Callable[[str], None]) -> int:
|
||||
client, api = _get_client(), self._api()
|
||||
self.stop_rsync()
|
||||
helper = client.containers.run(
|
||||
api.image.id, arguments, entrypoint="rsync", name=self.HELPER_NAME, detach=True,
|
||||
helper = client.containers.create(
|
||||
api.image.id, arguments, entrypoint="rsync", name=self.HELPER_NAME,
|
||||
user=api.attrs["Config"].get("User") or None, volumes_from=[api.name])
|
||||
try:
|
||||
stream = helper.logs(stream=True, follow=True)
|
||||
# Attached, not followed through the logs: Docker's logs hold a line until it ends, and rsync's progress
|
||||
# for one large file is a single line, rewritten with carriage returns until the file is done.
|
||||
# Attached before it starts, so nothing it writes is missed.
|
||||
stream = helper.attach(stdout=True, stderr=True, stream=True)
|
||||
helper.start()
|
||||
|
||||
def read(_: int = 0) -> bytes:
|
||||
return next(stream, b"")
|
||||
|
|
@ -672,16 +691,17 @@ def run_sync(runner=None) -> dict:
|
|||
|
||||
|
||||
def _finish(runner, api_stopped: bool, **changes) -> dict:
|
||||
"""Whatever happened, the API runs again and the lock is free."""
|
||||
"""Whatever happened, the API runs again and the lock is free. The outcome is recorded before the lock is
|
||||
freed: a status that says "running" while no sync holds the lock is taken for an interrupted sync."""
|
||||
try:
|
||||
if api_stopped:
|
||||
runner.start_api()
|
||||
trash = get_media_directory() / TRASH_DIRECTORY
|
||||
return update_status(save=True, phase=None, finished_at=_now(),
|
||||
trash_bytes=directory_size(trash) if trash.is_dir() and not trash.is_symlink() else 0,
|
||||
**changes)
|
||||
finally:
|
||||
_run_lock.release()
|
||||
trash = get_media_directory() / TRASH_DIRECTORY
|
||||
return update_status(save=True, phase=None, finished_at=_now(),
|
||||
trash_bytes=directory_size(trash) if trash.is_dir() and not trash.is_symlink() else 0,
|
||||
**changes)
|
||||
|
||||
|
||||
def _read_yaml(file: Path) -> dict:
|
||||
|
|
@ -698,10 +718,11 @@ def request_cancel():
|
|||
|
||||
|
||||
def recover_interrupted_sync(runner=None):
|
||||
"""Called when the Controller starts. A sync that was running when the Controller stopped left the API stopped;
|
||||
start it, and say the sync was interrupted."""
|
||||
"""A status that says "running" while no sync holds the lock was left by a Controller that stopped during a
|
||||
sync, which left the API stopped: start it, and say the sync was interrupted. Looked at by every tick of the
|
||||
schedule, before it starts anything, and so also once the state file can be read after the drive mounts."""
|
||||
status = get_status()
|
||||
if status["state"] != "running":
|
||||
if status["state"] != "running" or _run_lock.locked():
|
||||
return
|
||||
logger.warning("A sync was interrupted; starting the API")
|
||||
runner = runner or get_runner()
|
||||
|
|
@ -712,3 +733,64 @@ def recover_interrupted_sync(runner=None):
|
|||
runner.start_api()
|
||||
update_status(save=True, state="error", phase=None, finished_at=_now(),
|
||||
last_error="The sync was interrupted")
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------------------------------------------------
|
||||
# The schedule.
|
||||
|
||||
|
||||
def local_now() -> datetime.datetime:
|
||||
"""Now, in WROLPi's timezone (wrolpi.yaml), or the system's when none is set."""
|
||||
timezone = _read_yaml(get_media_directory() / "config/wrolpi.yaml").get("timezone")
|
||||
if timezone:
|
||||
with contextlib.suppress(zoneinfo.ZoneInfoNotFoundError, ValueError):
|
||||
return datetime.datetime.now(zoneinfo.ZoneInfo(timezone))
|
||||
return datetime.datetime.now().astimezone()
|
||||
|
||||
|
||||
def _parse_time(value) -> Optional[datetime.time]:
|
||||
with contextlib.suppress(TypeError, ValueError):
|
||||
return datetime.time.fromisoformat(str(value))
|
||||
return None
|
||||
|
||||
|
||||
def in_window(start: Optional[str], end: Optional[str], now: datetime.time) -> bool:
|
||||
"""True if `now` is within the daily window from `start` to `end`, which may run past midnight."""
|
||||
start, end = _parse_time(start), _parse_time(end)
|
||||
if start is None or end is None:
|
||||
return False
|
||||
if start <= end:
|
||||
return start <= now < end
|
||||
return now >= start or now < end
|
||||
|
||||
|
||||
def sync_is_due(config: dict, status: dict, now: datetime.datetime) -> bool:
|
||||
"""True if the daily sync window is open and the last sync finished long enough ago. No window: never (a sync
|
||||
only runs when asked)."""
|
||||
if not in_window(config.get("window_start"), config.get("window_end"), now.time()):
|
||||
return False
|
||||
with contextlib.suppress(TypeError, ValueError):
|
||||
finished_at = datetime.datetime.fromisoformat(status["finished_at"])
|
||||
return (now - finished_at).total_seconds() >= SYNC_REPEAT_SECONDS
|
||||
return True
|
||||
|
||||
|
||||
def schedule_tick():
|
||||
"""Start a sync if one is due. A sync that cannot start (WROL Mode, a primary, no primary) is not due."""
|
||||
recover_interrupted_sync()
|
||||
config = read_sync_config()
|
||||
if skip_reason(config) is not None:
|
||||
return
|
||||
if sync_is_due(config, get_status(), local_now()):
|
||||
logger.info("The sync window is open")
|
||||
run_sync()
|
||||
|
||||
|
||||
async def sync_schedule_loop():
|
||||
"""Sync within the daily window for as long as the Controller runs."""
|
||||
while True:
|
||||
try:
|
||||
await asyncio.to_thread(schedule_tick)
|
||||
except Exception as e:
|
||||
logger.error("The sync schedule failed", exc_info=e)
|
||||
await asyncio.sleep(SCHEDULE_INTERVAL_SECONDS)
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ from controller.lib.status import (
|
|||
get_uptime_status,
|
||||
)
|
||||
from controller.lib.status_worker import status_worker_loop
|
||||
from controller.lib.sync import recover_interrupted_sync
|
||||
from controller.lib.sync import sync_schedule_loop
|
||||
from controller.lib.systemd import get_all_services_status
|
||||
|
||||
# Templates directory
|
||||
|
|
@ -126,8 +126,9 @@ async def lifespan(app: FastAPI):
|
|||
else:
|
||||
logger.info("Primary drive not mounted - using default configuration")
|
||||
|
||||
# A sync that was running when the Controller stopped left the API stopped.
|
||||
app.state.sync_recovery_task = asyncio.create_task(asyncio.to_thread(recover_interrupted_sync))
|
||||
# A secondary syncs within its daily window. Each tick first recovers a sync that was interrupted (the API it
|
||||
# stopped is started again), so nothing races that recovery.
|
||||
app.state.sync_schedule_task = asyncio.create_task(sync_schedule_loop())
|
||||
|
||||
# Docker: the library is served to secondary WROLPis from a helper container, kept matching the api container.
|
||||
if is_docker_mode():
|
||||
|
|
@ -152,8 +153,9 @@ async def lifespan(app: FastAPI):
|
|||
await status_task
|
||||
except asyncio.CancelledError:
|
||||
pass
|
||||
if rsync_daemon_task := getattr(app.state, "rsync_daemon_task", None):
|
||||
rsync_daemon_task.cancel()
|
||||
for name in ("rsync_daemon_task", "sync_schedule_task"):
|
||||
if task := getattr(app.state, name, None):
|
||||
task.cancel()
|
||||
|
||||
|
||||
# Create FastAPI app
|
||||
|
|
|
|||
|
|
@ -169,6 +169,33 @@
|
|||
display: block;
|
||||
}
|
||||
|
||||
/* A secondary WROLPi copying its primary; its API is stopped meanwhile. */
|
||||
.sync-card {
|
||||
border: 2px solid #3498db;
|
||||
border-radius: 4px;
|
||||
padding: 0.75rem 1rem;
|
||||
margin-bottom: 1rem;
|
||||
display: none;
|
||||
}
|
||||
|
||||
.sync-card.show {
|
||||
display: block;
|
||||
}
|
||||
|
||||
.sync-card .sync-bar {
|
||||
height: 0.6rem;
|
||||
background: rgba(127, 127, 127, 0.3);
|
||||
border-radius: 3px;
|
||||
overflow: hidden;
|
||||
margin: 0.5rem 0;
|
||||
}
|
||||
|
||||
.sync-card .sync-bar div {
|
||||
height: 100%;
|
||||
background: #3498db;
|
||||
width: 0;
|
||||
}
|
||||
|
||||
.fallback-banner .loading {
|
||||
border-color: #333;
|
||||
border-top-color: #000;
|
||||
|
|
@ -945,6 +972,15 @@
|
|||
<a id="view-files-btn" class="btn btn-info" style="{% if not app_ready %}display: none;{% endif %} font-size: 1.1rem; padding: 0.6rem 1.5rem; margin-left: 0.5rem;" href="#">View Files</a>
|
||||
</div>
|
||||
|
||||
<div id="sync-card" class="sync-card" role="status" aria-live="polite">
|
||||
<strong>Syncing from the primary</strong>
|
||||
<p id="sync-text" style="margin: 0.25rem 0;"></p>
|
||||
<div class="sync-bar"><div id="sync-bar"></div></div>
|
||||
<p style="margin: 0.25rem 0; font-size: 0.9em;">WROLPi is unavailable until the sync finishes, then this page
|
||||
returns to it.</p>
|
||||
<button id="sync-stop" class="btn btn-danger" type="button">Stop Sync</button>
|
||||
</div>
|
||||
|
||||
<div id="fallback-banner" class="fallback-banner">
|
||||
<span class="loading"></span>
|
||||
Main UI is unavailable. Using Controller fallback. Will redirect when ready...
|
||||
|
|
@ -3077,6 +3113,52 @@
|
|||
refreshToggleButtons();
|
||||
}, updateInterval);
|
||||
}
|
||||
|
||||
// A secondary WROLPi's sync (controller/lib/sync.py). Its API is stopped while it runs, which is why this page
|
||||
// is shown; say what is happening. Relative URLs: /controller/api/... behind Caddy, /api/... when direct.
|
||||
(function () {
|
||||
const PHASES = {
|
||||
checks: 'Checking the primary', snapshot: 'The primary is copying its database',
|
||||
files: 'Copying files', database: 'Copying the database', starting: 'Starting WROLPi',
|
||||
purge: 'Emptying the trash',
|
||||
};
|
||||
const card = document.getElementById('sync-card');
|
||||
const text = document.getElementById('sync-text');
|
||||
const bar = document.getElementById('sync-bar');
|
||||
const stop = document.getElementById('sync-stop');
|
||||
|
||||
async function checkSync() {
|
||||
try {
|
||||
const resp = await fetch('api/sync/status', {signal: AbortSignal.timeout(5000)});
|
||||
if (!resp.ok) {
|
||||
return;
|
||||
}
|
||||
const sync = await resp.json();
|
||||
const running = sync.state === 'running';
|
||||
card.classList.toggle('show', running);
|
||||
if (running) {
|
||||
const copied = (sync.phase === 'files' || sync.phase === 'database')
|
||||
? ` (${formatBytes(sync.bytes_transferred || 0)})` : '';
|
||||
text.textContent = `${PHASES[sync.phase] || 'Syncing'}${copied} from ${sync.primary_url || 'the primary'}`;
|
||||
bar.style.width = `${Math.min(100, sync.percent || 0)}%`;
|
||||
}
|
||||
} catch (e) {
|
||||
// The Controller is busy or restarting; ask again.
|
||||
}
|
||||
}
|
||||
|
||||
stop.addEventListener('click', async () => {
|
||||
stop.disabled = true;
|
||||
try {
|
||||
await fetch('api/sync/cancel', {method: 'POST'});
|
||||
} finally {
|
||||
setTimeout(() => { stop.disabled = false; }, 3000);
|
||||
}
|
||||
});
|
||||
|
||||
checkSync();
|
||||
setInterval(checkSync, 2000);
|
||||
})();
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
|
|
|||
|
|
@ -2,8 +2,10 @@
|
|||
Unit tests for controller.lib.sync: a secondary WROLPi copying its primary with rsync.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import datetime
|
||||
import os
|
||||
import re
|
||||
import shutil
|
||||
import socket
|
||||
import sqlite3
|
||||
|
|
@ -35,6 +37,7 @@ def media(tmp_path, monkeypatch) -> Path:
|
|||
(directory / "config/sync.yaml").write_text(yaml.safe_dump(dict(role="secondary", primary_url=PRIMARY)))
|
||||
monkeypatch.setenv("MEDIA_DIRECTORY", str(directory))
|
||||
monkeypatch.setattr(sync, "_status", None)
|
||||
monkeypatch.setattr(sync, "_status_from_file", False)
|
||||
sync._cancel.clear()
|
||||
yield directory
|
||||
assert not sync._run_lock.locked(), "the test left a sync running"
|
||||
|
|
@ -352,7 +355,7 @@ class TestRunSync:
|
|||
assert yaml.safe_load((media / sync.PRIMARY_INFO).read_text())["media_directory"] == "/media/wrolpi"
|
||||
assert (media / sync.PRIMARY_WROLPI_CONFIG).is_file()
|
||||
# A restarted Controller reads the state file.
|
||||
sync._status = None
|
||||
sync._status, sync._status_from_file = None, False
|
||||
assert sync.get_status()["last_success_at"] == status["last_success_at"]
|
||||
|
||||
def test_leaves_out_the_playlists_of_both_machines(self, media, primary_api):
|
||||
|
|
@ -428,7 +431,7 @@ class TestRunSync:
|
|||
def test_interrupted_sync_is_recovered(self, media):
|
||||
"""A Controller that stopped during a sync left the API stopped."""
|
||||
sync.update_status(save=True, state="running", phase="files")
|
||||
sync._status = None
|
||||
sync._status, sync._status_from_file = None, False
|
||||
runner = FakeRunner(media)
|
||||
sync.recover_interrupted_sync(runner)
|
||||
# In Docker rsync is its own container and is still replacing files: it is stopped before the API starts.
|
||||
|
|
@ -671,3 +674,178 @@ def test_never_delete_still_removes_stale_logs(tmp_path):
|
|||
assert (secondary / "kept folder/a.txt").read_text() == "kept"
|
||||
assert read_database(secondary / "map/region.search.db") == "the primary's new index"
|
||||
assert not (secondary / ".sync-trash").exists()
|
||||
|
||||
|
||||
class TestSchedule:
|
||||
|
||||
@pytest.mark.parametrize("start, end, now, expected", [
|
||||
("01:00", "05:00", "03:00", True),
|
||||
("01:00", "05:00", "05:00", False),
|
||||
("01:00", "05:00", "00:59", False),
|
||||
# Past midnight.
|
||||
("22:00", "02:00", "23:30", True),
|
||||
("22:00", "02:00", "01:59", True),
|
||||
("22:00", "02:00", "12:00", False),
|
||||
(None, "02:00", "01:00", False),
|
||||
("not a time", "02:00", "01:00", False),
|
||||
])
|
||||
def test_in_window(self, start, end, now, expected):
|
||||
assert sync.in_window(start, end, datetime.time.fromisoformat(now)) is expected
|
||||
|
||||
def test_due_every_fifteen_minutes_inside_the_window(self):
|
||||
tz = datetime.timezone(datetime.timedelta(hours=-7))
|
||||
now = datetime.datetime(2026, 11, 19, 3, 0, tzinfo=tz)
|
||||
config = dict(window_start="01:00", window_end="05:00")
|
||||
assert sync.sync_is_due(config, dict(finished_at=None), now)
|
||||
recent = (now - datetime.timedelta(minutes=14)).astimezone(datetime.timezone.utc).isoformat()
|
||||
assert not sync.sync_is_due(config, dict(finished_at=recent), now)
|
||||
older = (now - datetime.timedelta(minutes=15)).astimezone(datetime.timezone.utc).isoformat()
|
||||
assert sync.sync_is_due(config, dict(finished_at=older), now)
|
||||
assert not sync.sync_is_due(dict(window_start=None, window_end=None), dict(finished_at=None), now), \
|
||||
"without a window a sync only runs when asked"
|
||||
|
||||
def test_local_now_uses_wrolpis_timezone(self, media):
|
||||
(media / "config/wrolpi.yaml").write_text("timezone: Asia/Tokyo\n")
|
||||
assert sync.local_now().utcoffset() == datetime.timedelta(hours=9)
|
||||
(media / "config/wrolpi.yaml").write_text("timezone: Not/AZone\n")
|
||||
assert sync.local_now().tzinfo is not None
|
||||
|
||||
def test_tick(self, media, monkeypatch):
|
||||
runs = []
|
||||
monkeypatch.setattr(sync, "run_sync", lambda: runs.append(1))
|
||||
monkeypatch.setattr(sync, "local_now",
|
||||
lambda: datetime.datetime(2026, 11, 19, 3, 0).astimezone())
|
||||
sync.schedule_tick()
|
||||
assert runs == [], "no window"
|
||||
|
||||
(media / "config/sync.yaml").write_text(yaml.safe_dump(dict(
|
||||
role="secondary", primary_url=PRIMARY, window_start="01:00", window_end="05:00")))
|
||||
sync.schedule_tick()
|
||||
assert runs == [1]
|
||||
|
||||
(media / "config/.wrol_mode").touch()
|
||||
sync.schedule_tick()
|
||||
assert runs == [1], "WROL Mode stops the schedule"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_loop_survives_a_failure(self, monkeypatch):
|
||||
ticks = []
|
||||
|
||||
def tick():
|
||||
ticks.append(1)
|
||||
raise RuntimeError("the drive went away")
|
||||
|
||||
monkeypatch.setattr(sync, "schedule_tick", tick)
|
||||
monkeypatch.setattr(sync, "SCHEDULE_INTERVAL_SECONDS", 0.01)
|
||||
task = asyncio.create_task(sync.sync_schedule_loop())
|
||||
await asyncio.sleep(0.2)
|
||||
task.cancel()
|
||||
assert len(ticks) >= 3
|
||||
|
||||
|
||||
def test_fallback_page_shows_the_sync(media):
|
||||
"""The browser is sent to the Controller's page while the API is stopped; it says a sync is the reason."""
|
||||
response = TestClient(app).get("/")
|
||||
assert response.status_code == 200
|
||||
assert b'id="sync-card"' in response.content and b"api/sync/status" in response.content
|
||||
assert b"api/sync/cancel" in response.content
|
||||
|
||||
|
||||
def test_frontend_fixture_matches_the_status():
|
||||
"""The frontend's tests render the sync from controllerSyncStatusFixture; it must have the keys the Controller
|
||||
sends, or a renamed field passes those tests and breaks the page."""
|
||||
source = (Path(__file__).parent.parent.parent / "app/src/test-fixtures.js").read_text()
|
||||
match = re.search(r"export const controllerSyncStatusFixture = \(overrides = \{}\) => \(\{(.*?)\n}\);",
|
||||
source, re.S)
|
||||
assert match, "controllerSyncStatusFixture is not in test-fixtures.js in the expected form"
|
||||
keys = set(re.findall(r"^ ([a-z0-9_]+):", match.group(1), re.M))
|
||||
assert keys == set(sync.default_status())
|
||||
|
||||
|
||||
def test_docker_rsync_reads_progress_as_it_comes(media, monkeypatch):
|
||||
"""The helper's output is read by attaching to it before it starts. Docker's logs would hold rsync's progress
|
||||
for a large file (one line, rewritten with carriage returns) until the file was done."""
|
||||
from unittest import mock
|
||||
events = []
|
||||
helper = mock.Mock()
|
||||
helper.attach.side_effect = lambda **kwargs: events.append(("attach", kwargs)) or iter(
|
||||
[b" 1000 10%\r", b" 5000 50%\r", b"Number of regular files transferred: 1\n"])
|
||||
helper.start.side_effect = lambda: events.append(("start", None))
|
||||
helper.wait.return_value = {"StatusCode": 0}
|
||||
api = mock.Mock()
|
||||
api.image.id, api.name, api.attrs = "sha256:image", "wrolpi-api-1", {"Config": {"User": "1000:1000"}}
|
||||
client = mock.Mock()
|
||||
client.containers.create.return_value = helper
|
||||
client.containers.get.side_effect = lambda name: api if name == "wrolpi-api-1" else (_ for _ in ()).throw(
|
||||
RuntimeError("no such container"))
|
||||
monkeypatch.setattr(sync, "_get_client", lambda: client)
|
||||
|
||||
lines = []
|
||||
assert sync.DockerRunner().rsync(["--version"], lines.append) == 0
|
||||
|
||||
assert [i[0] for i in events] == ["attach", "start"], "attached before it starts"
|
||||
assert events[0][1]["stream"] is True
|
||||
assert lines == [" 1000 10%", " 5000 50%", "Number of regular files transferred: 1"]
|
||||
helper.logs.assert_not_called()
|
||||
helper.remove.assert_called_once_with(force=True)
|
||||
|
||||
|
||||
class TestRecoveryAndSchedule:
|
||||
"""A Controller that starts after an interrupted sync must recover it before the schedule starts another."""
|
||||
|
||||
def interrupted(self, media):
|
||||
(media / "config/sync.yaml").write_text(yaml.safe_dump(dict(
|
||||
role="secondary", primary_url=PRIMARY, window_start="00:00", window_end="23:59")))
|
||||
(media / "config/sync_state.yaml").write_text(yaml.safe_dump(dict(state="running", phase="files",
|
||||
finished_at=None)))
|
||||
|
||||
def test_tick_recovers_first_and_starts_nothing(self, media, monkeypatch):
|
||||
self.interrupted(media)
|
||||
runner, runs = FakeRunner(media), []
|
||||
monkeypatch.setattr(sync, "get_runner", lambda: runner)
|
||||
monkeypatch.setattr(sync, "run_sync", lambda: runs.append(1))
|
||||
monkeypatch.setattr(sync, "local_now", lambda: datetime.datetime.now().astimezone())
|
||||
|
||||
sync.schedule_tick()
|
||||
|
||||
assert runner.events == ["stop_rsync", "start_api"]
|
||||
assert sync.get_status()["state"] == "error" and sync.get_status()["last_error"] == "The sync was interrupted"
|
||||
assert runs == [], "the recovered sync just finished; the next one waits its 15 minutes"
|
||||
|
||||
def test_a_running_sync_is_not_recovered(self, media, monkeypatch):
|
||||
"""A status of "running" while this Controller's own sync holds the lock is no interrupted sync."""
|
||||
self.interrupted(media)
|
||||
runner = FakeRunner(media)
|
||||
assert sync._run_lock.acquire(blocking=False)
|
||||
try:
|
||||
sync.recover_interrupted_sync(runner)
|
||||
finally:
|
||||
sync._run_lock.release()
|
||||
assert runner.events == [] and sync.get_status()["state"] == "running"
|
||||
|
||||
def test_state_file_read_once_the_drive_is_mounted(self, media):
|
||||
"""The Controller may start before the drive is mounted: what it could not read is read later, not
|
||||
replaced by an empty record."""
|
||||
state = media / "config/sync_state.yaml"
|
||||
assert not state.exists()
|
||||
assert sync.get_status()["last_success_at"] is None
|
||||
state.write_text(yaml.safe_dump(dict(state="idle", last_success_at="2026-11-18T03:00:00+00:00")))
|
||||
assert sync.get_status()["last_success_at"] == "2026-11-18T03:00:00+00:00"
|
||||
# Once read, what is in memory is the record.
|
||||
sync.update_status(last_error="x")
|
||||
state.write_text(yaml.safe_dump(dict(state="idle", last_success_at=None)))
|
||||
assert sync.get_status()["last_success_at"] == "2026-11-18T03:00:00+00:00"
|
||||
|
||||
def test_outcome_is_recorded_before_the_lock_is_freed(self, media, primary_api, monkeypatch):
|
||||
"""Otherwise a tick between the two would take the sync for an interrupted one."""
|
||||
locked_when_saved = []
|
||||
update_status = sync.update_status
|
||||
|
||||
def watch(save=False, **changes):
|
||||
if save and changes.get("finished_at"):
|
||||
locked_when_saved.append(sync._run_lock.locked())
|
||||
return update_status(save=save, **changes)
|
||||
|
||||
monkeypatch.setattr(sync, "update_status", watch)
|
||||
sync.run_sync(FakeRunner(media))
|
||||
assert locked_when_saved == [True]
|
||||
|
|
|
|||
|
|
@ -9,7 +9,8 @@ from sanic_ext.extensions.openapi import openapi
|
|||
|
||||
from wrolpi import flags
|
||||
from wrolpi.api_utils import api_app, json_response, perpetual_signal
|
||||
from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled
|
||||
from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled, \
|
||||
CONTROLLER_URL, aiohttp_post
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled
|
||||
from wrolpi.sync import backup, lib, schema, snapshot
|
||||
|
|
@ -48,8 +49,7 @@ async def check_sync_api_lock(request: Request):
|
|||
|
||||
@perpetual_signal(sleep=5)
|
||||
async def perpetual_sync_worker():
|
||||
# One loop, so a secondary's sync and its backup never run at once; the backup copies what the sync pulled.
|
||||
await lib.sync_tick()
|
||||
# A secondary's sync is the Controller's (controller/lib/sync.py), which stops this API while it runs.
|
||||
await backup.backup_tick()
|
||||
await snapshot.snapshot_tick()
|
||||
|
||||
|
|
@ -93,25 +93,38 @@ async def post_snapshot(_: Request):
|
|||
return response.empty(HTTPStatus.ACCEPTED)
|
||||
|
||||
|
||||
def controller_url() -> str:
|
||||
return 'http://controller' if DOCKERIZED else CONTROLLER_URL
|
||||
|
||||
|
||||
async def _ask_controller(path: str):
|
||||
"""A secondary's sync runs in the Controller, because this API is stopped while the library is replaced."""
|
||||
try:
|
||||
async with aiohttp_post(f'{controller_url()}/api/sync/{path}', json_={}, timeout=15) as controller_response:
|
||||
if controller_response.status < 300:
|
||||
return response.empty(HTTPStatus.ACCEPTED)
|
||||
body = await controller_response.json(content_type=None)
|
||||
except Exception as e:
|
||||
logger.error(f'Could not reach the Controller to {path} the sync', exc_info=e)
|
||||
raise InvalidConfig('The Controller did not answer')
|
||||
raise InvalidConfig((body or {}).get('detail') or f'The Controller answered {controller_response.status}')
|
||||
|
||||
|
||||
@sync_bp.post('/run')
|
||||
@openapi.definition(summary='Sync from the primary now', body=schema.SyncRunRequest, validate=True)
|
||||
@openapi.definition(summary='Sync from the primary now (the Controller runs it)')
|
||||
@wrol_mode_check
|
||||
async def post_run(_: Request, body: schema.SyncRunRequest):
|
||||
async def post_run(_: Request):
|
||||
if not is_secondary():
|
||||
raise SecondaryRole('Only a secondary WROLPi syncs from a primary')
|
||||
if flags.sync_running.is_set():
|
||||
raise SyncInProgress()
|
||||
if not get_sync_config().primary_url:
|
||||
raise InvalidConfig('No primary is configured')
|
||||
lib.request_sync(force=body.force)
|
||||
return response.empty(HTTPStatus.ACCEPTED)
|
||||
return await _ask_controller('run')
|
||||
|
||||
|
||||
@sync_bp.post('/cancel')
|
||||
@openapi.definition(summary='Stop the running sync')
|
||||
@openapi.definition(summary='Stop the running sync (the Controller runs it)')
|
||||
async def post_cancel(_: Request):
|
||||
lib.request_sync_cancel()
|
||||
return response.empty(HTTPStatus.ACCEPTED)
|
||||
return await _ask_controller('cancel')
|
||||
|
||||
|
||||
@sync_bp.get('/test')
|
||||
|
|
|
|||
|
|
@ -18,12 +18,6 @@ class ManifestRequest:
|
|||
raise ValidationError('limit must be between 1 and 5000')
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncRunRequest:
|
||||
# Import every synced config even if the primary's copy did not change.
|
||||
force: bool = False
|
||||
|
||||
|
||||
@dataclass
|
||||
class SyncDirectoriesRequest:
|
||||
# The folder whose sub-folders are listed, relative to the media directory ('' is the media directory).
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import contextlib
|
||||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
|
|
@ -8,7 +9,7 @@ from wrolpi import flags
|
|||
from wrolpi.api_utils import api_app
|
||||
from wrolpi.common import get_wrolpi_config, enable_wrol_mode
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.sync import lib
|
||||
from wrolpi.sync import api as sync_api, backup, lib
|
||||
from wrolpi.sync.config import get_sync_config
|
||||
from wrolpi.sync.status import get_sync_status, update_sync_status
|
||||
|
||||
|
|
@ -38,19 +39,71 @@ async def test_info_and_manifest(async_client, test_session, test_wrolpi_config,
|
|||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
|
||||
|
||||
class FakeControllerResponse:
|
||||
def __init__(self, status, body=None):
|
||||
self.status, self.body = status, body
|
||||
|
||||
async def json(self, content_type=None):
|
||||
return self.body
|
||||
|
||||
|
||||
def fake_controller(monkeypatch, status=None, body=None, error=None):
|
||||
"""The Controller answers the API's sync requests with `status` and `body`, or not at all (`error`)."""
|
||||
calls = []
|
||||
|
||||
@contextlib.asynccontextmanager
|
||||
async def aiohttp_post(url, json_, timeout=None, headers=None):
|
||||
calls.append(url)
|
||||
if error:
|
||||
raise error
|
||||
yield FakeControllerResponse(status, body)
|
||||
|
||||
monkeypatch.setattr(sync_api, 'aiohttp_post', aiohttp_post)
|
||||
return calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run(async_client, test_wrolpi_config, make_secondary):
|
||||
async def test_run(async_client, test_wrolpi_config, make_secondary, monkeypatch):
|
||||
"""The Controller runs a secondary's sync, because it stops this API; the API only asks it to."""
|
||||
calls = fake_controller(monkeypatch, status=HTTPStatus.ACCEPTED)
|
||||
request, response = await async_client.post('/api/sync/run', content='{}')
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN, 'a primary does not sync'
|
||||
assert calls == []
|
||||
|
||||
make_secondary('https://primary.example')
|
||||
request, response = await async_client.post('/api/sync/run', content=json.dumps(dict(force=True)))
|
||||
request, response = await async_client.post('/api/sync/run', content='{}')
|
||||
assert response.status_code == HTTPStatus.ACCEPTED
|
||||
assert get_sync_status()['requested'] is True
|
||||
assert get_sync_status()['force'] is True
|
||||
assert calls == ['http://127.0.0.1/api/sync/run']
|
||||
request, response = await async_client.post('/api/sync/cancel')
|
||||
assert response.status_code == HTTPStatus.ACCEPTED
|
||||
assert calls[-1] == 'http://127.0.0.1/api/sync/cancel'
|
||||
|
||||
request, response = await async_client.get('/api/sync/status')
|
||||
assert response.json['sync']['requested'] is True
|
||||
# The Controller's reason is the API's.
|
||||
fake_controller(monkeypatch, status=HTTPStatus.CONFLICT, body=dict(detail='A sync is already running'))
|
||||
request, response = await async_client.post('/api/sync/run', content='{}')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json['error'] == 'A sync is already running'
|
||||
|
||||
fake_controller(monkeypatch, error=ConnectionRefusedError())
|
||||
request, response = await async_client.post('/api/sync/run', content='{}')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert response.json['error'] == 'The Controller did not answer'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_old_engine_does_not_run(async_client, test_wrolpi_config, make_secondary, monkeypatch):
|
||||
"""Only the Controller syncs now; the API's loop runs the backup drive and nothing else."""
|
||||
make_secondary('https://primary.example')
|
||||
lib.request_sync()
|
||||
called = []
|
||||
|
||||
async def tick(name):
|
||||
called.append(name)
|
||||
|
||||
monkeypatch.setattr(lib, 'sync_tick', lambda: tick('sync'))
|
||||
monkeypatch.setattr(backup, 'backup_tick', lambda: tick('backup'))
|
||||
await sync_api.perpetual_sync_worker()
|
||||
assert called == ['backup']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -162,7 +215,7 @@ async def test_directories_primary_unreachable(async_client, test_wrolpi_config,
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_lock, make_secondary):
|
||||
async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_lock, make_secondary, monkeypatch):
|
||||
"""While a secondary syncs, its API refuses everything but the sync's own progress."""
|
||||
make_secondary()
|
||||
with lib.lock_api():
|
||||
|
|
@ -175,6 +228,7 @@ async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_
|
|||
for path in ('/api/status', '/api/sync/status', '/api/settings', '/api/echo'):
|
||||
request, response = await async_client.get(path)
|
||||
assert response.status_code == HTTPStatus.OK, path
|
||||
fake_controller(monkeypatch, status=HTTPStatus.ACCEPTED)
|
||||
request, response = await async_client.post('/api/sync/cancel')
|
||||
assert response.status_code == HTTPStatus.ACCEPTED
|
||||
# Nothing may change the sync while it runs, except stopping it.
|
||||
|
|
|
|||
|
|
@ -273,8 +273,9 @@ async def test_which_fixtures_the_spec_can_vouch_for(async_client):
|
|||
api_fixtures = set(re.findall(r'^export const ([a-zA-Z]+Fixture) =', source, re.M))
|
||||
# domainsFixture is a list built from domainFixture, syncStatusFixture and backupStatusFixture are
|
||||
# statusFixture's `sync` and `backup` (test_sync_status_fixture_matches_the_seed holds them), and the *Context* fixtures are held to
|
||||
# their React contexts by the frontend's own test-fixtures.test.js.
|
||||
api_fixtures -= {'domainsFixture', 'syncStatusFixture', 'backupStatusFixture'}
|
||||
# their React contexts by the frontend's own test-fixtures.test.js. controllerSyncStatusFixture is the
|
||||
# Controller's, not this API's: controller/test/test_sync.py holds it to the Controller's sync status.
|
||||
api_fixtures -= {'domainsFixture', 'syncStatusFixture', 'backupStatusFixture', 'controllerSyncStatusFixture'}
|
||||
api_fixtures = {n for n in api_fixtures if 'Context' not in n}
|
||||
|
||||
assert api_fixtures == set(checked) | set(unchecked), (
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue