diff --git a/app/src/api.js b/app/src/api.js
index b69edded..15129c84 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -482,19 +482,6 @@ export async function getSyncDirectories(path, source) {
throw new Error(message);
}
-export async function runSync(force = false) {
- const response = await apiPost(`${API_URI}/sync/run`, {force});
- if (!response.ok) {
- const message = await getErrorMessage(response, 'Could not start the sync.');
- toast({type: 'error', title: 'Sync Failed', description: message, time: 5000});
- }
- return response;
-}
-
-export async function cancelSync() {
- return await apiPost(`${API_URI}/sync/cancel`);
-}
-
export async function runBackup() {
const response = await apiPost(`${API_URI}/sync/backup/run`);
if (!response.ok) {
diff --git a/app/src/api/controller.js b/app/src/api/controller.js
index 9e54a01c..feed933d 100644
--- a/app/src/api/controller.js
+++ b/app/src/api/controller.js
@@ -279,3 +279,18 @@ export async function healthCheck() {
export async function getControllerInfo() {
return controllerFetch('/info');
}
+
+// --- Sync (a secondary WROLPi copying its primary; the Controller runs it, because the API is stopped) ---
+
+export async function getSyncStatus() {
+ return controllerFetch('/sync/status');
+}
+
+/** Start a sync now. Throws with the reason a sync cannot start. */
+export async function runSync() {
+ return controllerFetch('/sync/run', {method: 'POST'});
+}
+
+export async function cancelSync() {
+ return controllerFetch('/sync/cancel', {method: 'POST'});
+}
diff --git a/app/src/components/admin/Sync.js b/app/src/components/admin/Sync.js
index 0af1ce19..1ce4ef92 100644
--- a/app/src/components/admin/Sync.js
+++ b/app/src/components/admin/Sync.js
@@ -2,15 +2,13 @@ import React from "react";
import {SettingsContext, StatusContext} from "../../contexts/contexts";
import {
cancelBackup,
- cancelSync,
getSyncDirectories,
PrimaryUnreachableError,
runBackup,
- runSync,
saveSyncSettings,
testSync
} from "../../api";
-import {getDisks} from "../../api/controller";
+import {cancelSync, getDisks, getSyncStatus, runSync} from "../../api/controller";
import {humanFileSize, isoDatetimeToAgoPopup, useTitle} from "../Common";
import {Link} from "react-router";
import {useDockerized, useSyncRole, useWROLMode} from "../../hooks/customHooks";
@@ -34,18 +32,67 @@ const roleOptions = [
{value: 'secondary', label: 'Secondary'},
];
-/** The sync's progress: how many of the primary's FileGroups have been compared. */
-function syncPercent(sync) {
- if (!sync || !sync.max_id) {
- return 0;
+// What each step of a sync is called (see controller/lib/sync.py).
+const SYNC_PHASES = {
+ checks: 'Checking the primary',
+ snapshot: 'The primary is copying its database',
+ files: 'Copying files',
+ database: 'Copying the database',
+ starting: 'Starting WROLPi',
+ purge: 'Emptying the trash',
+};
+
+export function syncProgressLabel(sync) {
+ const phase = SYNC_PHASES[sync?.phase] || 'Syncing';
+ if (sync?.phase === 'files' || sync?.phase === 'database') {
+ return `${phase}: ${humanFileSize(sync.bytes_transferred || 0)}`;
}
- // FileGroup ids are not contiguous; `groups_seen / max_id` underestimates, it never overshoots.
- return Math.min(100, Math.round(100 * sync.groups_seen / sync.max_id));
+ return phase;
}
-function syncProgressLabel(sync) {
- const downloaded = `${sync.files_downloaded} / ${sync.files_to_download} files`;
- return `${sync.phase === 'configs' ? 'Importing configs' : downloaded} (${humanFileSize(sync.bytes_downloaded)})`;
+// How often the sync's status is asked for, while one runs, and otherwise.
+const SYNC_POLL_RUNNING_MS = 2000;
+const SYNC_POLL_IDLE_MS = 10000;
+
+/**
+ * The sync's status, from the Controller: a secondary's API is stopped while it syncs, so the Controller is the
+ * only one that can say how the sync is going. Not asked for at all on a primary.
+ */
+export function useSyncStatus() {
+ const role = useSyncRole();
+ const [sync, setSync] = React.useState(null);
+ // The settings (and the role) cannot be read while the API is stopped; a role that is not known yet may be a
+ // secondary in the middle of a sync.
+ const enabled = role !== 'primary';
+ const running = sync?.state === 'running';
+
+ const fetchSync = React.useCallback(async () => {
+ try {
+ setSync(await getSyncStatus());
+ } catch (e) {
+ // The Controller is restarting; ask again later.
+ }
+ }, []);
+
+ React.useEffect(() => {
+ if (!enabled) {
+ setSync(null);
+ return;
+ }
+ fetchSync();
+ const interval = setInterval(fetchSync, running ? SYNC_POLL_RUNNING_MS : SYNC_POLL_IDLE_MS);
+ return () => clearInterval(interval);
+ }, [enabled, running, fetchSync]);
+
+ return {sync, running, fetchSync};
+}
+
+async function stopSync() {
+ try {
+ await cancelSync();
+ } catch (e) {
+ toast({type: 'error', title: 'Could not stop the sync', description: e.message, time: 5000});
+ }
}
/**
@@ -73,38 +120,38 @@ export function SecondaryRoleNotice({primaryUrl, content}) {
}
+// A sync that was stopped on purpose is not a problem.
+const STOPPED = ['The sync was cancelled', 'WROL Mode was enabled'];
+
/** The progress and outcome of a secondary's sync. */
export function SyncStatus({sync, running}) {
if (!sync) {
return null;
}
+ const stopped = STOPPED.includes(sync.last_error);
return
- {running &&
}
-/** Covers the app while a secondary syncs; its API refuses requests until the sync finishes. */
+/** Covers the app while a secondary syncs; its API is stopped until the sync finishes. */
export function SyncingOverlay() {
- const {status} = React.useContext(StatusContext);
- const role = useSyncRole();
- const running = role === 'secondary' && status?.flags?.sync_running === true;
+ const {sync, running} = useSyncStatus();
if (!running) {
return null;
@@ -114,7 +161,7 @@ export function SyncingOverlay() {
position: 'fixed', inset: 0, zIndex: 1000, display: 'flex', alignItems: 'center',
justifyContent: 'center', padding: 16, background: 'rgba(0, 0, 0, 0.6)',
}}>
-
+
}
@@ -126,7 +173,7 @@ export function SyncingPanel({sync, onCancel}) {
This WROLPi is copying the library of {sync?.primary_url || 'its primary'}. It is unavailable
until the sync finishes.
-
+
}
@@ -226,7 +273,7 @@ export function ExcludeFolderModal({open, onClose, value, onSubmit, source}) {
}
/** The folders a sync or a backup does not copy, and the button that chooses them. */
-export function ExcludedDirectories({value, onChange, source, disabled}) {
+export function ExcludedDirectories({value, onChange, source, disabled, note}) {
const [open, setOpen] = React.useState(false);
const excluded = value || [];
return
Files already copied into an excluded folder are kept; delete them yourself to free space.
+ {note && <> {note}>}
setOpen(false)} value={excluded} source={source}
onSubmit={chosen => {
@@ -254,7 +302,7 @@ export function ExcludedDirectories({value, onChange, source, disabled}) {
export function SyncSection() {
const {settings, fetchSettings} = React.useContext(SettingsContext);
- const {status} = React.useContext(StatusContext);
+ const {sync, running, fetchSync} = useSyncStatus();
const wrolMode = useWROLMode();
const [state, setState] = React.useState({});
const [testResult, setTestResult] = React.useState(null);
@@ -267,6 +315,7 @@ export function SyncSection() {
const windowStart = settings?.sync_window_start;
const windowEnd = settings?.sync_window_end;
const excludedKey = JSON.stringify(settings?.sync_excluded_directories || []);
+ const deleteAfterDays = settings?.sync_delete_after_days;
React.useEffect(() => {
if (loaded) {
setState({
@@ -275,9 +324,11 @@ export function SyncSection() {
sync_window_start: windowStart || '',
sync_window_end: windowEnd || '',
sync_excluded_directories: JSON.parse(excludedKey),
+ // null: never delete.
+ sync_delete_after_days: deleteAfterDays === undefined ? 30 : deleteAfterDays,
});
}
- }, [loaded, role, primaryUrl, windowStart, windowEnd, excludedKey]);
+ }, [loaded, role, primaryUrl, windowStart, windowEnd, excludedKey, deleteAfterDays]);
if (!settings) {
return null;
@@ -285,12 +336,24 @@ export function SyncSection() {
const secondary = state.role === 'secondary';
const saved = secondary && settings.role === 'secondary' && settings.sync_primary_url;
- const running = status?.flags?.sync_running === true;
+ const neverDelete = state.sync_delete_after_days === null;
const save = async () => {
+ // An empty box is not "delete at once": that would empty the trash at the next sync.
+ const days = String(state.sync_delete_after_days ?? '').trim();
+ if (secondary && !neverDelete && !/^\d+$/.test(days)) {
+ toast({
+ type: 'error', title: 'Sync settings were not saved', time: 5000,
+ description: 'Enter how many days to keep deleted files, or choose to never delete them.',
+ });
+ return;
+ }
setSaving(true);
try {
- const response = await saveSyncSettings(state);
+ // The API takes -1 for "never".
+ const response = await saveSyncSettings({
+ ...state, sync_delete_after_days: neverDelete ? -1 : Number(days),
+ });
if (response?.ok) {
toast({type: 'success', title: 'Sync settings saved', time: 3000});
setTestResult(null);
@@ -306,17 +369,20 @@ export function SyncSection() {
};
const syncNow = async () => {
- const response = await runSync();
- if (response?.ok) {
- toast({type: 'info', title: 'Sync requested', description: 'The sync starts in a few seconds.', time: 3000});
+ try {
+ await runSync();
+ toast({type: 'info', title: 'Sync started', time: 3000});
+ await fetchSync();
+ } catch (e) {
+ toast({type: 'error', title: 'The sync did not start', description: e.message, time: 5000});
}
};
return Sync
- A secondary WROLPi keeps a copy of a primary WROLPi's library: its files, tags,
- channels, domains, playlists, and other library settings. Files deleted on the primary are kept.
+ A secondary WROLPi keeps a copy of a primary WROLPi's library: all of its files (maps
+ and repositories too), its database, tags, channels, playlists, and other library settings.
@@ -326,8 +392,9 @@ export function SyncSection() {
{secondary && <>
- Every sync replaces this WROLPi's library with the primary's; changes made here are lost.
- It does not download, and it is unavailable while it syncs.
+ Every sync replaces this WROLPi's library and database with the primary's; changes made here are
+ lost. It does not download, and it is unavailable while it syncs. Both WROLPis must run the same
+ version.
@@ -353,11 +420,32 @@ export function SyncSection() {
Within the window, this secondary syncs every 15 minutes. Leave both empty to sync only with
"Sync Now".
+
+
+
+
+ Delete after
+ setState({...state, sync_delete_after_days: e.currentTarget.value})}/>
+ days
+
+ setState({...state, sync_delete_after_days: neverDelete ? 30 : null})}/>
+
+
+ A file the primary deletes, or the old version of one it changes, waits in a trash folder
+ (.sync-trash) for this long. It is removed sooner if the drive is nearly full.
+
+
>}
{/* The primary's folders can only be listed once this WROLPi knows its primary. */}
{secondary && saved && setState({
...state,
sync_excluded_directories: value
@@ -379,7 +467,7 @@ export function SyncSection() {
WROL Mode is enabled; this WROLPi does not sync.
}
- {settings.role === 'secondary' && }
+ {settings.role === 'secondary' && }
}
diff --git a/app/src/components/admin/Sync.test.js b/app/src/components/admin/Sync.test.js
index 627e1c54..1ae338c3 100644
--- a/app/src/components/admin/Sync.test.js
+++ b/app/src/components/admin/Sync.test.js
@@ -1,21 +1,24 @@
import React from 'react';
import {fireEvent, screen, waitFor} from '@testing-library/react';
import {renderWithProviders} from '../../test-utils';
-import {backupStatusFixture, settingsFixture, statusFixture, syncStatusFixture} from '../../test-fixtures';
+import {backupStatusFixture, controllerSyncStatusFixture, settingsFixture, statusFixture} from '../../test-fixtures';
import {
BackupDriveSection, backupDriveOptions, ExcludedDirectories, isExcludedPath, SecondaryRoleMessage, simplifyExcluded,
splitBackupDirectory, SyncingOverlay, SyncSection,
} from './Sync';
+jest.mock('../ui/toast', () => ({...jest.requireActual('../ui/toast'), toast: jest.fn()}));
+
jest.mock('../../api/controller', () => ({
getDisks: jest.fn(),
+ getSyncStatus: jest.fn(),
+ runSync: jest.fn(),
+ cancelSync: jest.fn(),
}));
jest.mock('../../api', () => ({
cancelBackup: jest.fn(),
runBackup: jest.fn(),
- cancelSync: jest.fn(),
- runSync: jest.fn(),
saveSyncSettings: jest.fn(),
testSync: jest.fn(),
getSyncDirectories: jest.fn(),
@@ -24,14 +27,16 @@ jest.mock('../../api', () => ({
}));
import {
- cancelBackup, cancelSync, getSyncDirectories, PrimaryUnreachableError, runBackup, runSync, saveSyncSettings, testSync,
+ cancelBackup, getSyncDirectories, PrimaryUnreachableError, runBackup, saveSyncSettings, testSync,
} from '../../api';
-import {getDisks} from '../../api/controller';
+import {cancelSync, getDisks, getSyncStatus, runSync} from '../../api/controller';
+import {toast} from '../ui/toast';
beforeEach(() => {
// react-scripts resets mock implementations before each test.
- cancelSync.mockResolvedValue({ok: true});
- runSync.mockResolvedValue({ok: true});
+ cancelSync.mockResolvedValue({success: true});
+ runSync.mockResolvedValue({success: true});
+ getSyncStatus.mockResolvedValue(controllerSyncStatusFixture());
saveSyncSettings.mockResolvedValue({ok: true});
testSync.mockResolvedValue({ok: true, info: {version: '0.30.5-beta'}});
runBackup.mockResolvedValue({ok: true});
@@ -76,29 +81,39 @@ describe('SecondaryRoleMessage', () => {
});
describe('SyncingOverlay', () => {
- const syncing = (sync = {}) => ({
- status: statusFixture({
- flags: {sync_running: true},
- sync: syncStatusFixture({state: 'running', phase: 'files', groups_seen: 50, max_id: 100, ...sync}),
- }),
+ const running = (sync = {}) => controllerSyncStatusFixture({
+ state: 'running', phase: 'files', percent: 40, bytes_transferred: 5 * 1024 ** 3,
+ primary_url: 'https://192.168.0.10', ...sync,
});
- it('covers the app while a secondary syncs', () => {
- renderWithProviders(, {settings: secondary(), status: syncing()});
- expect(screen.getByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
+ it('covers the app while a secondary syncs, from the Controller (the API is stopped)', async () => {
+ getSyncStatus.mockResolvedValue(running());
+ renderWithProviders(, {settings: secondary()});
+ expect(await screen.findByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
expect(screen.getByText('Syncing from the primary')).toBeInTheDocument();
+ expect(screen.getByText(/Copying files: 5/)).toBeInTheDocument();
fireEvent.click(screen.getByText('Stop Sync'));
- expect(cancelSync).toHaveBeenCalled();
+ await waitFor(() => expect(cancelSync).toHaveBeenCalled());
});
- it('is absent when not syncing', () => {
+ it('is shown when the settings cannot be read, as while the API is stopped', async () => {
+ getSyncStatus.mockResolvedValue(running());
+ renderWithProviders(, {settings: {settings: null}});
+ expect(await screen.findByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument();
+ });
+
+ it('is absent when not syncing', async () => {
renderWithProviders(, {settings: secondary()});
+ await waitFor(() => expect(getSyncStatus).toHaveBeenCalled());
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
});
- it('is absent on a primary', () => {
- renderWithProviders(, {status: syncing()});
+ it('is absent on a primary, which never asks', async () => {
+ getSyncStatus.mockResolvedValue(running());
+ renderWithProviders();
+ await new Promise(resolve => setTimeout(resolve, 50));
+ expect(getSyncStatus).not.toHaveBeenCalled();
expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument();
});
});
@@ -115,11 +130,16 @@ describe('SyncSection', () => {
});
it('a secondary shows its primary, window, and sync controls', async () => {
- const status = {status: statusFixture({sync: syncStatusFixture({last_error: 'The primary did not answer'})})};
- renderWithProviders(, {settings: secondary({sync_window_start: '01:00'}), status});
+ getSyncStatus.mockResolvedValue(controllerSyncStatusFixture({
+ finished_at: '2026-11-19T03:00:00+00:00', last_error: 'The primary did not answer',
+ files_transferred: 12, files_deleted: 3, trash_bytes: 2048,
+ }));
+ renderWithProviders(, {settings: secondary({sync_window_start: '01:00'})});
expect(screen.getByDisplayValue('https://192.168.0.10')).toBeInTheDocument();
expect(screen.getByDisplayValue('01:00')).toBeInTheDocument();
- expect(screen.getByText('The primary did not answer')).toBeInTheDocument();
+ expect(await screen.findByText('The primary did not answer')).toBeInTheDocument();
+ expect(screen.getByText(/12 files copied/)).toBeInTheDocument();
+ expect(screen.getByText(/3 moved to the trash/)).toBeInTheDocument();
fireEvent.click(screen.getByText('Sync Now'));
await waitFor(() => expect(runSync).toHaveBeenCalled());
@@ -127,10 +147,55 @@ describe('SyncSection', () => {
fireEvent.click(screen.getByText('Save'));
await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({
role: 'secondary', sync_primary_url: 'https://192.168.0.10', sync_window_start: '01:00',
- sync_window_end: '', sync_excluded_directories: [],
+ sync_window_end: '', sync_excluded_directories: [], sync_delete_after_days: 30,
}));
});
+ it('says why a sync did not start', async () => {
+ runSync.mockRejectedValue(new Error('A sync is already running'));
+ renderWithProviders(, {settings: secondary()});
+ fireEvent.click(screen.getByText('Sync Now'));
+ await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({
+ type: 'error', title: 'The sync did not start', description: 'A sync is already running'})));
+ });
+
+ it('a stopped sync is not a problem', async () => {
+ getSyncStatus.mockResolvedValue(controllerSyncStatusFixture({last_error: 'The sync was cancelled'}));
+ renderWithProviders(, {settings: secondary()});
+ expect(await screen.findByText('The sync was cancelled.')).toBeInTheDocument();
+ expect(screen.queryByText('Sync problem')).not.toBeInTheDocument();
+ });
+
+ it('saves how long deleted files are kept, or never to delete them', async () => {
+ renderWithProviders(, {settings: secondary({sync_delete_after_days: 7})});
+ expect(screen.getByLabelText('Days')).toHaveValue(7);
+
+ fireEvent.click(screen.getByLabelText('Never delete them'));
+ expect(screen.getByLabelText('Days')).toBeDisabled();
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
+ expect.objectContaining({sync_delete_after_days: -1})));
+ });
+
+ it('an empty number of days is not saved as "delete at once"', async () => {
+ renderWithProviders(, {settings: secondary({sync_delete_after_days: 30})});
+ fireEvent.change(screen.getByLabelText('Days'), {target: {value: ''}});
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(toast).toHaveBeenCalledWith(expect.objectContaining({
+ type: 'error', title: 'Sync settings were not saved'})));
+ expect(saveSyncSettings).not.toHaveBeenCalled();
+
+ fireEvent.change(screen.getByLabelText('Days'), {target: {value: '0'}});
+ fireEvent.click(screen.getByText('Save'));
+ await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith(
+ expect.objectContaining({sync_delete_after_days: 0})));
+ });
+
+ it('a secondary that never deletes shows it', () => {
+ renderWithProviders(, {settings: secondary({sync_delete_after_days: null})});
+ expect(screen.getByLabelText('Never delete them')).toBeChecked();
+ });
+
it('WROL Mode stops syncing', () => {
renderWithProviders(, {settings: secondary({wrol_mode: true})});
expect(screen.getByText('Syncing is stopped')).toBeInTheDocument();
diff --git a/app/src/test-fixtures.js b/app/src/test-fixtures.js
index 0e71bb23..7c19f160 100644
--- a/app/src/test-fixtures.js
+++ b/app/src/test-fixtures.js
@@ -123,6 +123,25 @@ export const syncStatusFixture = (overrides = {}) => ({
...overrides,
});
+/** GET /controller/api/sync/status: a secondary's sync that has never run (controller/lib/sync.py). */
+export const controllerSyncStatusFixture = (overrides = {}) => ({
+ state: 'idle',
+ phase: null,
+ started_at: null,
+ finished_at: null,
+ last_success_at: null,
+ last_error: null,
+ primary_url: null,
+ primary_version: null,
+ percent: 0,
+ bytes_transferred: 0,
+ files_transferred: 0,
+ files_deleted: 0,
+ trash_bytes: 0,
+ warnings: [],
+ ...overrides,
+});
+
/** The `backup` of GET /api/status: a backup drive that has never been written. */
export const backupStatusFixture = (overrides = {}) => ({
state: 'idle',
diff --git a/controller/lib/sync.py b/controller/lib/sync.py
index e66698b3..670545bb 100644
--- a/controller/lib/sync.py
+++ b/controller/lib/sync.py
@@ -29,10 +29,13 @@ import subprocess
import threading
import time
import uuid
+import zoneinfo
from pathlib import Path
from typing import Callable, Optional
from urllib.parse import urlparse
+import asyncio
+
import httpx
import yaml
@@ -58,6 +61,10 @@ STATE_FILE = "config/sync_state.yaml"
DEFAULT_DELETE_AFTER_DAYS = 30
# Never fill the secondary's drive; leave this much free.
MIN_FREE_BYTES = 1024 ** 3
+# Within the daily sync window, a sync starts this long after the last one finished.
+SYNC_REPEAT_SECONDS = 15 * 60
+# How often the schedule is looked at.
+SCHEDULE_INTERVAL_SECONDS = 60
SNAPSHOT_TIMEOUT_SECONDS = 60 * 60
API_START_TIMEOUT_SECONDS = 10 * 60
# A sync runs unattended with the API stopped: a primary that loses power or its link must end the run, not hold it.
@@ -118,6 +125,9 @@ def default_status() -> dict:
_status = None
+# True once the state file was read or written: until then the drive may not be mounted, and what is in memory must
+# not be mistaken for (or saved over) what the last sync recorded.
+_status_from_file = False
_status_lock = threading.Lock()
_run_lock = threading.Lock()
_cancel = threading.Event()
@@ -132,16 +142,19 @@ def _state_file() -> Path:
def get_status() -> dict:
- """The progress and outcome of the sync. Read from the state file the first time, so a restarted Controller
- still knows when the last sync succeeded."""
- global _status
+ """The progress and outcome of the sync. Read from the state file, so a restarted Controller still knows when
+ the last sync succeeded; read again until that works, as the Controller may start before the drive is mounted."""
+ global _status, _status_from_file
with _status_lock:
if _status is None:
_status = default_status()
+ if not _status_from_file:
try:
saved = yaml.safe_load(checked_path(_state_file()).read_text())
if isinstance(saved, dict):
+ _status = default_status()
_status.update({k: v for k, v in saved.items() if k in _status})
+ _status_from_file = True
except (OSError, yaml.YAMLError, SyncError):
pass
return dict(_status)
@@ -154,8 +167,10 @@ def update_status(save: bool = False, **changes) -> dict:
_status.update(changes)
status = dict(_status)
if save:
+ global _status_from_file
try:
write_file(_state_file(), yaml.safe_dump(status))
+ _status_from_file = True
except (OSError, SyncError) as e:
logger.warning("Could not save the sync state: %s", e)
return status
@@ -374,11 +389,15 @@ class DockerRunner:
def rsync(self, arguments: list, on_line: Callable[[str], None]) -> int:
client, api = _get_client(), self._api()
self.stop_rsync()
- helper = client.containers.run(
- api.image.id, arguments, entrypoint="rsync", name=self.HELPER_NAME, detach=True,
+ helper = client.containers.create(
+ api.image.id, arguments, entrypoint="rsync", name=self.HELPER_NAME,
user=api.attrs["Config"].get("User") or None, volumes_from=[api.name])
try:
- stream = helper.logs(stream=True, follow=True)
+ # Attached, not followed through the logs: Docker's logs hold a line until it ends, and rsync's progress
+ # for one large file is a single line, rewritten with carriage returns until the file is done.
+ # Attached before it starts, so nothing it writes is missed.
+ stream = helper.attach(stdout=True, stderr=True, stream=True)
+ helper.start()
def read(_: int = 0) -> bytes:
return next(stream, b"")
@@ -672,16 +691,17 @@ def run_sync(runner=None) -> dict:
def _finish(runner, api_stopped: bool, **changes) -> dict:
- """Whatever happened, the API runs again and the lock is free."""
+ """Whatever happened, the API runs again and the lock is free. The outcome is recorded before the lock is
+ freed: a status that says "running" while no sync holds the lock is taken for an interrupted sync."""
try:
if api_stopped:
runner.start_api()
+ trash = get_media_directory() / TRASH_DIRECTORY
+ return update_status(save=True, phase=None, finished_at=_now(),
+ trash_bytes=directory_size(trash) if trash.is_dir() and not trash.is_symlink() else 0,
+ **changes)
finally:
_run_lock.release()
- trash = get_media_directory() / TRASH_DIRECTORY
- return update_status(save=True, phase=None, finished_at=_now(),
- trash_bytes=directory_size(trash) if trash.is_dir() and not trash.is_symlink() else 0,
- **changes)
def _read_yaml(file: Path) -> dict:
@@ -698,10 +718,11 @@ def request_cancel():
def recover_interrupted_sync(runner=None):
- """Called when the Controller starts. A sync that was running when the Controller stopped left the API stopped;
- start it, and say the sync was interrupted."""
+ """A status that says "running" while no sync holds the lock was left by a Controller that stopped during a
+ sync, which left the API stopped: start it, and say the sync was interrupted. Looked at by every tick of the
+ schedule, before it starts anything, and so also once the state file can be read after the drive mounts."""
status = get_status()
- if status["state"] != "running":
+ if status["state"] != "running" or _run_lock.locked():
return
logger.warning("A sync was interrupted; starting the API")
runner = runner or get_runner()
@@ -712,3 +733,64 @@ def recover_interrupted_sync(runner=None):
runner.start_api()
update_status(save=True, state="error", phase=None, finished_at=_now(),
last_error="The sync was interrupted")
+
+
+# ---------------------------------------------------------------------------------------------------------------------
+# The schedule.
+
+
+def local_now() -> datetime.datetime:
+ """Now, in WROLPi's timezone (wrolpi.yaml), or the system's when none is set."""
+ timezone = _read_yaml(get_media_directory() / "config/wrolpi.yaml").get("timezone")
+ if timezone:
+ with contextlib.suppress(zoneinfo.ZoneInfoNotFoundError, ValueError):
+ return datetime.datetime.now(zoneinfo.ZoneInfo(timezone))
+ return datetime.datetime.now().astimezone()
+
+
+def _parse_time(value) -> Optional[datetime.time]:
+ with contextlib.suppress(TypeError, ValueError):
+ return datetime.time.fromisoformat(str(value))
+ return None
+
+
+def in_window(start: Optional[str], end: Optional[str], now: datetime.time) -> bool:
+ """True if `now` is within the daily window from `start` to `end`, which may run past midnight."""
+ start, end = _parse_time(start), _parse_time(end)
+ if start is None or end is None:
+ return False
+ if start <= end:
+ return start <= now < end
+ return now >= start or now < end
+
+
+def sync_is_due(config: dict, status: dict, now: datetime.datetime) -> bool:
+ """True if the daily sync window is open and the last sync finished long enough ago. No window: never (a sync
+ only runs when asked)."""
+ if not in_window(config.get("window_start"), config.get("window_end"), now.time()):
+ return False
+ with contextlib.suppress(TypeError, ValueError):
+ finished_at = datetime.datetime.fromisoformat(status["finished_at"])
+ return (now - finished_at).total_seconds() >= SYNC_REPEAT_SECONDS
+ return True
+
+
+def schedule_tick():
+ """Start a sync if one is due. A sync that cannot start (WROL Mode, a primary, no primary) is not due."""
+ recover_interrupted_sync()
+ config = read_sync_config()
+ if skip_reason(config) is not None:
+ return
+ if sync_is_due(config, get_status(), local_now()):
+ logger.info("The sync window is open")
+ run_sync()
+
+
+async def sync_schedule_loop():
+ """Sync within the daily window for as long as the Controller runs."""
+ while True:
+ try:
+ await asyncio.to_thread(schedule_tick)
+ except Exception as e:
+ logger.error("The sync schedule failed", exc_info=e)
+ await asyncio.sleep(SCHEDULE_INTERVAL_SECONDS)
diff --git a/controller/main.py b/controller/main.py
index 7f8a4230..bffb946e 100644
--- a/controller/main.py
+++ b/controller/main.py
@@ -59,7 +59,7 @@ from controller.lib.status import (
get_uptime_status,
)
from controller.lib.status_worker import status_worker_loop
-from controller.lib.sync import recover_interrupted_sync
+from controller.lib.sync import sync_schedule_loop
from controller.lib.systemd import get_all_services_status
# Templates directory
@@ -126,8 +126,9 @@ async def lifespan(app: FastAPI):
else:
logger.info("Primary drive not mounted - using default configuration")
- # A sync that was running when the Controller stopped left the API stopped.
- app.state.sync_recovery_task = asyncio.create_task(asyncio.to_thread(recover_interrupted_sync))
+ # A secondary syncs within its daily window. Each tick first recovers a sync that was interrupted (the API it
+ # stopped is started again), so nothing races that recovery.
+ app.state.sync_schedule_task = asyncio.create_task(sync_schedule_loop())
# Docker: the library is served to secondary WROLPis from a helper container, kept matching the api container.
if is_docker_mode():
@@ -152,8 +153,9 @@ async def lifespan(app: FastAPI):
await status_task
except asyncio.CancelledError:
pass
- if rsync_daemon_task := getattr(app.state, "rsync_daemon_task", None):
- rsync_daemon_task.cancel()
+ for name in ("rsync_daemon_task", "sync_schedule_task"):
+ if task := getattr(app.state, name, None):
+ task.cancel()
# Create FastAPI app
diff --git a/controller/templates/index.html b/controller/templates/index.html
index 68282d4e..f8ddd478 100644
--- a/controller/templates/index.html
+++ b/controller/templates/index.html
@@ -169,6 +169,33 @@
display: block;
}
+ /* A secondary WROLPi copying its primary; its API is stopped meanwhile. */
+ .sync-card {
+ border: 2px solid #3498db;
+ border-radius: 4px;
+ padding: 0.75rem 1rem;
+ margin-bottom: 1rem;
+ display: none;
+ }
+
+ .sync-card.show {
+ display: block;
+ }
+
+ .sync-card .sync-bar {
+ height: 0.6rem;
+ background: rgba(127, 127, 127, 0.3);
+ border-radius: 3px;
+ overflow: hidden;
+ margin: 0.5rem 0;
+ }
+
+ .sync-card .sync-bar div {
+ height: 100%;
+ background: #3498db;
+ width: 0;
+ }
+
.fallback-banner .loading {
border-color: #333;
border-top-color: #000;
@@ -945,6 +972,15 @@
View Files
+
+ Syncing from the primary
+
+
+
WROLPi is unavailable until the sync finishes, then this page
+ returns to it.
+
+
+
Main UI is unavailable. Using Controller fallback. Will redirect when ready...
@@ -3077,6 +3113,52 @@
refreshToggleButtons();
}, updateInterval);
}
+
+ // A secondary WROLPi's sync (controller/lib/sync.py). Its API is stopped while it runs, which is why this page
+ // is shown; say what is happening. Relative URLs: /controller/api/... behind Caddy, /api/... when direct.
+ (function () {
+ const PHASES = {
+ checks: 'Checking the primary', snapshot: 'The primary is copying its database',
+ files: 'Copying files', database: 'Copying the database', starting: 'Starting WROLPi',
+ purge: 'Emptying the trash',
+ };
+ const card = document.getElementById('sync-card');
+ const text = document.getElementById('sync-text');
+ const bar = document.getElementById('sync-bar');
+ const stop = document.getElementById('sync-stop');
+
+ async function checkSync() {
+ try {
+ const resp = await fetch('api/sync/status', {signal: AbortSignal.timeout(5000)});
+ if (!resp.ok) {
+ return;
+ }
+ const sync = await resp.json();
+ const running = sync.state === 'running';
+ card.classList.toggle('show', running);
+ if (running) {
+ const copied = (sync.phase === 'files' || sync.phase === 'database')
+ ? ` (${formatBytes(sync.bytes_transferred || 0)})` : '';
+ text.textContent = `${PHASES[sync.phase] || 'Syncing'}${copied} from ${sync.primary_url || 'the primary'}`;
+ bar.style.width = `${Math.min(100, sync.percent || 0)}%`;
+ }
+ } catch (e) {
+ // The Controller is busy or restarting; ask again.
+ }
+ }
+
+ stop.addEventListener('click', async () => {
+ stop.disabled = true;
+ try {
+ await fetch('api/sync/cancel', {method: 'POST'});
+ } finally {
+ setTimeout(() => { stop.disabled = false; }, 3000);
+ }
+ });
+
+ checkSync();
+ setInterval(checkSync, 2000);
+ })();