diff --git a/app/src/App.js b/app/src/App.js index 088cfa95..37c210c8 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -15,6 +15,7 @@ import {MoreRoute} from "./components/Apps"; import {InventoryRoute} from "./components/inventory/InventoryRoute"; import {ArchiveRoute} from "./components/Archive"; import {FilesRoute} from "./components/Files"; +import {SyncingOverlay} from "./components/admin/Sync"; import {QueryProvider, StatusProvider} from "./hooks/customHooks"; import {FileWorkerStatusProvider} from "./contexts/FileWorkerStatusContext"; import {BookmarksProvider} from "./contexts/BookmarksContext"; @@ -75,6 +76,7 @@ function Root() {
+ diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js index 1ab785ab..0a32b0ee 100644 --- a/app/src/DashboardPage.js +++ b/app/src/DashboardPage.js @@ -8,6 +8,7 @@ import { useLocalStorage, useTitle } from "./components/Common"; +import {SecondaryRoleMessage} from "./components/admin/Sync"; import React, {useContext, useState} from "react"; import {useHotkeys} from "react-hotkeys-hook"; import {Media, SettingsContext, StatusContext} from "./contexts/contexts"; @@ -436,6 +437,7 @@ export function DashboardPage() { // Only show dashboard parts if not searching. let body = + diff --git a/app/src/Events.js b/app/src/Events.js index a9959b44..d43b5073 100644 --- a/app/src/Events.js +++ b/app/src/Events.js @@ -46,6 +46,22 @@ export function handleEvents(events) { eventToast('Downloads Disabled', message); } + if (event === 'backup_completed') { + eventToast('Backup Completed', message, 'success'); + } + + if (event === 'backup_failed') { + eventToast('Backup Failed', message, 'error'); + } + + if (event === 'sync_completed') { + eventToast('Sync Completed', message, 'success'); + } + + if (event === 'sync_failed') { + eventToast('Sync Failed', message, 'error'); + } + if (event === 'map_import_complete') { eventToast('Map import completed', message, 'success'); } diff --git a/app/src/api.getSettings.test.js b/app/src/api.getSettings.test.js new file mode 100644 index 00000000..4574f766 --- /dev/null +++ b/app/src/api.getSettings.test.js @@ -0,0 +1,32 @@ +import {getSettings} from './api'; +import {settingsFixture} from './test-fixtures'; + +describe('getSettings', () => { + afterEach(() => { + delete global.fetch; + }); + + it('keeps every setting the API sends', async () => { + // getSettings copies each field by name, so a field the API added but nobody listed here vanished + // silently (the Sync tab showed no backup drive while one was configured). + const sent = settingsFixture({ + backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup', + role: 'secondary', sync_primary_url: 'https://10.0.0.5', sync_window_start: '01:00', + sync_window_end: '05:00', + }); + global.fetch = jest.fn(() => Promise.resolve({ + ok: true, status: 200, + json: () => Promise.resolve(sent), + clone() { + return this; + }, + })); + + const settings = await getSettings(); + + const dropped = Object.keys(sent).filter(key => !(key in settings)); + expect(dropped).toEqual([]); + expect(settings.backup_directory).toBe('/media/wrolpi_backup'); + expect(settings.backup_device_uuid).toBe('uuid-backup'); + }); +}); diff --git a/app/src/api.js b/app/src/api.js index e46553da..252e372b 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -34,6 +34,11 @@ export class ApiDownError extends Error { const apiDownError = new ApiDownError('API is down'); +// A secondary WROLPi refuses requests while it copies its library from its primary. Like a down API, callers +// treat it quietly; the app shows the sync's progress instead (see SyncingOverlay). +export class SyncingError extends ApiDownError { +} + async function apiCall(url, method, body, ms = 60_000) { let init = {method}; if (body !== undefined) { @@ -58,8 +63,11 @@ async function apiCall(url, method, body, ms = 60_000) { // Request encountered an error. let copy = response.clone(); const content = await copy.json(); - console.error('API error response json', content); const code = content['code']; + if (response.status === 503 && code === 'SYNCING') { + throw new SyncingError('This WROLPi is syncing from its primary'); + } + console.error('API error response json', content); if (response.status === 403 && code === 'WROL_MODE_ENABLED') { toast({ type: 'error', @@ -384,6 +392,11 @@ export async function getSettings() { const content = await response.json(); return { archive_destination: content.archive_destination, + backup_device_uuid: content.backup_device_uuid, + backup_directory: content.backup_directory, + backup_excluded_directories: content.backup_excluded_directories, + backup_window_end: content.backup_window_end, + backup_window_start: content.backup_window_start, download_manager_disabled: content.download_manager_disabled, download_manager_stopped: content.download_manager_stopped, download_daily_limit_global: content.download_daily_limit_global, @@ -410,11 +423,17 @@ export async function getSettings() { repos_destination: content.repos_destination, require_cookies_unlocked: content.require_cookies_unlocked, require_media_mounted: content.require_media_mounted, + role: content.role, save_ffprobe_json: content.save_ffprobe_json, + sync_excluded_directories: content.sync_excluded_directories, + sync_primary_url: content.sync_primary_url, + sync_window_end: content.sync_window_end, + sync_window_start: content.sync_window_start, tags_directory: content.tags_directory, throttle_on_startup: content.throttle_on_startup, throttle_status: content.throttle_status, timezone: content.timezone, + version: content.version, videos_destination: content.videos_destination, wrol_mode: content.wrol_mode, zims_destination: content.zims_destination, @@ -430,6 +449,70 @@ export async function getSettings() { } } +// Sync: a secondary WROLPi copies its library from its primary. + +export async function saveSyncSettings(settings) { + const response = await apiPut(`${API_URI}/sync/settings`, settings); + if (!response.ok) { + const message = await getErrorMessage(response, 'Could not save sync settings.'); + toast({type: 'error', title: 'Saving Sync Settings Failed', description: message, time: 5000}); + } + return response; +} + +export class PrimaryUnreachableError extends Error { +} + +/** The folders directly inside `path`; `source` is 'local' (this WROLPi) or 'primary' (its primary). */ +export async function getSyncDirectories(path, source) { + const response = await apiPost(`${API_URI}/sync/directories`, {path: path || '', source}); + if (response.ok) { + const {directories} = await response.json(); + return directories; + } + if (response.status === 502) { + const {error} = await response.json(); + throw new PrimaryUnreachableError(error || 'The primary did not answer'); + } + const message = await getErrorMessage(response, 'Could not list the folders.'); + throw new Error(message); +} + +export async function runSync(force = false) { + const response = await apiPost(`${API_URI}/sync/run`, {force}); + if (!response.ok) { + const message = await getErrorMessage(response, 'Could not start the sync.'); + toast({type: 'error', title: 'Sync Failed', description: message, time: 5000}); + } + return response; +} + +export async function cancelSync() { + return await apiPost(`${API_URI}/sync/cancel`); +} + +export async function runBackup() { + const response = await apiPost(`${API_URI}/sync/backup/run`); + if (!response.ok) { + const message = await getErrorMessage(response, 'Could not start the backup.'); + toast({type: 'error', title: 'Backup Failed', description: message, time: 5000}); + } + return response; +} + +export async function cancelBackup() { + return await apiPost(`${API_URI}/sync/backup/cancel`); +} + +export async function testSync() { + const response = await apiGet(`${API_URI}/sync/test`); + if (response.ok) { + return await response.json(); + } + const message = await getErrorMessage(response, 'Could not reach the primary.'); + return {ok: false, error: message}; +} + export async function saveSettings(settings) { const response = await apiPatch(`${API_URI}/settings`, settings); if (!response.ok) { diff --git a/app/src/components/admin/AdminRoute.js b/app/src/components/admin/AdminRoute.js index 3d5b4295..b0e777a1 100644 --- a/app/src/components/admin/AdminRoute.js +++ b/app/src/components/admin/AdminRoute.js @@ -6,6 +6,7 @@ import {SettingsPage} from "./Settings"; import {StatusPage} from "./Status"; import {ControllerPage} from "./ControllerPage"; import {ExtensionPage} from "./ExtensionPage"; +import {SyncPage} from "./Sync"; export default function AdminRoute() { @@ -13,6 +14,7 @@ export default function AdminRoute() { {text: 'Downloads', to: '/admin', key: 'admin', end: true}, {text: 'Settings', to: '/admin/settings', key: 'settings'}, {text: 'Status', to: '/admin/status', key: 'status'}, + {text: 'Sync', to: '/admin/sync', key: 'sync'}, {text: 'Control', to: '/admin/controller', key: 'controller'}, {text: 'Extension', to: '/admin/extension', key: 'extension'}, ]; @@ -23,6 +25,7 @@ export default function AdminRoute() { }/> }/> }/> + }/> }/> }/> diff --git a/app/src/components/admin/Downloads.js b/app/src/components/admin/Downloads.js index 198f2862..98ec5ead 100644 --- a/app/src/components/admin/Downloads.js +++ b/app/src/components/admin/Downloads.js @@ -26,6 +26,7 @@ import { useTitle, WROLModeMessage } from "../Common"; +import {SecondaryRoleMessage} from "./Sync"; import { Button, ButtonGroup, @@ -715,6 +716,8 @@ export function DownloadsPage() { return <> + diff --git a/app/src/components/admin/Sync.js b/app/src/components/admin/Sync.js new file mode 100644 index 00000000..0af1ce19 --- /dev/null +++ b/app/src/components/admin/Sync.js @@ -0,0 +1,575 @@ +import React from "react"; +import {SettingsContext, StatusContext} from "../../contexts/contexts"; +import { + cancelBackup, + cancelSync, + getSyncDirectories, + PrimaryUnreachableError, + runBackup, + runSync, + saveSyncSettings, + testSync +} from "../../api"; +import {getDisks} from "../../api/controller"; +import {humanFileSize, isoDatetimeToAgoPopup, useTitle} from "../Common"; +import {Link} from "react-router"; +import {useDockerized, useSyncRole, useWROLMode} from "../../hooks/customHooks"; +import { + Button, + Checkbox, + Header, + Icon, + IconButton, + Message, + Modal, + Panel, + Progress, + Select, + TextInput, + toast +} from "../ui"; + +const roleOptions = [ + {value: 'primary', label: 'Primary'}, + {value: 'secondary', label: 'Secondary'}, +]; + +/** The sync's progress: how many of the primary's FileGroups have been compared. */ +function syncPercent(sync) { + if (!sync || !sync.max_id) { + return 0; + } + // FileGroup ids are not contiguous; `groups_seen / max_id` underestimates, it never overshoots. + return Math.min(100, Math.round(100 * sync.groups_seen / sync.max_id)); +} + +function syncProgressLabel(sync) { + const downloaded = `${sync.files_downloaded} / ${sync.files_to_download} files`; + return `${sync.phase === 'configs' ? 'Importing configs' : downloaded} (${humanFileSize(sync.bytes_downloaded)})`; +} + +/** + * Shown on every page of a secondary WROLPi, so nobody mistakes it for the primary. WROL Mode stops syncing, + * so the warning goes away with it. + */ +export function SecondaryRoleMessage({content}) { + const role = useSyncRole(); + const wrolMode = useWROLMode(); + const {settings} = React.useContext(SettingsContext); + + if (role !== 'secondary' || wrolMode) { + return null; + } + return +} + +export function SecondaryRoleNotice({primaryUrl, content}) { + return + {content || <> + This WROLPi is a copy of {primaryUrl || 'its primary'}. Every sync replaces its library with the + primary's, so changes made here are lost. Do not use it as a primary. Enable WROL Mode to stop syncing. + {' '}Sync settings + } + +} + +/** The progress and outcome of a secondary's sync. */ +export function SyncStatus({sync, running}) { + if (!sync) { + return null; + } + return
+ {running && } +

+ Last successful sync: {sync.last_success_at ? isoDatetimeToAgoPopup(sync.last_success_at) : 'Never'} +

+ {!running && sync.finished_at &&

+ Last sync: {sync.files_downloaded} files downloaded + ({humanFileSize(sync.bytes_downloaded)}), {sync.files_failed} failed + {sync.files_excluded > 0 && <>, {sync.files_excluded} excluded} +

} + {sync.disk_full && + Some files were not synced because the drive is nearly full. + } + {sync.last_error && !running && + {sync.last_error} + } + {sync.last_skip_reason && !running &&

+ Not synced: {sync.last_skip_reason} +

} +
+} + +/** Covers the app while a secondary syncs; its API refuses requests until the sync finishes. */ +export function SyncingOverlay() { + const {status} = React.useContext(StatusContext); + const role = useSyncRole(); + const running = role === 'secondary' && status?.flags?.sync_running === true; + + if (!running) { + return null; + } + return
+ +
+} + +/** What the SyncingOverlay shows. */ +export function SyncingPanel({sync, onCancel}) { + return +
Syncing from the primary
+

+ This WROLPi is copying the library of {sync?.primary_url || 'its primary'}. It is unavailable + until the sync finishes. +

+ + +
+} + +/** The Settings panel: this WROLPi's role, and (for a secondary) its primary and sync window. */ +/** Is `path` an excluded folder, or inside one? */ +export function isExcludedPath(path, excluded) { + return (excluded || []).some(i => path === i || path.startsWith(`${i}/`)); +} + +/** Excluded folders without the ones already inside another excluded folder, sorted. */ +export function simplifyExcluded(excluded) { + const unique = Array.from(new Set(excluded || [])); + return unique.filter(i => !unique.some(j => j !== i && i.startsWith(`${j}/`))).sort(); +} + +function ExcludeFolderRow({folder, depth, source, chosen, onToggle}) { + const [open, setOpen] = React.useState(false); + const [children, setChildren] = React.useState(null); + const [error, setError] = React.useState(null); + + const checked = chosen.has(folder.path); + // A folder inside a chosen folder is excluded with it, whether or not it was chosen itself. + const inherited = isExcludedPath(folder.path, Array.from(chosen).filter(i => i !== folder.path)); + + const toggleOpen = async () => { + if (!open && children === null) { + try { + setChildren(await getSyncDirectories(folder.path, source)); + } catch (e) { + setError(e.message); + } + } + setOpen(!open); + }; + + return <> +
+ + onToggle(folder.path)} + label={ {folder.name}}/> + {inherited && excluded with its parent} +
+ {open && error &&

{error}

} + {open && children && children.length === 0 &&

No folders

} + {open && children && children.map(child => )} + +} + +/** Choose folders in a tree; `source` is 'local' (this WROLPi's media directory) or 'primary'. */ +export function ExcludeFolderModal({open, onClose, value, onSubmit, source}) { + const [chosen, setChosen] = React.useState(new Set()); + const [folders, setFolders] = React.useState(null); + const [error, setError] = React.useState(null); + + React.useEffect(() => { + if (!open) { + return; + } + setChosen(new Set(value || [])); + setFolders(null); + setError(null); + getSyncDirectories('', source) + .then(setFolders) + .catch(e => setError(e instanceof PrimaryUnreachableError + ? `The primary did not answer: ${e.message}` : e.message)); + }, [open, source, value]); + + const onToggle = path => { + const next = new Set(chosen); + next.has(path) ? next.delete(path) : next.add(path); + setChosen(next); + }; + + return + {source === 'primary' ? "Exclude the primary's folders" : 'Exclude folders'} + +

Checked folders, and everything inside them, are not copied.

+ {error && {error}} + {!error && folders === null &&

Loading folders...

} + {folders && folders.length === 0 &&

No folders

} + {folders && folders.map(folder => )} +
+ + + + +
+} + +/** The folders a sync or a backup does not copy, and the button that chooses them. */ +export function ExcludedDirectories({value, onChange, source, disabled}) { + const [open, setOpen] = React.useState(false); + const excluded = value || []; + return
+ + {excluded.length === 0 + ?

Nothing excluded; every folder is copied.

+ :
    + {excluded.map(path =>
  • + {path} + onChange(excluded.filter(i => i !== path))}/> +
  • )} +
} + +

+ Files already copied into an excluded folder are kept; delete them yourself to free space. +

+ setOpen(false)} value={excluded} source={source} + onSubmit={chosen => { + onChange(chosen); + setOpen(false); + }}/> +
+} + +export function SyncSection() { + const {settings, fetchSettings} = React.useContext(SettingsContext); + const {status} = React.useContext(StatusContext); + const wrolMode = useWROLMode(); + const [state, setState] = React.useState({}); + const [testResult, setTestResult] = React.useState(null); + const [saving, setSaving] = React.useState(false); + + // Only these values reset the form; the settings are re-fetched while the user types. + const loaded = Boolean(settings); + const role = settings?.role; + const primaryUrl = settings?.sync_primary_url; + const windowStart = settings?.sync_window_start; + const windowEnd = settings?.sync_window_end; + const excludedKey = JSON.stringify(settings?.sync_excluded_directories || []); + React.useEffect(() => { + if (loaded) { + setState({ + role: role || 'primary', + sync_primary_url: primaryUrl || '', + sync_window_start: windowStart || '', + sync_window_end: windowEnd || '', + sync_excluded_directories: JSON.parse(excludedKey), + }); + } + }, [loaded, role, primaryUrl, windowStart, windowEnd, excludedKey]); + + if (!settings) { + return null; + } + + const secondary = state.role === 'secondary'; + const saved = secondary && settings.role === 'secondary' && settings.sync_primary_url; + const running = status?.flags?.sync_running === true; + + const save = async () => { + setSaving(true); + try { + const response = await saveSyncSettings(state); + if (response?.ok) { + toast({type: 'success', title: 'Sync settings saved', time: 3000}); + setTestResult(null); + } + await fetchSettings(); + } finally { + setSaving(false); + } + }; + + const test = async () => { + setTestResult(await testSync()); + }; + + const syncNow = async () => { + const response = await runSync(); + if (response?.ok) { + toast({type: 'info', title: 'Sync requested', description: 'The sync starts in a few seconds.', time: 3000}); + } + }; + + return +
Sync
+

+ A secondary WROLPi keeps a copy of a primary WROLPi's library: its files, tags, + channels, domains, playlists, and other library settings. Files deleted on the primary are kept. +

+
+ + setDrive(value === 'none' ? '' : value)}/> +
+ {drive &&
+ + setFolder(e.currentTarget.value)}/> +

+ The media directory is copied into this folder; leave it empty to use the whole drive. +

+
} + {drive && <> +
+
+ + setWindowStart(e.currentTarget.value)}/> +
+
+ + setWindowEnd(e.currentTarget.value)}/> +
+
+

+ Within the window, the backup drive is updated every hour. Leave both empty to back up only with + "Back Up Now". +

+ + } +
+ + {savedDirectory && !running && } + {/* The database snapshot runs to the end; a half-written database is worse than waiting for it. */} + {running && } +
+ {savedDirectory && } +
+} + +/** The Sync tab of /admin. */ +export function SyncPage() { + useTitle('Sync'); + return
+ + + +
+} diff --git a/app/src/components/admin/Sync.test.js b/app/src/components/admin/Sync.test.js new file mode 100644 index 00000000..627e1c54 --- /dev/null +++ b/app/src/components/admin/Sync.test.js @@ -0,0 +1,343 @@ +import React from 'react'; +import {fireEvent, screen, waitFor} from '@testing-library/react'; +import {renderWithProviders} from '../../test-utils'; +import {backupStatusFixture, settingsFixture, statusFixture, syncStatusFixture} from '../../test-fixtures'; +import { + BackupDriveSection, backupDriveOptions, ExcludedDirectories, isExcludedPath, SecondaryRoleMessage, simplifyExcluded, + splitBackupDirectory, SyncingOverlay, SyncSection, +} from './Sync'; + +jest.mock('../../api/controller', () => ({ + getDisks: jest.fn(), +})); + +jest.mock('../../api', () => ({ + cancelBackup: jest.fn(), + runBackup: jest.fn(), + cancelSync: jest.fn(), + runSync: jest.fn(), + saveSyncSettings: jest.fn(), + testSync: jest.fn(), + getSyncDirectories: jest.fn(), + PrimaryUnreachableError: class PrimaryUnreachableError extends Error { + }, +})); + +import { + cancelBackup, cancelSync, getSyncDirectories, PrimaryUnreachableError, runBackup, runSync, saveSyncSettings, testSync, +} from '../../api'; +import {getDisks} from '../../api/controller'; + +beforeEach(() => { + // react-scripts resets mock implementations before each test. + cancelSync.mockResolvedValue({ok: true}); + runSync.mockResolvedValue({ok: true}); + saveSyncSettings.mockResolvedValue({ok: true}); + testSync.mockResolvedValue({ok: true, info: {version: '0.30.5-beta'}}); + runBackup.mockResolvedValue({ok: true}); + cancelBackup.mockResolvedValue({ok: true}); + getDisks.mockResolvedValue(disks); + getSyncDirectories.mockImplementation(async path => folderTree[path] || []); +}); + +const folderTree = { + '': [{name: 'Movies', path: 'Movies'}, {name: 'videos', path: 'videos'}], + 'videos': [{name: 'channel', path: 'videos/channel'}], + 'videos/channel': [], +}; + +const disks = [ + {name: 'sda1', size: '9.1T', mountpoint: '/media/wrolpi_backup', label: 'Decay2of2', uuid: 'uuid-backup'}, + {name: 'sdb1', size: '9.1T', mountpoint: '/media/wrolpi', label: 'Decay1of2', uuid: 'uuid-media'}, + {name: 'mmcblk0p2', size: '118.6G', mountpoint: '/', label: 'rootfs', uuid: 'uuid-root'}, + {name: 'sdc1', size: '64G', mountpoint: null, label: 'stick', uuid: 'uuid-stick'}, +]; + +const secondary = (overrides = {}) => ({ + settings: settingsFixture({role: 'secondary', sync_primary_url: 'https://192.168.0.10', ...overrides}), +}); + +describe('SecondaryRoleMessage', () => { + it('warns on a secondary', () => { + renderWithProviders(, {settings: secondary()}); + expect(screen.getByText('Secondary WROLPi')).toBeInTheDocument(); + expect(screen.getByText('https://192.168.0.10')).toBeInTheDocument(); + }); + + it('is absent on a primary', () => { + renderWithProviders(); + expect(screen.queryByText('Secondary WROLPi')).not.toBeInTheDocument(); + }); + + it('goes away in WROL Mode, which stops syncing', () => { + renderWithProviders(, {settings: secondary({wrol_mode: true})}); + expect(screen.queryByText('Secondary WROLPi')).not.toBeInTheDocument(); + }); +}); + +describe('SyncingOverlay', () => { + const syncing = (sync = {}) => ({ + status: statusFixture({ + flags: {sync_running: true}, + sync: syncStatusFixture({state: 'running', phase: 'files', groups_seen: 50, max_id: 100, ...sync}), + }), + }); + + it('covers the app while a secondary syncs', () => { + renderWithProviders(, {settings: secondary(), status: syncing()}); + expect(screen.getByRole('alertdialog', {name: 'Syncing'})).toBeInTheDocument(); + expect(screen.getByText('Syncing from the primary')).toBeInTheDocument(); + + fireEvent.click(screen.getByText('Stop Sync')); + expect(cancelSync).toHaveBeenCalled(); + }); + + it('is absent when not syncing', () => { + renderWithProviders(, {settings: secondary()}); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + }); + + it('is absent on a primary', () => { + renderWithProviders(, {status: syncing()}); + expect(screen.queryByRole('alertdialog')).not.toBeInTheDocument(); + }); +}); + +describe('SyncSection', () => { + it('a primary shows only its role', () => { + // Even with a backup drive: the backup has its own window, in its own panel. + renderWithProviders(, {settings: {settings: settingsFixture({ + backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup'})}}); + expect(screen.getByText('This WROLPi is')).toBeInTheDocument(); + expect(screen.queryByText('Primary address')).not.toBeInTheDocument(); + expect(screen.queryByText('Sync Window Start')).not.toBeInTheDocument(); + expect(screen.queryByText('Sync Now')).not.toBeInTheDocument(); + }); + + it('a secondary shows its primary, window, and sync controls', async () => { + const status = {status: statusFixture({sync: syncStatusFixture({last_error: 'The primary did not answer'})})}; + renderWithProviders(, {settings: secondary({sync_window_start: '01:00'}), status}); + expect(screen.getByDisplayValue('https://192.168.0.10')).toBeInTheDocument(); + expect(screen.getByDisplayValue('01:00')).toBeInTheDocument(); + expect(screen.getByText('The primary did not answer')).toBeInTheDocument(); + + fireEvent.click(screen.getByText('Sync Now')); + await waitFor(() => expect(runSync).toHaveBeenCalled()); + + fireEvent.click(screen.getByText('Save')); + await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({ + role: 'secondary', sync_primary_url: 'https://192.168.0.10', sync_window_start: '01:00', + sync_window_end: '', sync_excluded_directories: [], + })); + }); + + it('WROL Mode stops syncing', () => { + renderWithProviders(, {settings: secondary({wrol_mode: true})}); + expect(screen.getByText('Syncing is stopped')).toBeInTheDocument(); + expect(screen.getByText('Sync Now').closest('button')).toBeDisabled(); + }); +}); + +describe('backupDriveOptions', () => { + it('lists mounted drives under /media that are not the media drive', () => { + expect(backupDriveOptions(disks, '/media/wrolpi')).toEqual([ + {value: '/media/wrolpi_backup', uuid: 'uuid-backup', label: 'Decay2of2 (9.1T) at /media/wrolpi_backup'}, + ]); + }); +}); + +describe('BackupDriveSection', () => { + it('saves the chosen drive with its UUID', async () => { + renderWithProviders(); + await waitFor(() => expect(getDisks).toHaveBeenCalled()); + fireEvent.click(screen.getByDisplayValue('None')); + fireEvent.click(await screen.findByText('Decay2of2 (9.1T) at /media/wrolpi_backup')); + fireEvent.click(screen.getByText('Save')); + await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({ + backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup', + backup_window_start: '', backup_window_end: '', backup_excluded_directories: [], + })); + }); + + it('shows the backup and runs it on request', async () => { + const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup', + backup_device_uuid: 'uuid-backup'})}; + const status = {status: statusFixture({backup: backupStatusFixture({last_error: 'rsync failed (12)'})})}; + renderWithProviders(, {settings, status}); + expect(await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup')).toBeInTheDocument(); + expect(screen.getByText('rsync failed (12)')).toBeInTheDocument(); + fireEvent.click(screen.getByText('Back Up Now')); + await waitFor(() => expect(runBackup).toHaveBeenCalled()); + }); + + it('keeps an unplugged drive selected', async () => { + getDisks.mockResolvedValue([]); + const settings = {settings: settingsFixture({backup_directory: '/media/gone', backup_device_uuid: 'x'})}; + renderWithProviders(, {settings}); + expect(await screen.findByDisplayValue('/media/gone (not mounted)')).toBeInTheDocument(); + }); + + it('offers to stop a running backup', async () => { + const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup', + backup_device_uuid: 'uuid-backup'})}; + const status = {status: statusFixture({flags: {backup_running: true}, + backup: backupStatusFixture({state: 'running', phase: 'files', percent: 40})})}; + renderWithProviders(, {settings, status}); + fireEvent.click(await screen.findByText('Stop Backup')); + expect(cancelBackup).toHaveBeenCalled(); + }); + + it('is absent in Docker, which cannot reach other drives', () => { + renderWithProviders(, {status: {status: statusFixture({dockerized: true})}}); + expect(screen.queryByText('Backup Drive')).not.toBeInTheDocument(); + expect(getDisks).not.toHaveBeenCalled(); + }); +}); + +describe('splitBackupDirectory', () => { + const options = [ + {value: '/media/backup', uuid: 'uuid-backup'}, + {value: '/media/backup/nested', uuid: 'uuid-nested'}, + ]; + + it('finds the drive and the folder on it', () => { + expect(splitBackupDirectory('/media/backup', 'uuid-backup', options)) + .toEqual({drive: '/media/backup', folder: ''}); + expect(splitBackupDirectory('/media/backup/wrolpi/library', 'uuid-backup', options)) + .toEqual({drive: '/media/backup', folder: 'wrolpi/library'}); + }); + + it('prefers the drive with the saved UUID', () => { + expect(splitBackupDirectory('/media/backup/nested/wrolpi', 'uuid-nested', options)) + .toEqual({drive: '/media/backup/nested', folder: 'wrolpi'}); + expect(splitBackupDirectory('/media/backup/nested/wrolpi', 'uuid-backup', options)) + .toEqual({drive: '/media/backup', folder: 'nested/wrolpi'}); + }); + + it('does not mistake a similarly named drive', () => { + expect(splitBackupDirectory('/media/backup2/wrolpi', 'x', options)).toEqual({drive: '/media/backup2/wrolpi', folder: ''}); + }); + + it('is empty without a backup', () => { + expect(splitBackupDirectory('', '', options)).toEqual({drive: '', folder: ''}); + }); +}); + +describe('BackupDriveSection folder', () => { + it('saves a folder on the chosen drive', async () => { + renderWithProviders(); + await waitFor(() => expect(getDisks).toHaveBeenCalled()); + fireEvent.click(screen.getByDisplayValue('None')); + fireEvent.click(await screen.findByText('Decay2of2 (9.1T) at /media/wrolpi_backup')); + fireEvent.change(screen.getByPlaceholderText('The whole drive'), {target: {value: '/wrolpi/library/'}}); + fireEvent.click(screen.getByText('Save')); + await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({ + backup_directory: '/media/wrolpi_backup/wrolpi/library', backup_device_uuid: 'uuid-backup', + backup_window_start: '', backup_window_end: '', backup_excluded_directories: [], + })); + }); + + it('shows the saved folder', async () => { + const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup/wrolpi', + backup_device_uuid: 'uuid-backup'})}; + renderWithProviders(, {settings}); + expect(await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup')).toBeInTheDocument(); + expect(screen.getByPlaceholderText('The whole drive')).toHaveValue('wrolpi'); + }); +}); + +describe('BackupDriveSection database phase', () => { + it('cannot be stopped while the database is copied', async () => { + const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup', + backup_device_uuid: 'uuid-backup'})}; + const status = {status: statusFixture({flags: {backup_running: true}, + backup: backupStatusFixture({state: 'running', phase: 'database'})})}; + renderWithProviders(, {settings, status}); + expect((await screen.findByText('Stop Backup')).closest('button')).toBeDisabled(); + }); +}); + +describe('BackupDriveSection window', () => { + it('saves its own window with the drive', async () => { + const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup', + backup_device_uuid: 'uuid-backup', backup_window_start: '01:00', backup_window_end: '05:00'})}; + renderWithProviders(, {settings}); + await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup'); + expect(screen.getByLabelText('Backup Window Start')).toHaveValue('01:00'); + fireEvent.change(screen.getByLabelText('Backup Window End'), {target: {value: '04:30'}}); + fireEvent.click(screen.getByText('Save')); + await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith({ + backup_directory: '/media/wrolpi_backup', backup_device_uuid: 'uuid-backup', + backup_window_start: '01:00', backup_window_end: '04:30', backup_excluded_directories: [], + })); + }); +}); + + +describe('excluded folders', () => { + it('isExcludedPath matches a folder and what is inside it, not a similar name', () => { + expect(isExcludedPath('Movies', ['Movies'])).toBe(true); + expect(isExcludedPath('Movies/a', ['Movies'])).toBe(true); + expect(isExcludedPath('Movies2', ['Movies'])).toBe(false); + }); + + it('simplifyExcluded drops folders inside another', () => { + expect(simplifyExcluded(['videos/channel', 'Movies', 'videos', 'Movies'])).toEqual(['Movies', 'videos']); + }); + + it('lists the excluded folders and removes one', () => { + const onChange = jest.fn(); + renderWithProviders(); + expect(screen.getByText('Movies')).toBeInTheDocument(); + fireEvent.click(screen.getByLabelText('Copy Movies again')); + expect(onChange).toHaveBeenCalledWith(['videos/channel']); + }); + + it('chooses folders in the tree; a folder inside a checked folder is excluded with it', async () => { + const onChange = jest.fn(); + renderWithProviders(); + expect(screen.getByText('Nothing excluded; every folder is copied.')).toBeInTheDocument(); + fireEvent.click(screen.getByText('Choose Folders')); + await screen.findByLabelText('videos'); + expect(getSyncDirectories).toHaveBeenCalledWith('', 'primary'); + + fireEvent.click(screen.getByLabelText('Open videos')); + const child = await screen.findByLabelText('videos/channel'); + expect(getSyncDirectories).toHaveBeenCalledWith('videos', 'primary'); + fireEvent.click(child); + fireEvent.click(screen.getByLabelText('videos')); + await waitFor(() => expect(screen.getByLabelText('videos/channel')).toBeDisabled()); + expect(screen.getByLabelText('videos/channel')).toBeChecked(); + expect(screen.getByText('excluded with its parent')).toBeInTheDocument(); + + fireEvent.click(screen.getByText('Done')); + expect(onChange).toHaveBeenCalledWith(['videos']); + }); + + it('says when the primary does not answer', async () => { + getSyncDirectories.mockRejectedValue(new PrimaryUnreachableError('connection refused')); + renderWithProviders(); + fireEvent.click(screen.getByText('Choose Folders')); + expect(await screen.findByText('The primary did not answer: connection refused')).toBeInTheDocument(); + }); + + it('a secondary saves its excluded folders', async () => { + renderWithProviders(, {settings: secondary({sync_excluded_directories: ['Movies']})}); + expect(screen.getByText('Movies')).toBeInTheDocument(); + fireEvent.click(screen.getByText('Save')); + await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith( + expect.objectContaining({sync_excluded_directories: ['Movies']}))); + }); + + it('the backup drive saves its own excluded folders', async () => { + const settings = {settings: settingsFixture({backup_directory: '/media/wrolpi_backup', + backup_device_uuid: 'uuid-backup', backup_excluded_directories: ['Movies']})}; + renderWithProviders(, {settings}); + await screen.findByDisplayValue('Decay2of2 (9.1T) at /media/wrolpi_backup'); + fireEvent.click(screen.getByLabelText('Copy Movies again')); + fireEvent.click(screen.getByText('Save')); + await waitFor(() => expect(saveSyncSettings).toHaveBeenCalledWith( + expect.objectContaining({backup_excluded_directories: []}))); + }); +}); diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index a5beeeb5..612d3e13 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -2119,6 +2119,12 @@ export const useWROLMode = () => { return settings ? settings.wrol_mode : null; } +export const useSyncRole = () => { + // This WROLPi's role ('primary' or 'secondary', see Sync); during fetch this returns null. + const {settings} = useContext(SettingsContext); + return settings ? settings.role || 'primary' : null; +} + export const useDockerized = () => { const {status} = React.useContext(StatusContext); return status && status['dockerized'] === true; diff --git a/app/src/test-fixtures.js b/app/src/test-fixtures.js index 7746d375..d988878d 100644 --- a/app/src/test-fixtures.js +++ b/app/src/test-fixtures.js @@ -45,6 +45,11 @@ import {defaultTheme} from './themes/names'; /** GET /api/settings */ export const settingsFixture = (overrides = {}) => ({ archive_destination: 'archive/%(domain_tag)s/%(domain)s', + backup_device_uuid: null, + backup_directory: null, + backup_excluded_directories: [], + backup_window_end: null, + backup_window_start: null, check_for_upgrades: true, download_daily_limit_global: null, download_daily_limit_per_domain: null, @@ -74,7 +79,12 @@ export const settingsFixture = (overrides = {}) => ({ repos_destination: 'repos/%(repo_tag)s/%(repo_name)s', require_cookies_unlocked: false, require_media_mounted: false, + role: 'primary', save_ffprobe_json: true, + sync_excluded_directories: [], + sync_primary_url: null, + sync_window_end: null, + sync_window_start: null, tags_directory: true, throttle_on_startup: false, timezone: null, @@ -85,6 +95,52 @@ export const settingsFixture = (overrides = {}) => ({ ...overrides, }); +/** The `sync` of GET /api/status: a secondary that has never synced. */ +export const syncStatusFixture = (overrides = {}) => ({ + state: 'idle', + phase: null, + requested: false, + force: false, + cancel_requested: false, + started_at: null, + finished_at: null, + last_success_at: null, + last_error: null, + last_skip_reason: null, + primary_url: null, + primary_version: null, + groups_seen: 0, + max_id: 0, + files_checked: 0, + files_to_download: 0, + files_downloaded: 0, + bytes_downloaded: 0, + files_failed: 0, + files_excluded: 0, + configs_imported: [], + disk_full: false, + ...overrides, +}); + +/** The `backup` of GET /api/status: a backup drive that has never been written. */ +export const backupStatusFixture = (overrides = {}) => ({ + state: 'idle', + phase: null, + requested: false, + cancel_requested: false, + started_at: null, + finished_at: null, + last_success_at: null, + last_error: null, + last_skip_reason: null, + directory: null, + bytes_transferred: 0, + percent: 0, + files_transferred: 0, + files_to_check: 0, + ...overrides, +}); + /** * GET /api/status * @@ -122,6 +178,10 @@ export const statusFixture = (overrides = {}) => ({ memory_stats: {total: 4000000000, used: 1000000000, free: 3000000000, cached: 500000000}, processes_stats: [], sanic_workers: {}, + // A secondary's sync (wrolpi.sync); idle on a primary. + sync: syncStatusFixture(), + // A backup to a second drive (wrolpi.sync.backup); idle without one. + backup: backupStatusFixture(), version: '1.0.0', wrol_mode: false, // Upgrade info, seeded by the API and refreshed by its upgrade check. diff --git a/docker/web/Caddyfile b/docker/web/Caddyfile index 1d745032..368d518c 100644 --- a/docker/web/Caddyfile +++ b/docker/web/Caddyfile @@ -8,6 +8,12 @@ :443 { tls /etc/ssl/caddy/cert.crt /etc/ssl/caddy/cert.key + # Access log: `docker compose logs -f web`. + log { + output stdout + format console + } + header Access-Control-Allow-Origin "*" header Access-Control-Allow-Methods "GET, POST, PUT, DELETE, OPTIONS" header Access-Control-Allow-Headers "Content-Type, Accept" diff --git a/main.py b/main.py index 91748324..1778af70 100755 --- a/main.py +++ b/main.py @@ -14,7 +14,7 @@ from wrolpi import root_api # noqa from wrolpi.api_utils import api_app, perpetual_signal, per_worker_task from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \ cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, have_internet_connection, wrol_mode_enabled, \ - create_empty_config_files, import_file_configs, TRACE_LEVEL + create_empty_config_files, import_file_configs, TRACE_LEVEL, is_secondary from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts from wrolpi.dates import Seconds from modules.videos.cookies import lock_cookies @@ -286,7 +286,8 @@ async def start_single_tasks(app: Sanic): # Hotspot/throttle on startup are now handled by the Controller service. - if get_wrolpi_config().download_on_startup and not wrol_mode_enabled(): + # A secondary never downloads; its library comes from its primary (see wrolpi.sync). + if get_wrolpi_config().download_on_startup and not wrol_mode_enabled() and not is_secondary(): # Only start downloading when prerequisites have been met. try: async with flags.have_internet.wait_for(timeout=30): diff --git a/scripts/vulture_baseline.txt b/scripts/vulture_baseline.txt index b8fd5403..ba2fd89f 100644 --- a/scripts/vulture_baseline.txt +++ b/scripts/vulture_baseline.txt @@ -39,6 +39,7 @@ wrolpi/media_path.py: unused method 'process_result_value' wrolpi/media_path.py: unused variable 'PATH_TYPE' wrolpi/media_path.py: unused variable 'dialect' wrolpi/media_path.py: unused variable 'dialect' +wrolpi/sync/conftest.py: unused method 'log_message' wrolpi/tags.py: unused function 'get_name_by_id_key' wrolpi/test/common.py: unused class 'PytestCase' wrolpi/test/common.py: unused method 'assertDictContains' diff --git a/wrolpi/common.py b/wrolpi/common.py index 45bc7db8..321b5902 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -490,6 +490,24 @@ def library_on_internal_disk(media_directory: Path = None) -> bool: DB_CONFIG_FILE_NAMES ={'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'playlists.yaml', 'repos.yaml'} +# The library configs a secondary WROLPi copies from its primary (see wrolpi.sync). A secondary never saves these; +# the primary's copy is the only one that changes them. repos.yaml is absent because git clones are never synced. +SYNCED_CONFIG_FILE_NAMES = frozenset({ + 'archives_downloader.yaml', + 'bookmarks.yaml', + 'channels.yaml', + 'domains.yaml', + 'download_manager.yaml', + 'inventory/catalog.yaml', + 'map_pins.yaml', + 'playlists.yaml', + 'tags.yaml', + 'videos_downloader.yaml', + 'zims.yaml', +}) +# MultiFileConfig names (directories of configs) a secondary copies from its primary. +SYNCED_MULTIFILE_CONFIG_NAMES = frozenset({'inventory'}) + class _DatabaseChanged(Exception): """The database changed a config's rows between an import's snapshot and its write.""" @@ -733,6 +751,11 @@ class ConfigFile: from wrolpi.api_utils import api_app from wrolpi.events import Events + if self.file_name in SYNCED_CONFIG_FILE_NAMES and is_secondary(): + # The primary's copy of this config is the only one that changes; the next sync replaces this one. + logger_.debug(f'Not saving {self.file_name} on a secondary WROLPi') + return + file = file or self.get_file() # Compare basenames so a nested file_name (e.g. 'inventory/catalog.yaml') is accepted. if pathlib.Path(self.file_name).name != file.name: @@ -1004,6 +1027,8 @@ class ConfigFile: from wrolpi.db import get_db_session with get_db_session() as session: db_version = get_config_version(session, self.file_name) + # A secondary's synced configs are the primary's; its database never outranks them (see wrolpi.sync). + forced = forced or (self.file_name in SYNCED_CONFIG_FILE_NAMES and is_secondary()) decision = IMPORT if forced else decide(file_version, db_version) logger_.debug(f'{self.file_name}: file version {file_version}, database version {db_version}: {decision}') return decision, db_version @@ -1281,6 +1306,10 @@ class MultiFileConfig: """Write one entity's in-memory config to its YAML file (version bump + daily backup). The caller MUST already hold `save_lock()`.""" + if self.name in SYNCED_MULTIFILE_CONFIG_NAMES and is_secondary(): + # The primary's copy is the only one that changes; the next sync replaces this one. + logger_.debug(f'Not saving {self.name}/{slug} on a secondary WROLPi') + return file = self.get_file(slug) if PYTEST and not is_tempfile(file): raise ValueError(f'Refusing to save config file while testing: {file}') @@ -1490,6 +1519,10 @@ def get_all_configs() -> Dict[str, ConfigFile]: if bookmarks_config := get_bookmarks_config(): all_configs[bookmarks_config.file_name] = bookmarks_config + from wrolpi.sync.config import get_sync_config + if sync_config := get_sync_config(): + all_configs[sync_config.file_name] = sync_config + from wrolpi.collections.config import get_playlists_config if playlists_config := get_playlists_config(): all_configs[playlists_config.file_name] = playlists_config @@ -1527,6 +1560,14 @@ async def import_file_configs(): except Exception as e: logger.warning(f'Failed to import wrolpi config: {e}') + # Whether this WROLPi is a secondary decides how the other configs import (see `ConfigFile.version_decision`). + from wrolpi.sync.config import get_sync_config + try: + await asyncio.to_thread(get_sync_config().import_config) + logger.debug('sync config imported') + except Exception as e: + logger.warning(f'Failed to import sync config: {e}') + if not wrolpi_config.successful_import: return if wrolpi_config.wrol_mode: @@ -2095,6 +2136,13 @@ def wrol_mode_enabled() -> bool: return get_wrolpi_config().wrol_mode +def is_secondary() -> bool: + """Return True if this WROLPi copies its library from a primary WROLPi (see wrolpi.sync).""" + # wrolpi.sync.config imports this module. + from wrolpi.sync.config import get_sync_config + return get_sync_config().role == 'secondary' + + def wrol_mode_check(func): """Wraps a function so that it cannot be called when WROL Mode is enabled.""" @@ -2120,6 +2168,11 @@ def enable_wrol_mode(): get_wrolpi_config().wrol_mode = True from wrolpi.downloader import download_manager download_manager.stop() + # A secondary in WROL Mode is in use; a running sync stops between pages. A backup stops too. + from wrolpi.sync.lib import request_sync_cancel + from wrolpi.sync.backup import request_backup_cancel + request_sync_cancel() + request_backup_cancel() async def disable_wrol_mode(): diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index be9eba13..f569f6e4 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -52,6 +52,7 @@ from wrolpi.errors import UnrecoverableDownloadError from wrolpi.files.models import Directory, FileGroup from wrolpi.files.worker import file_worker from wrolpi.switches import await_switches as await_switches_ +from wrolpi.sync.config import get_sync_config from wrolpi.tags import Tag, upsert_tag from wrolpi.vars import PROJECT_DIR @@ -236,6 +237,15 @@ def test_directory() -> Generator[Path, Any, None]: yield tmp_path +@pytest.fixture(autouse=True) +def test_sync_config(): + """Every test starts as a primary with no sync settings (sync.yaml is in memory unless a test saves it).""" + config = get_sync_config() + config._config.update(copy.deepcopy(config.default_config)) + yield + config._config.update(copy.deepcopy(config.default_config)) + + @pytest.fixture def test_wrolpi_config(test_directory) -> Generator[Path, Any, None]: """ diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index 9fb35bf8..75715cdb 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -5,6 +5,7 @@ import queue from sanic import Sanic from wrolpi.common import logger +from wrolpi.sync.status import default_sync_status, default_backup_status from wrolpi.vars import LOG_LEVEL_INT logger = logger.getChild(__name__) @@ -41,6 +42,7 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.map_pins_config = manager.dict() app.shared_ctx.flasher_config = manager.dict() app.shared_ctx.bookmarks_config = manager.dict() + app.shared_ctx.sync_config = manager.dict() app.shared_ctx.recently_viewed_config = manager.dict() # Shared dicts. app.shared_ctx.uploaded_files = manager.dict() @@ -65,6 +67,9 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.download_manager_stopped.set() app.shared_ctx.single_tasks_started = multiprocessing.Event() + # Set while a secondary syncs (wrolpi.sync); the API refuses requests. An Event so the request middleware can + # check it without a manager round trip. + app.shared_ctx.sync_api_locked = multiprocessing.Event() app.shared_ctx.flags_initialized = multiprocessing.Event() # Transcode @@ -152,6 +157,8 @@ def reset_shared_contexts(app: Sanic): current_commit=None, commits_behind=0, git_branch=None, + sync=default_sync_status(), + backup=default_backup_status(), )) app.shared_ctx.cache.clear() # Secure cookies @@ -240,6 +247,7 @@ def reset_shared_contexts(app: Sanic): # Clear events_history in place to preserve references held by fixtures del app.shared_ctx.events_history[:] app.shared_ctx.single_tasks_started.clear() + app.shared_ctx.sync_api_locked.clear() app.shared_ctx.flags_initialized.clear() if hasattr(app.shared_ctx, 'flag_holders'): app.shared_ctx.flag_holders.clear() @@ -351,6 +359,12 @@ def initialize_configs_contexts(app: Sanic): except Exception as e: logger.error(f'Failed to initialize in-memory bookmarks config: {e}') + try: + from wrolpi.sync.config import SYNC_CONFIG + SYNC_CONFIG.initialize(app.shared_ctx.sync_config) + except Exception as e: + logger.error(f'Failed to initialize in-memory sync config: {e}') + try: from wrolpi.files.progress import RECENTLY_VIEWED_CONFIG RECENTLY_VIEWED_CONFIG.initialize(app.shared_ctx.recently_viewed_config) diff --git a/wrolpi/dates.py b/wrolpi/dates.py index 539ecab8..713373a3 100644 --- a/wrolpi/dates.py +++ b/wrolpi/dates.py @@ -1,5 +1,5 @@ import logging -from datetime import datetime, timezone +from datetime import datetime, timezone, time from enum import Enum from typing import Union, List @@ -216,3 +216,13 @@ def date_range_to_where(wheres: list, params: dict, from_year: int, to_year: int params['to_year'] = to_year return wheres, params + + +def is_within_daily_window(start: str | None, end: str | None, current: time) -> bool: + """True if `current` is within the daily window from `start` to `end` ('HH:MM'). A window whose end is before + its start crosses midnight (e.g. 22:00-06:00). Callers decide what a missing window means.""" + start_time = datetime.strptime(start, '%H:%M').time() + end_time = datetime.strptime(end, '%H:%M').time() + if start_time <= end_time: + return start_time <= current < end_time + return current >= start_time or current < end_time diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 13b65aca..573c9769 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -33,10 +33,10 @@ from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.cmd import which, run_command, CommandResult from wrolpi.common import loaded, Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, \ wrol_mode_enabled, background_task, get_absolute_media_path, timer, aiohttp_get, \ - get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain + get_download_info, trim_file_name, get_wrolpi_config, TRACE_LEVEL, normalize_domain, is_secondary from wrolpi.config_import import ConfigSection, Unreadable, chunks from wrolpi.config_version import collection_kinds, count_config_change, register_ownership -from wrolpi.dates import TZDateTime, now, Seconds +from wrolpi.dates import TZDateTime, now, Seconds, is_within_daily_window from wrolpi.db import get_db_session, session_curs, serializer from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, BotBlockedDownloadError, UnknownDownload, \ ValidationError, DownloadError @@ -817,14 +817,8 @@ class DownloadManager: return True current = self._get_local_time() - start_time = datetime.strptime(start, "%H:%M").time() - end_time = datetime.strptime(end, "%H:%M").time() - - if start_time <= end_time: - within = start_time <= current < end_time - else: - # Overnight window (e.g., 22:00-06:00) - within = current >= start_time or current < end_time + # Overnight windows (e.g., 22:00-06:00) are supported. + within = is_within_daily_window(start, end, current) # Diagnostic: log only when the window opens/closes, including the computed local time and # timezone, so a clock/timezone mismatch (e.g. system UTC-5 vs configured America/Denver) @@ -845,6 +839,9 @@ class DownloadManager: Single source of truth for `can_download`; returning a reason string (rather than a bare bool) lets the dispatcher log *why* a cycle was skipped.""" + if is_secondary(): + # A secondary's library comes from its primary (see wrolpi.sync). + return 'secondary role' if PYTEST: return None if self.is_disabled: diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 3f2a0062..c2852ad6 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -45,6 +45,18 @@ class WROLModeEnabled(APIError): status_code = HTTPStatus.FORBIDDEN +class SyncInProgress(APIError): + code = 'SYNCING' + summary = 'This WROLPi is copying its library from its primary. Try again when the sync finishes.' + status_code = HTTPStatus.SERVICE_UNAVAILABLE + + +class SecondaryRole(APIError): + code = 'SECONDARY_ROLE' + summary = 'This method is disabled on a secondary WROLPi.' + status_code = HTTPStatus.FORBIDDEN + + class InvalidDownload(APIError): code = 'INVALID_DOWNLOAD' summary = 'The URL cannot be downloaded.' diff --git a/wrolpi/events.py b/wrolpi/events.py index 5a88bcd5..d4e52d29 100644 --- a/wrolpi/events.py +++ b/wrolpi/events.py @@ -40,6 +40,22 @@ class Events: def send_downloads_disabled(message: str = None): send_event('downloads_disabled', message, subject='downloads') + @staticmethod + def send_backup_completed(message: str): + send_event('backup_completed', message, subject='backup') + + @staticmethod + def send_backup_failed(message: str): + send_event('backup_failed', message, subject='backup') + + @staticmethod + def send_sync_completed(message: str): + send_event('sync_completed', message, subject='sync') + + @staticmethod + def send_sync_failed(message: str): + send_event('sync_failed', message, subject='sync') + @staticmethod def send_user_notify(message: str, url: str = None): send_event('user_notify_message', message, subject='user_notify', url=url) diff --git a/wrolpi/flags.py b/wrolpi/flags.py index 139970a7..88f3c2a5 100644 --- a/wrolpi/flags.py +++ b/wrolpi/flags.py @@ -161,6 +161,11 @@ map_search_building = Flag('map_search_building') # The global refresh has been performed. This is False on a fresh instance of WROLPi. refresh_complete = Flag('refresh_complete', store_db=True) +# A secondary WROLPi is copying its library from its primary (see wrolpi.sync). Its API refuses requests meanwhile. +sync_running = Flag('sync_running') +# This WROLPi is copying its media directory to its backup drive (see wrolpi.sync.backup). +backup_running = Flag('backup_running') + # Cookies flags for yt-dlp. cookies_exist = Flag('cookies_exist') cookies_unlocked = Flag('cookies_unlocked') @@ -171,6 +176,7 @@ def get_flags() -> dict: flags = dict( cookies_exist=cookies_exist.is_set(), cookies_unlocked=cookies_unlocked.is_set(), + backup_running=backup_running.is_set(), db_up=db_up.is_set(), file_worker_busy=file_worker_busy.is_set(), file_worker_cleanup=file_worker_cleanup.is_set(), @@ -184,6 +190,7 @@ def get_flags() -> dict: media_mounted=media_mounted.is_set(), outdated_zims=outdated_zims.is_set(), refresh_complete=refresh_complete.is_set(), + sync_running=sync_running.is_set(), ) return flags diff --git a/wrolpi/perpetual.py b/wrolpi/perpetual.py index afa76abb..80c40d17 100644 --- a/wrolpi/perpetual.py +++ b/wrolpi/perpetual.py @@ -55,7 +55,8 @@ SHARED_LOCK_HEAL_TIMEOUT = 10 # when the recorded holder pid is dead, because a live worker may be inside the block right now. # file_worker_busy is deliberately absent: the collection-move path in an API worker holds it as a mutex, and # process_queue already drops a stale set on its next idle tick. -PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active') +PERPETUAL_ONLY_FLAGS = ('file_worker_counting', 'file_worker_discovery', 'global_refresh_active', 'sync_running', + 'backup_running') SHARED_PROGRESS_FLAGS = ('file_worker_modeling', 'file_worker_indexing', 'file_worker_cleanup', 'map_search_building') diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index a9ccb412..c778e8f5 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -28,18 +28,20 @@ from wrolpi.bookmarks_api import bookmarks_bp from wrolpi.collections.api import collection_bp from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \ - set_global_log_level, get_relative_to_media_directory, search_other_estimates, set_system_timezone + set_global_log_level, get_relative_to_media_directory, search_other_estimates, set_system_timezone, is_secondary from wrolpi.config_api import config_bp from wrolpi.db_query import db_query_bp from wrolpi.dates import now from wrolpi.downloader import download_manager -from wrolpi.errors import WROLModeEnabled, InvalidConfig, ValidationError +from wrolpi.errors import WROLModeEnabled, InvalidConfig, ValidationError, SecondaryRole from wrolpi.events import get_events, Events from wrolpi.files import files_bp from wrolpi.files.lib import get_file_statistics, search_file_suggestion_count from wrolpi.jobs import jobs_bp from wrolpi.log_levels import int_to_name from wrolpi.speedtest import speedtest_bp +from wrolpi.sync.api import sync_bp +from wrolpi.sync.config import get_sync_config from wrolpi.tags import Tag from wrolpi.vars import DOCKERIZED, IS_RPI, IS_RPI4, IS_RPI5, API_HOST, API_PORT, API_WORKERS, API_DEBUG, \ API_ACCESS_LOG, truthy_arg, API_AUTO_RELOAD @@ -69,6 +71,7 @@ api_app.blueprint(jobs_bp) api_app.blueprint(map_bp) api_app.blueprint(repos_bp) api_app.blueprint(speedtest_bp) +api_app.blueprint(sync_bp) api_app.blueprint(videos_bp) api_app.blueprint(zim_bp) @@ -164,6 +167,7 @@ async def echo(request: Request): ) def get_settings(_: Request): wrolpi_config = get_wrolpi_config() + sync_config = get_sync_config() # Drop special directories (e.g. zims) that must never be ignored, if present from older configs. from wrolpi.files.lib import sanitize_ignored_directories @@ -173,6 +177,11 @@ def get_settings(_: Request): settings = { 'archive_destination': wrolpi_config.archive_destination, + 'backup_device_uuid': sync_config.backup_device_uuid, + 'backup_directory': sync_config.backup_directory, + 'backup_excluded_directories': sync_config.backup_excluded_directories, + 'backup_window_end': sync_config.backup_window_end, + 'backup_window_start': sync_config.backup_window_start, 'download_manager_disabled': download_manager.is_disabled, 'download_manager_stopped': download_manager.is_stopped, 'download_daily_limit_global': wrolpi_config.download_daily_limit_global, @@ -205,7 +214,12 @@ def get_settings(_: Request): 'zims_destination': wrolpi_config.zims_destination, 'require_cookies_unlocked': wrolpi_config.require_cookies_unlocked, 'require_media_mounted': wrolpi_config.require_media_mounted, + 'role': sync_config.role, 'save_ffprobe_json': wrolpi_config.save_ffprobe_json, + 'sync_primary_url': sync_config.primary_url, + 'sync_excluded_directories': sync_config.sync_excluded_directories, + 'sync_window_end': sync_config.window_end, + 'sync_window_start': sync_config.window_start, 'timezone': wrolpi_config.timezone, } return json_response(settings) @@ -494,6 +508,8 @@ async def kill_downloads(_: Request): @api_bp.post('/download/enable') @openapi.description('Enable and start downloading.') async def enable_downloads(_: Request): + if is_secondary(): + raise SecondaryRole('Downloads are disabled on a secondary WROLPi; its primary downloads.') await download_manager.enable() return response.empty() diff --git a/wrolpi/schema.py b/wrolpi/schema.py index a8019cb0..03b74d0f 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -9,6 +9,11 @@ from wrolpi.errors import InvalidDownload, ValidationError @dataclass class SettingsResponse: + backup_device_uuid: Optional[str] + backup_directory: Optional[str] + backup_excluded_directories: List[str] + backup_window_end: Optional[str] + backup_window_start: Optional[str] download_manager_disabled: bool download_manager_stopped: bool download_daily_limit_global: Optional[int] @@ -38,6 +43,11 @@ class SettingsResponse: nav_color: str media_directory: str require_cookies_unlocked: bool + role: str + sync_primary_url: Optional[str] + sync_excluded_directories: List[str] + sync_window_end: Optional[str] + sync_window_start: Optional[str] require_media_mounted: bool save_ffprobe_json: bool tags_directory: bool @@ -116,6 +126,7 @@ class SettingsRequest: @dataclass class FlagsStatusResponse: + backup_running: bool db_up: bool file_worker_busy: bool file_worker_cleanup: bool @@ -128,6 +139,7 @@ class FlagsStatusResponse: media_mounted: bool outdated_zims: bool refresh_complete: bool + sync_running: bool @dataclass @@ -187,6 +199,52 @@ class DownloadsSummaryResponse: daily_limit_reached: bool +@dataclass +class SyncStatusResponse: + """The progress of a secondary WROLPi copying its library from its primary (see wrolpi.sync).""" + state: str # 'idle', 'running', or 'error' + phase: Optional[str] # 'files', 'configs', or None + requested: bool + force: bool + cancel_requested: bool + started_at: Optional[str] + finished_at: Optional[str] + last_success_at: Optional[str] + last_error: Optional[str] + last_skip_reason: Optional[str] + primary_url: Optional[str] + primary_version: Optional[str] + groups_seen: int + max_id: int + files_checked: int + files_to_download: int + files_downloaded: int + bytes_downloaded: int + files_failed: int + files_excluded: int + configs_imported: List[str] + disk_full: bool + + +@dataclass +class BackupStatusResponse: + """The progress of copying this WROLPi's media directory to its backup drive (see wrolpi.sync.backup).""" + state: str # 'idle', 'running', or 'error' + phase: Optional[str] # 'files', 'database', or None + requested: bool + cancel_requested: bool + started_at: Optional[str] + finished_at: Optional[str] + last_success_at: Optional[str] + last_error: Optional[str] + last_skip_reason: Optional[str] + directory: Optional[str] + bytes_transferred: int + percent: int + files_transferred: int + files_to_check: int + + @dataclass class StatusResponse: """The response of GET /api/status. @@ -201,6 +259,7 @@ class StatusResponse: `disk_bandwidth_stats` and `nic_bandwidth_stats` were declared here after the handler had stopped returning them; hotspot and throttle moved to the Controller. """ + backup: BackupStatusResponse cpu_stats: CPUStatusResponse dockerized: bool # Declared `int` for a long time while the handler sent the summary object, so the published @@ -216,6 +275,7 @@ class StatusResponse: memory_stats: MemoryStatusStat processes_stats: List[ProcessStatusStat] sanic_workers: dict + sync: SyncStatusResponse version: str # `str` here was wrong too; wrol_mode_enabled() returns a bool. wrol_mode: bool diff --git a/wrolpi/sync/__init__.py b/wrolpi/sync/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wrolpi/sync/api.py b/wrolpi/sync/api.py new file mode 100644 index 00000000..9e2fc371 --- /dev/null +++ b/wrolpi/sync/api.py @@ -0,0 +1,205 @@ +import asyncio +import os +import re +from http import HTTPStatus + +from sanic import Request, Blueprint, response +from sanic_ext import validate +from sanic_ext.extensions.openapi import openapi + +from wrolpi import flags +from wrolpi.api_utils import api_app, json_response, perpetual_signal +from wrolpi.common import logger, get_wrolpi_config, is_secondary, wrol_mode_check, wrol_mode_enabled +from wrolpi.db import get_db_session +from wrolpi.errors import InvalidConfig, SecondaryRole, SyncInProgress, WROLModeEnabled +from wrolpi.sync import backup, lib, schema +from wrolpi.sync.config import get_sync_config +from wrolpi.sync.status import get_backup_status, get_sync_status +from wrolpi.vars import DOCKERIZED + +sync_bp = Blueprint('Sync', '/api/sync') + +logger = logger.getChild(__name__) + +# Requests a secondary answers while it syncs: enough for the UI to show the sync's progress, and to cancel it. +SYNC_ALLOWED_REQUESTS = frozenset({ + ('GET', '/api/echo'), + ('GET', '/api/events/feed'), + ('GET', '/api/settings'), + ('GET', '/api/status'), + ('GET', '/api/sync/info'), + ('GET', '/api/sync/status'), + ('POST', '/api/sync/cancel'), +}) + + +@api_app.middleware('request') +async def check_sync_api_lock(request: Request): + """A secondary refuses requests while it replaces its library with its primary's (see wrolpi.sync).""" + # The Event costs nothing to check; the Flag confirms it (the Event survives a perpetual process that died). + if not api_app.shared_ctx.sync_api_locked.is_set() or not flags.sync_running.is_set(): + return + if request.method == 'OPTIONS' or (request.method, request.path.rstrip('/')) in SYNC_ALLOWED_REQUESTS: + return + # Answered here rather than raised: the UI polls while it waits, and each refusal would be logged as an error. + return json_response(dict(error=SyncInProgress.summary, code=SyncInProgress.code), + SyncInProgress.status_code) + + +@perpetual_signal(sleep=5) +async def perpetual_sync_worker(): + # One loop, so a secondary's sync and its backup never run at once; the backup copies what the sync pulled. + await lib.sync_tick() + await backup.backup_tick() + + +@sync_bp.get('/info') +@openapi.definition(summary='This WROLPi, as a primary that secondaries sync from') +async def get_info(_: Request): + def info(): + with get_db_session() as session: + return lib.get_sync_info(session) + + return json_response(await asyncio.to_thread(info)) + + +@sync_bp.get('/manifest') +@openapi.definition(summary="One page of this WROLPi's files, for a secondary to compare with its own") +@validate(query=schema.ManifestRequest) +async def get_manifest(_: Request, query: schema.ManifestRequest): + def page(): + with get_db_session() as session: + return lib.build_manifest_page(session, query.after_id, query.limit) + + return json_response(await asyncio.to_thread(page)) + + +@sync_bp.get('/status') +@openapi.definition(summary="The progress of this secondary's sync") +async def get_status(_: Request): + return json_response(dict(sync=get_sync_status(), sync_running=bool(flags.sync_running.is_set()), + backup=get_backup_status(), backup_running=bool(flags.backup_running.is_set()))) + + +@sync_bp.post('/run') +@openapi.definition(summary='Sync from the primary now', body=schema.SyncRunRequest, validate=True) +@wrol_mode_check +async def post_run(_: Request, body: schema.SyncRunRequest): + if not is_secondary(): + raise SecondaryRole('Only a secondary WROLPi syncs from a primary') + if flags.sync_running.is_set(): + raise SyncInProgress() + if not get_sync_config().primary_url: + raise InvalidConfig('No primary is configured') + lib.request_sync(force=body.force) + return response.empty(HTTPStatus.ACCEPTED) + + +@sync_bp.post('/cancel') +@openapi.definition(summary='Stop the running sync') +async def post_cancel(_: Request): + lib.request_sync_cancel() + return response.empty(HTTPStatus.ACCEPTED) + + +@sync_bp.get('/test') +@openapi.definition(summary='Check that the configured primary answers') +async def get_test(_: Request): + url = get_sync_config().primary_url + if not url: + raise InvalidConfig('No primary is configured') + try: + async with lib.PrimaryClient(url) as client: + info = await asyncio.wait_for(client.get_info(), timeout=15) + except Exception as e: + logger.warning(f'Primary {url} did not answer', exc_info=e) + return json_response(dict(ok=False, error=str(e) or type(e).__name__)) + ok = info.get('role') == 'primary' + return json_response(dict(ok=ok, info=info, error=None if ok else f'{url} is not a primary WROLPi')) + + +@sync_bp.post('/directories') +@openapi.definition(summary='List the folders inside a folder, to choose which are not copied', + body=schema.SyncDirectoriesRequest, validate=True) +async def post_directories(_: Request, body: schema.SyncDirectoriesRequest): + if body.source == 'local': + return json_response(dict(directories=lib.list_local_directories(body.path))) + if not is_secondary() or not get_sync_config().primary_url: + raise InvalidConfig('Only a secondary with a primary can list its primary\'s folders') + try: + directories = await asyncio.wait_for(lib.list_primary_directories(body.path), timeout=15) + except InvalidConfig: + raise + except Exception as e: + logger.warning(f'Could not list the primary\'s folder {body.path!r}', exc_info=e) + return json_response(dict(error=str(e) or type(e).__name__), status=HTTPStatus.BAD_GATEWAY) + return json_response(dict(directories=directories)) + + +@sync_bp.put('/settings') +@openapi.definition(summary="Set this WROLPi's role and its primary", body=schema.SyncSettingsRequest, validate=True) +async def put_settings(_: Request, body: schema.SyncSettingsRequest): + if wrol_mode_enabled(): + raise WROLModeEnabled() + changes = dict() + if body.role is not None: + changes['role'] = body.role + if body.sync_primary_url is not None: + try: + changes['primary_url'] = lib.normalize_primary_url(body.sync_primary_url) \ + if body.sync_primary_url else None + except ValueError as e: + raise InvalidConfig(str(e)) + for request_key, config_key in (('sync_window_start', 'window_start'), ('sync_window_end', 'window_end'), + ('backup_window_start', 'backup_window_start'), + ('backup_window_end', 'backup_window_end')): + value = getattr(body, request_key) + if value is None: + continue + if value and not re.match(r'^([01]\d|2[0-3]):[0-5]\d$', value): + raise InvalidConfig(f'{request_key} must be in HH:MM format') + changes[config_key] = value or None + for key in ('sync_excluded_directories', 'backup_excluded_directories'): + if (value := getattr(body, key)) is not None: + changes[key] = lib.normalize_excluded_directories(value) + if body.backup_directory is not None or body.backup_device_uuid is not None: + changes.update(_backup_drive_changes(body.backup_directory, body.backup_device_uuid)) + if not changes: + raise InvalidConfig('No settings to change') + + get_sync_config().update(changes) + if changes.get('role') == 'secondary': + # A secondary's library comes from its primary. + from wrolpi.downloader import download_manager + download_manager.disable() + return response.empty() + + +def _backup_drive_changes(directory: str, uuid: str) -> dict: + if not directory and not uuid: + # No backup drive. + return dict(backup_directory=None, backup_device_uuid=None) + if DOCKERIZED: + raise InvalidConfig('Backup drives are not supported in Docker') + if reason := backup.check_backup_target(directory, uuid): + raise InvalidConfig(reason) + return dict(backup_directory=os.path.normpath(directory), backup_device_uuid=uuid) + + +@sync_bp.post('/backup/run') +@openapi.definition(summary='Copy the media directory to the backup drive now') +@wrol_mode_check +async def post_backup_run(_: Request): + if not get_sync_config().backup_directory: + raise InvalidConfig('No backup drive is configured') + if flags.backup_running.is_set(): + raise InvalidConfig('A backup is already running') + backup.request_backup() + return response.empty(HTTPStatus.ACCEPTED) + + +@sync_bp.post('/backup/cancel') +@openapi.definition(summary='Stop the running backup') +async def post_backup_cancel(_: Request): + backup.request_backup_cancel() + return response.empty(HTTPStatus.ACCEPTED) diff --git a/wrolpi/sync/backup.py b/wrolpi/sync/backup.py new file mode 100644 index 00000000..1b6a66aa --- /dev/null +++ b/wrolpi/sync/backup.py @@ -0,0 +1,430 @@ +"""Copy this WROLPi's media directory to a second drive (its "backup drive"). + +The backup drive mirrors the media directory, so if the media drive dies the backup drive can be mounted as the media +directory and WROLPi starts from it: + +1. rsync copies every file that is new or changed (never `--delete`: a file deleted from the media directory stays on + the backup drive). The Tags Directory and the playlists directory are not copied; they are hard links that + WROLPi rebuilds from the configs when it starts. +2. The database is not copied by rsync (a live database copied as files can be torn); SQLite's backup API writes a + consistent snapshot instead. + +A backup only reads the media directory, so the API stays usable while it runs. It runs in the sync window, or when +the user asks. Nothing is written unless the configured drive is mounted where it was chosen: an unplugged drive +leaves an empty directory on the system drive, which must never be filled. +""" +import asyncio +import contextlib +import os +import pathlib +import re +import shutil +import sqlite3 +import time +from typing import List, Optional, Tuple + +from wrolpi import flags +from wrolpi.common import logger, get_media_directory, get_wrolpi_config, wrol_mode_enabled +from wrolpi.dates import now +from wrolpi.db import get_db_file +from wrolpi.events import Events +from wrolpi.sync.config import get_sync_config +from wrolpi.sync.lib import window_is_due, PROGRESS_INTERVAL_SECONDS +from wrolpi.sync.status import get_backup_status, update_backup_status +from wrolpi.vars import DOCKERIZED, PYTEST + +logger = logger.getChild(__name__) + +# Within the sync window, back up again this long after the last backup finished; each backup walks the whole library. +BACKUP_REPEAT_SECONDS = 60 * 60 +# rsync keeps an interrupted file here (on the backup drive) so the next backup resumes it. +PARTIAL_DIRECTORY = '.rsync-partial' +# rsync exit codes: 23 = some files could not be read, 24 = some files vanished while copying. +RSYNC_VANISHED = 24 +RSYNC_PARTIAL = 23 +# How often a running backup checks that its drive is still mounted. +MOUNT_CHECK_SECONDS = 5.0 +PROC_MOUNTS = pathlib.Path('/proc/mounts') +# Backup drives are mounted in this directory (the Controller mounts drives here). +MOUNT_ROOT = pathlib.Path('/media') +# Mount points (in MOUNT_ROOT) that are never a backup drive. +RESERVED_MOUNT_NAMES = frozenset({'wrolpi_temp_onboarding'}) + +# `--info=progress2` lines: " 1234567 45% 10.5MB/s 0:01:23 (xfr#12, to-chk=100/2000)" +PROGRESS_LINE = re.compile(r'^\s*([\d,]+)\s+(\d+)%\s+\S+\s+\S+(?:\s+\(xfr#(\d+), (?:ir|to)-chk=(\d+)/(\d+)\))?') +FILES_TRANSFERRED_LINE = re.compile(r'^Number of regular files transferred: ([\d,]+)') + + +class BackupError(Exception): + pass + + +class BackupCancelled(BackupError): + pass + + +def _unescape_mount_path(path: str) -> str: + # /proc/mounts escapes spaces and the like as octal, e.g. "\040". + return re.sub(r'\\([0-7]{3})', lambda m: chr(int(m.group(1), 8)), path) + + +def mount_source(mount_point: str) -> Optional[str]: + """The device mounted at `mount_point`, or None. A path mounted twice reports the last (visible) mount.""" + try: + lines = PROC_MOUNTS.read_text().splitlines() + except OSError: + return None + source = None + for line in lines: + parts = line.split() + if len(parts) >= 2 and _unescape_mount_path(parts[1]) == mount_point: + source = parts[0] + return source + + +def uuid_device(uuid: str) -> Optional[str]: + """The device with this filesystem UUID, or None.""" + if not uuid or '/' in uuid: + return None + path = pathlib.Path('/dev/disk/by-uuid') / uuid + return os.path.realpath(path) if path.exists() else None + + +def device_id(path) -> int: + return os.stat(path).st_dev + + +def is_mount(path) -> bool: + return os.path.ismount(path) + + +def find_mount_point(path: pathlib.Path) -> pathlib.Path: + """The mount point of the filesystem that holds `path` (an existing, resolved path).""" + while not is_mount(path) and path != path.parent: + path = path.parent + return path + + +def check_backup_target(directory: Optional[str], uuid: Optional[str]) -> Optional[str]: + """Why `directory` must not be written as the backup, or None if it is on the drive with `uuid`, and that drive is + mounted, and is not the media drive or the system drive. + + `directory` is the drive's mount point, or a folder on the drive. The folder may not exist yet (the backup + creates it); the drive is checked through its nearest existing folder, so a folder on an unplugged drive (which + would be on the system drive) is refused.""" + if not directory or not uuid: + return 'No backup drive is configured' + path = pathlib.Path(directory) + if not path.is_absolute() or any(i in ('.', '..') for i in path.parts): + return f'{directory} is not a valid directory' + existing = path + while not existing.exists() and existing != existing.parent: + existing = existing.parent + try: + resolved_existing = existing.resolve(strict=True) + except (FileNotFoundError, RuntimeError): + return f'{directory} does not exist' + resolved = resolved_existing.joinpath(path.relative_to(existing)) + mount_root = MOUNT_ROOT.resolve() + if mount_root not in resolved.parents: + return f'{directory} is not under {MOUNT_ROOT}' + media_directory = get_media_directory().resolve() + if resolved == media_directory or media_directory in resolved.parents or resolved in media_directory.parents: + return 'The backup drive cannot be the media directory' + mount_point = find_mount_point(resolved_existing) + if mount_root not in mount_point.parents: + # The folder is not on a drive mounted in MOUNT_ROOT; e.g. the drive is unplugged. + return f'No drive is mounted at {directory}' + if mount_point.parent == mount_root and mount_point.name in RESERVED_MOUNT_NAMES: + return f'{mount_point} cannot be a backup drive' + if device_id(mount_point) == device_id(media_directory): + return 'The backup drive is the media drive' + if device_id(mount_point) == device_id('/'): + return 'The backup drive is the system drive' + expected = uuid_device(uuid) + source = mount_source(str(mount_point)) + if not expected or not source or os.path.realpath(source) != expected: + return f'The chosen drive is not mounted at {mount_point}' + if not os.access(resolved_existing, os.W_OK): + return f'{directory} is not writable' + return None + + +def rsync_literal(path: str) -> str: + """`path` as an rsync pattern that matches only itself (rsync reads `* ? [ \\` as wildcards).""" + return re.sub(r'([*?\[\\])', r'\\\1', path) + + +def build_rsync_command(source: pathlib.Path, target: pathlib.Path, playlists_destination: str = None, + excluded_directories: List[str] = None) -> List[str]: + """The rsync that copies `source` (the media directory) to `target` (the backup drive), never deleting.""" + command = list() + # The backup should not slow the WROLPi down. + if ionice := shutil.which('ionice'): + command.extend([ionice, '-c3']) + if nice := shutil.which('nice'): + command.extend([nice, '-n', '19']) + command.extend([ + shutil.which('rsync') or 'rsync', + '--archive', + '--partial', f'--partial-dir={PARTIAL_DIRECTORY}', + '--info=progress2,stats2', '--no-human-readable', + # The database is copied as a snapshot (see `snapshot_database`). + '--exclude=/config/wrolpi.db', '--exclude=/config/wrolpi.db-*', '--exclude=/config/.wrolpi.db.backup', + # Hard links rebuilt from the configs when WROLPi starts. + '--exclude=/tags/', + # Files still being written by a sync, and rsync's own partials. + '--exclude=.*.partial', f'--exclude={PARTIAL_DIRECTORY}/', + # Root-only directory on every ext4 drive. + '--exclude=/lost+found/', + ]) + if playlists_destination and (playlists := playlists_destination.strip('/')): + command.append(f'--exclude=/{playlists}/') + # Folders the user chose not to back up. + for directory in excluded_directories or []: + if directory := directory.strip('/'): + command.append(f'--exclude=/{rsync_literal(directory)}/') + command.extend([f'{source}/', f'{target}/']) + return command + + +def parse_progress(line: str) -> Optional[dict]: + if match := PROGRESS_LINE.match(line): + progress = dict(bytes_transferred=int(match.group(1).replace(',', '')), percent=int(match.group(2))) + if match.group(3): + progress['files_transferred'] = int(match.group(3)) + progress['files_to_check'] = int(match.group(5)) + return progress + if match := FILES_TRANSFERRED_LINE.match(line): + return dict(files_transferred=int(match.group(1).replace(',', ''))) + return None + + +def snapshot_database(target: pathlib.Path): + """Write a consistent copy of the live database to the backup drive's config directory.""" + config_directory = target / 'config' + config_directory.mkdir(parents=True, exist_ok=True) + temporary = config_directory / '.wrolpi.db.backup' + temporary.unlink(missing_ok=True) + with contextlib.closing(sqlite3.connect(get_db_file(), timeout=60)) as source, \ + contextlib.closing(sqlite3.connect(temporary)) as destination: + # One step: a read transaction, which writers in WAL mode do not wait for, and which never restarts. + source.backup(destination) + database = config_directory / 'wrolpi.db' + # A write-ahead log left by an older copy would be applied to the new database. + for suffix in ('-wal', '-shm'): + database.with_name(f'wrolpi.db{suffix}').unlink(missing_ok=True) + os.replace(temporary, database) + + +def database_fingerprint(database: pathlib.Path) -> Optional[list]: + """The size and modification time of a database and of its write-ahead log, or None if it does not exist. A + commit changes one of them. An empty WAL (opened by a reader) holds nothing, so it is ignored.""" + try: + stat = database.stat() + except FileNotFoundError: + return None + fingerprint = [stat.st_size, stat.st_mtime_ns] + try: + wal = database.with_name(f'{database.name}-wal').stat() + if wal.st_size: + fingerprint.extend([wal.st_size, wal.st_mtime_ns]) + except FileNotFoundError: + pass + return fingerprint + + +def snapshot_database_if_changed(target: pathlib.Path) -> bool: + """Write the snapshot (see `snapshot_database`) unless the live database and the drive's copy are both unchanged + since the last snapshot. Returns True if the snapshot was written. + + The record is kept in memory, so the first backup after a restart always writes it.""" + from wrolpi.api_utils import api_app + copy = target / 'config/wrolpi.db' + key = f'backup_database_snapshot:{copy}' + # Taken before the snapshot: a commit during the snapshot makes the next backup write it again. + source = database_fingerprint(get_db_file()) + previous = api_app.shared_ctx.cache.get(key) + if previous and source and previous == [source, database_fingerprint(copy)]: + logger.info(f'The database has not changed; keeping the snapshot in {copy.parent}') + return False + snapshot_database(target) + api_app.shared_ctx.cache[key] = [source, database_fingerprint(copy)] + return True + + +def backup_skip_reason() -> Optional[str]: + """Why a backup cannot start now, or None.""" + config = get_sync_config() + if not config.backup_directory: + return 'No backup drive is configured' + if DOCKERIZED: + return 'Backup drives are not supported in Docker' + if wrol_mode_enabled(): + return 'WROL Mode is enabled' + if not PYTEST and not flags.db_up.is_set(): + return 'The database is not up' + if not shutil.which('rsync'): + return 'rsync is not installed' + if reason := check_backup_target(config.backup_directory, config.backup_device_uuid): + return reason + if flags.backup_running.is_set(): + return 'A backup is already running' + return None + + +def request_backup(): + update_backup_status(requested=True) + + +def request_backup_cancel(): + from wrolpi.api_utils import api_app + if 'backup' not in api_app.shared_ctx.status: + return + status = get_backup_status() + if status['state'] == 'running' or status['requested']: + update_backup_status(cancel_requested=status['state'] == 'running', requested=False) + + +def _check_backup_cancel(): + if get_backup_status()['cancel_requested']: + raise BackupCancelled('The backup was cancelled') + if wrol_mode_enabled(): + raise BackupCancelled('WROL Mode was enabled') + + +async def _run_rsync(command: List[str], target: pathlib.Path, uuid: str) -> Tuple[int, List[str]]: + """Run rsync, reporting its progress. Stops it if the backup is cancelled or the drive disappears.""" + logger.info(f'Running {" ".join(command)}') + process = await asyncio.create_subprocess_exec(*command, stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE) + errors: List[str] = list() + + async def read_errors(): + async for line in process.stderr: + errors.append(line.decode(errors='replace').rstrip()) + del errors[:-20] + + errors_task = asyncio.create_task(read_errors()) + buffer = '' + last_report = last_mount_check = 0.0 + progress = dict() + try: + while True: + try: + chunk = await asyncio.wait_for(process.stdout.read(65536), timeout=1) + except asyncio.TimeoutError: + chunk = None + if chunk == b'': + break + if chunk: + buffer += chunk.decode(errors='replace') + *lines, buffer = re.split(r'[\r\n]', buffer) + for line in lines: + if parsed := parse_progress(line): + progress.update(parsed) + if time.monotonic() - last_report >= PROGRESS_INTERVAL_SECONDS and progress: + last_report = time.monotonic() + update_backup_status(**progress) + if time.monotonic() - last_mount_check >= MOUNT_CHECK_SECONDS: + last_mount_check = time.monotonic() + if reason := check_backup_target(str(target), uuid): + raise BackupError(f'Stopped: {reason}') + _check_backup_cancel() + code = await process.wait() + except BaseException: + if process.returncode is None: + process.terminate() + try: + await asyncio.wait_for(process.wait(), timeout=30) + except asyncio.TimeoutError: + process.kill() + raise + finally: + await asyncio.gather(errors_task, return_exceptions=True) + if buffer and (parsed := parse_progress(buffer)): + progress.update(parsed) + if progress: + update_backup_status(**progress) + return code, errors + + +async def run_backup() -> dict: + """Copy the media directory to the backup drive. Returns the final status.""" + if reason := backup_skip_reason(): + logger.info(f'Not backing up: {reason}') + return update_backup_status(last_skip_reason=reason) + + config = get_sync_config() + target = pathlib.Path(config.backup_directory) + uuid = config.backup_device_uuid + started_at = now() + update_backup_status(state='running', phase='files', cancel_requested=False, started_at=started_at.isoformat(), + finished_at=None, last_error=None, last_skip_reason=None, directory=str(target), + bytes_transferred=0, percent=0, files_transferred=0, files_to_check=0) + logger.warning(f'Backing up to {target}') + warning = None + try: + with flags.backup_running: + # A folder on the drive is created once the drive has been checked (`backup_skip_reason`). + target.mkdir(parents=True, exist_ok=True) + command = build_rsync_command(get_media_directory(), target, get_wrolpi_config().playlists_destination, + config.backup_excluded_directories) + code, errors = await _run_rsync(command, target, uuid) + if code == RSYNC_PARTIAL: + # Unreadable files; everything else was copied. + warning = 'Some files could not be read: ' + ' '.join(errors[-3:]) + logger.warning(warning) + elif code not in (0, RSYNC_VANISHED): + raise BackupError(f'rsync failed ({code}): ' + ' '.join(errors[-3:])) + _check_backup_cancel() + + update_backup_status(phase='database') + if reason := check_backup_target(str(target), uuid): + raise BackupError(f'Stopped: {reason}') + await asyncio.to_thread(snapshot_database_if_changed, target) + except BackupCancelled as e: + logger.warning(f'Backup stopped: {e}') + return update_backup_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e), + cancel_requested=False) + except Exception as e: + logger.error('Backup failed', exc_info=e) + Events.send_backup_failed(f'Backup to {target} failed: {e}') + return update_backup_status(state='error', phase=None, finished_at=now().isoformat(), last_error=str(e), + cancel_requested=False) + + finished_at = now() + status = get_backup_status() + if not warning: + get_sync_config().update({'backup_last_success_at': finished_at.isoformat()}) + message = f'Backed up {status["files_transferred"]} files to {target}' + logger.warning(f'{message} in {(finished_at - started_at).total_seconds():.0f}s') + if status['files_transferred'] or warning: + Events.send_backup_completed(message + (f'; {warning}' if warning else '')) + return update_backup_status(state='idle', phase=None, finished_at=finished_at.isoformat(), + last_success_at=finished_at.isoformat() if not warning else status['last_success_at'], + last_error=warning, percent=100, + # A request to stop during the database snapshot (which runs to the end) is done. + cancel_requested=False) + + +def backup_is_due(status: dict, local_now) -> bool: + """True if the backup window is open and the last backup finished long enough ago.""" + config = get_sync_config() + return window_is_due(status, local_now, config.backup_window_start, config.backup_window_end, + BACKUP_REPEAT_SECONDS) + + +async def backup_tick(): + """Start a backup when one was requested, or when it is due.""" + status = get_backup_status() + if not get_sync_config().backup_directory or wrol_mode_enabled(): + if status['requested']: + reason = 'WROL Mode is enabled' if wrol_mode_enabled() else 'No backup drive is configured' + update_backup_status(requested=False, last_skip_reason=reason) + return + from wrolpi.downloader import DownloadManager + if not status['requested'] and not backup_is_due(status, DownloadManager._get_local_now()): + return + update_backup_status(requested=False) + await run_backup() diff --git a/wrolpi/sync/config.py b/wrolpi/sync/config.py new file mode 100644 index 00000000..da2c98f3 --- /dev/null +++ b/wrolpi/sync/config.py @@ -0,0 +1,110 @@ +"""sync.yaml: how this WROLPi syncs. + +These settings belong to this machine: they are never copied from a primary (a secondary's sync.yaml says it is a +secondary; its primary's says otherwise), so they are kept apart from wrolpi.yaml, whose library settings are. +""" +from dataclasses import dataclass, field +from typing import List, Optional + +from wrolpi.common import ConfigFile +from wrolpi.sync.status import get_backup_status, update_backup_status + + +@dataclass +class SyncConfigValidator: + version: int = None + # 'primary' or 'secondary'; a secondary copies its library from `primary_url` (see wrolpi.sync.lib). + role: str = None + primary_url: str = None + # The daily window ('HH:MM') in which a secondary syncs; none means only "Sync Now". + window_start: str = None + window_end: str = None + # Folders (relative to the media directory) a secondary never downloads from its primary. + sync_excluded_directories: list[str] = field(default_factory=list) + # A second drive this WROLPi copies its media directory to (see wrolpi.sync.backup), and the UUID of the drive + # that must be mounted there. + backup_directory: str = None + backup_device_uuid: str = None + # The daily window ('HH:MM') in which the backup drive is written; none means only "Back Up Now". + backup_window_start: str = None + backup_window_end: str = None + # Folders (relative to the media directory) never copied to the backup drive. + backup_excluded_directories: list[str] = field(default_factory=list) + # When a backup last finished without errors (ISO 8601); kept so it is known after a restart. + backup_last_success_at: str = None + + +class SyncConfig(ConfigFile): + file_name = 'sync.yaml' + default_config = dict( + version=0, + role='primary', + primary_url=None, + window_start=None, + window_end=None, + sync_excluded_directories=[], + backup_directory=None, + backup_device_uuid=None, + backup_window_start=None, + backup_window_end=None, + backup_excluded_directories=[], + backup_last_success_at=None, + ) + validator = SyncConfigValidator + + def import_config(self, file=None, send_events=False): + super().import_config(file, send_events) + # The backup's status lives in memory; a restart would otherwise forget when the last backup succeeded. + if self.backup_last_success_at and not get_backup_status()['last_success_at']: + update_backup_status(last_success_at=self.backup_last_success_at) + + @property + def role(self) -> str: + return self._config.get('role') or 'primary' + + @property + def primary_url(self) -> Optional[str]: + return self._config.get('primary_url') + + @property + def window_start(self) -> Optional[str]: + return self._config.get('window_start') + + @property + def window_end(self) -> Optional[str]: + return self._config.get('window_end') + + @property + def sync_excluded_directories(self) -> List[str]: + return list(self._config.get('sync_excluded_directories') or []) + + @property + def backup_directory(self) -> Optional[str]: + return self._config.get('backup_directory') + + @property + def backup_device_uuid(self) -> Optional[str]: + return self._config.get('backup_device_uuid') + + @property + def backup_window_start(self) -> Optional[str]: + return self._config.get('backup_window_start') + + @property + def backup_window_end(self) -> Optional[str]: + return self._config.get('backup_window_end') + + @property + def backup_excluded_directories(self) -> List[str]: + return list(self._config.get('backup_excluded_directories') or []) + + @property + def backup_last_success_at(self) -> Optional[str]: + return self._config.get('backup_last_success_at') + + +SYNC_CONFIG: SyncConfig = SyncConfig() + + +def get_sync_config() -> SyncConfig: + return SYNC_CONFIG diff --git a/wrolpi/sync/conftest.py b/wrolpi/sync/conftest.py new file mode 100644 index 00000000..0f2f09f9 --- /dev/null +++ b/wrolpi/sync/conftest.py @@ -0,0 +1,158 @@ +import contextlib +import email.utils +import functools +import http.server +import json +import os +import pathlib +import tempfile +import threading +from typing import Callable, List + +import pytest + +from wrolpi.common import get_wrolpi_config, set_test_media_directory +from wrolpi.sync import lib +from wrolpi.sync.config import get_sync_config + + +class CaddyLikeHandler(http.server.SimpleHTTPRequestHandler): + """Serves a directory at /media/ the way the primary's Caddy does: Range requests, and JSON directory listings + when asked with `Accept: application/json`.""" + + def __init__(self, *args, server_state: dict, **kwargs): + self.server_state = server_state + super().__init__(*args, **kwargs) + + def log_message(self, *args): + pass + + def translate_path(self, path): + path = path.split('?', 1)[0] + if path.startswith('/media/'): + return super().translate_path(path[len('/media'):]) + return '/nonexistent-wrolpi-sync-test-path' + + def do_GET(self): + self.server_state['requests'].append((self.path, self.headers.get('Range'))) + path = self.translate_path(self.path) + if os.path.isdir(path) and 'application/json' in (self.headers.get('Accept') or ''): + entries = list() + for name in sorted(os.listdir(path)): + full = os.path.join(path, name) + is_dir = os.path.isdir(full) + entries.append(dict(name=name + ('/' if is_dir else ''), size=os.path.getsize(full), is_dir=is_dir, + mod_time=email.utils.formatdate(os.path.getmtime(full)))) + body = json.dumps(entries).encode() + self.send_response(200) + self.send_header('Content-Type', 'application/json') + self.send_header('Content-Length', str(len(body))) + self.end_headers() + self.wfile.write(body) + return + + if self.server_state['omit_length'] and os.path.isfile(path): + # The body ends when the connection closes (HTTP/1.0), so the client cannot know its size. + self.send_response(200) + self.end_headers() + with open(path, 'rb') as fh: + self.wfile.write(fh.read()) + return + + range_ = self.headers.get('Range') + if range_ and self.server_state['honor_range'] and os.path.isfile(path): + start = int(range_.split('=', 1)[1].split('-', 1)[0]) + size = os.path.getsize(path) + if start >= size: + self.send_response(416) + self.send_header('Content-Length', '0') + self.end_headers() + return + self.send_response(206) + self.send_header('Content-Length', str(size - start)) + self.send_header('Content-Range', f'bytes {start}-{size - 1}/{size}') + self.end_headers() + with open(path, 'rb') as fh: + fh.seek(start) + self.wfile.write(fh.read()) + return + + super().do_GET() + + +@pytest.fixture +def primary_media_server() -> Callable[[pathlib.Path], tuple]: + """Start a server that serves a directory like the primary's Caddy. Returns (url, state); `state['requests']` + records (path, Range header) of each request, `state['honor_range'] = False` makes it ignore Range, and + `state['omit_length'] = True` makes it send files without a Content-Length.""" + servers = list() + + def start(directory: pathlib.Path): + state = dict(requests=list(), honor_range=True, omit_length=False) + handler = functools.partial(CaddyLikeHandler, directory=str(directory), server_state=state) + server = http.server.ThreadingHTTPServer(('127.0.0.1', 0), handler) + threading.Thread(target=server.serve_forever, daemon=True).start() + servers.append(server) + return f'http://127.0.0.1:{server.server_address[1]}', state + + yield start + + for server in servers: + server.shutdown() + server.server_close() + + +@pytest.fixture +def secondary_directory(test_directory) -> pathlib.Path: + """A second, empty media directory. The test switches to it (`use_secondary`) after preparing the primary in + `test_directory`; both "machines" share the test database.""" + with tempfile.TemporaryDirectory() as tmp_dir: + path = pathlib.Path(tmp_dir).resolve() + (path / 'config').mkdir() + try: + yield path + finally: + set_test_media_directory(test_directory) + + +@pytest.fixture +def make_secondary(test_wrolpi_config): + """Make this WROLPi a secondary of `url`.""" + + def make(url: str = 'http://127.0.0.1:1'): + get_sync_config()._config.update(role='secondary', primary_url=url) + + return make + + +@pytest.fixture +def fake_primary_api(monkeypatch): + """Answer the primary's sync API (info and manifest) from pages captured on the "primary" beforehand; media still + comes from the `primary_media_server`.""" + + def fake(pages: List[dict], media_directory: pathlib.Path, role: str = 'primary', version: str = None): + async def get_info(self): + from wrolpi.version import __version__ + return dict(role=role, version=version or __version__, media_directory=str(media_directory), + max_file_group_id=pages[-1]['max_id'] if pages else 0) + + async def iter_manifest(self, limit: int = lib.MANIFEST_PAGE_LIMIT): + for page in pages: + yield page + + monkeypatch.setattr(lib.PrimaryClient, 'get_info', get_info) + monkeypatch.setattr(lib.PrimaryClient, 'iter_manifest', iter_manifest) + + return fake + + +def manifest_page(*files: tuple, next_after_id=None, max_id: int = 1) -> dict: + """A manifest page of one group holding `files`, each (path, size, mtime).""" + return dict(groups=[dict(id=1, paths=[dict(path=p, size=s, mtime=m) for p, s, m in files])], + next_after_id=next_after_id, max_id=max_id, limit=lib.MANIFEST_PAGE_LIMIT) + + +@contextlib.contextmanager +def media_directory(path: pathlib.Path): + set_test_media_directory(path) + yield path diff --git a/wrolpi/sync/lib.py b/wrolpi/sync/lib.py new file mode 100644 index 00000000..e519e109 --- /dev/null +++ b/wrolpi/sync/lib.py @@ -0,0 +1,866 @@ +"""Copy a primary WROLPi's library to a secondary WROLPi. + +A secondary pulls; the primary only answers reads. The library is the media files plus the config YAMLs, because the +database is rebuilt from both, so a sync is: + +1. Walk the primary's manifest (`/api/sync/manifest`): every file of every FileGroup, paged by FileGroup id. +2. Download each file the secondary lacks, or has with a different size or modification time, from the primary's + `/media/` (Caddy). A file is written to a hidden partial (refresh never indexes hidden files), then renamed into + place with the primary's modification time, so a later comparison is exact and an interrupted download resumes. +3. Refresh the downloaded files, a page at a time. +4. Copy the synced configs (`SYNCED_CONFIG_FILE_NAMES`) and import them. Files come first so the configs never + refer to files the secondary does not have yet. + +Files deleted on the primary are never deleted on the secondary. + +While a sync runs the secondary's API refuses requests (`check_sync_api_lock`), so nobody edits a library that is +being replaced. +""" +import asyncio +import contextlib +import json +import os +import pathlib +import posixpath +import re +import shutil +import ssl +import time +from dataclasses import dataclass +from datetime import datetime +from http import HTTPStatus +from typing import AsyncGenerator, List, Optional +from urllib.parse import quote + +import aiohttp +import yaml +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from wrolpi import flags +from wrolpi.common import logger, get_media_directory, get_wrolpi_config, is_secondary, wrol_mode_enabled, \ + get_config_by_file_name, SYNCED_CONFIG_FILE_NAMES +from wrolpi.dates import now, is_within_daily_window +from wrolpi.errors import InvalidConfig +from wrolpi.events import Events +from wrolpi.sync.config import get_sync_config +from wrolpi.sync.status import get_sync_status, update_sync_status +from wrolpi.vars import DEFAULT_HTTP_HEADERS, PYTEST +from wrolpi.version import __version__ + +logger = logger.getChild(__name__) + +MANIFEST_PAGE_LIMIT = 500 +# Files downloaded at once. Small files are dominated by round trips; four keeps a Pi's disk and CPU busy. +DOWNLOAD_CONCURRENCY = 4 +DOWNLOAD_CHUNK_SIZE = 1024 * 1024 +# Never fill the secondary's drive; leave this much free. +MIN_FREE_BYTES = 1024 ** 3 +# exFAT stores modification times in 2-second steps. +MTIME_TOLERANCE_SECONDS = 2.0 +# How often a running sync reports its progress. +PROGRESS_INTERVAL_SECONDS = 2.0 +# Within the sync window, sync again this long after the last sync finished (an idle sync only compares sizes). +SYNC_WINDOW_REPEAT_SECONDS = 15 * 60 +REFRESH_TIMEOUT_SECONDS = 60 * 60 +# Configs and directory listings are small; never read more than this from the primary. +MAX_CONFIG_BYTES = 64 * 1024 * 1024 +# Enough for any real library; keeps a pasted list from growing without bound. +MAX_EXCLUDED_DIRECTORIES = 500 + +# Top-level directories never written by a sync: the configs and database, and the Tags Directory (hard links that +# the secondary rebuilds from tags.yaml). +PROTECTED_DIRECTORIES = frozenset({'config', 'tags'}) +# Top-level folders never offered for exclusion: the protected ones, and the root-only folder of every ext4 drive. +UNLISTED_DIRECTORIES = PROTECTED_DIRECTORIES | {'lost+found'} +# Files that are still being downloaded on the primary: aria2 and yt-dlp partials, and yt-dlp format files. +INCOMPLETE_FILE = re.compile(r'(\.part|\.aria2|\.ytdl|\.temp)$|\.f\d{2,4}\.[^.]+$') + +# wrolpi.yaml keys that describe the library rather than this machine (hotspot, timezone, etc. stay local; the role +# and other sync settings are in sync.yaml, which is never synced). +SYNCED_WROLPI_CONFIG_KEYS = ( + 'archive_destination', + 'ignore_outdated_zims', + 'ignored_directories', + 'map_default_location', + 'map_destination', + 'playlists_destination', + 'repos_destination', + 'save_ffprobe_json', + 'tags_directory', + 'videos_destination', + 'zims_destination', +) + + +class SyncError(Exception): + pass + + +class SyncDiskFull(SyncError): + pass + + +class SyncCancelled(SyncError): + pass + + +@dataclass(frozen=True) +class ManifestFile: + path: str # Relative to the media directory, '/' separated. + size: int + mtime: float + + +# --------------------------------------------------------------------------------------------------------------------- +# Primary: the manifest. + + +def build_manifest_page(session: Session, after_id: int = 0, limit: int = MANIFEST_PAGE_LIMIT) -> dict: + """One page of the primary's library: the files of the FileGroups after `after_id`, in id order. + + Sizes and modification times come from the files themselves, not the database, which does not record them for + every file of a group.""" + from wrolpi.files.models import FileGroup + + media_directory = get_media_directory().resolve() + rows = session.execute( + select(FileGroup.id, FileGroup.directory, FileGroup.files) + .where(FileGroup.id > after_id) + .order_by(FileGroup.id) + .limit(limit) + ).all() + max_id = session.execute(select(func.max(FileGroup.id))).scalar() or 0 + + groups = list() + for id_, directory, files in rows: + paths = list() + for entry in files or []: + if not isinstance(entry, dict) or not entry.get('path'): + continue + path = pathlib.Path(entry['path']) + if not path.is_absolute(): + # Files are stored relative to their FileGroup's directory; older rows stored absolute paths. + path = pathlib.Path(directory) / path + if manifest_file := _manifest_file(media_directory, path): + paths.append(manifest_file) + groups.append(dict(id=id_, paths=[dict(path=i.path, size=i.size, mtime=i.mtime) for i in paths])) + + next_after_id = rows[-1][0] if len(rows) == limit else None + return dict(groups=groups, next_after_id=next_after_id, max_id=max_id, limit=limit) + + +def _is_protected(name: str) -> bool: + """True if the top-level folder `name` is a protected one. Case is ignored: on a drive that ignores case (exFAT, + NTFS, macOS) 'Config' is the real config folder.""" + return name.casefold() in PROTECTED_DIRECTORIES + + +def _is_unlisted(name: str) -> bool: + return name.casefold() in UNLISTED_DIRECTORIES + + +def _manifest_file(media_directory: pathlib.Path, path: pathlib.Path) -> Optional[ManifestFile]: + try: + relative = path.relative_to(media_directory) + except ValueError: + return None + parts = relative.parts + if not parts or any(i.startswith('.') for i in parts) or _is_protected(parts[0]): + return None + if INCOMPLETE_FILE.search(path.name): + return None + try: + stat = path.stat() + except (FileNotFoundError, NotADirectoryError): + return None + if not path.is_file(): + return None + return ManifestFile(relative.as_posix(), stat.st_size, stat.st_mtime) + + +def get_sync_info(session: Session) -> dict: + from wrolpi.files.models import FileGroup + return dict( + version=__version__.strip(), + role=get_sync_config().role, + media_directory=str(get_media_directory()), + max_file_group_id=session.execute(select(func.max(FileGroup.id))).scalar() or 0, + ) + + +# --------------------------------------------------------------------------------------------------------------------- +# Secondary: comparing and writing files. + + +def resolve_destination(media_directory: pathlib.Path, path: str) -> Optional[pathlib.Path]: + """The secondary's path for a manifest path, or None if the path must not be written: outside the media directory, + hidden, protected, or reached through a symlink.""" + if not path or path.startswith('/') or '\\' in path: + return None + parts = pathlib.PurePosixPath(path).parts + if not parts or any(i in ('.', '..') or i.startswith('.') for i in parts): + return None + if _is_protected(parts[0]): + return None + media_directory = media_directory.resolve() + destination = media_directory.joinpath(*parts) + # A symlink anywhere on the way (or the file itself) resolves elsewhere. + if pathlib.Path(os.path.realpath(destination)) != destination: + return None + return destination + + +def normalize_excluded_directories(values: List[str]) -> List[str]: + """Excluded folders as '/'-separated paths relative to the media directory, without duplicates or folders that + are already inside another excluded folder.""" + if len(values) > MAX_EXCLUDED_DIRECTORIES: + raise InvalidConfig(f'No more than {MAX_EXCLUDED_DIRECTORIES} folders can be excluded') + directories = set() + for value in values: + if not isinstance(value, str) or '\\' in value or '\x00' in value: + raise InvalidConfig(f'Invalid folder: {value!r}') + stripped = value.strip().strip('/') + # Reject '..' before normalizing, which would quietly resolve it. + parts = [i for i in stripped.split('/') if i] if stripped else [] + directory = posixpath.normpath(stripped) if stripped else '' + if not parts or any(i.startswith('.') for i in parts): + raise InvalidConfig(f'Invalid folder: {value!r}') + if _is_protected(parts[0]): + raise InvalidConfig(f'{parts[0]} is never copied') + directories.add(directory) + return sorted(i for i in directories if not any(is_excluded(i, {j}) for j in directories if j != i)) + + +def is_excluded(path: str, excluded_directories) -> bool: + """True if `path` (relative, '/'-separated) is an excluded folder or inside one.""" + return any(path == i or path.startswith(f'{i}/') for i in excluded_directories) + + +def list_local_directories(path: str) -> List[dict]: + """The folders directly inside `path` (relative to the media directory) that a user may exclude.""" + media_directory = get_media_directory().resolve() + path = (path or '').strip('/') + directory = resolve_destination(media_directory, path) if path else media_directory + if directory is None or not directory.is_dir(): + raise InvalidConfig(f'Not a folder: {path!r}') + directories = list() + with os.scandir(directory) as entries: + for entry in entries: + if entry.name.startswith('.') or not entry.is_dir(follow_symlinks=False): + continue + if not path and _is_unlisted(entry.name): + continue + directories.append(dict(name=entry.name, path=f'{path}/{entry.name}' if path else entry.name)) + return sorted(directories, key=lambda i: i['name'].lower()) + + +async def list_primary_directories(path: str) -> List[dict]: + """The folders directly inside `path` in the primary's media directory (Caddy's browse listing).""" + path = (path or '').strip('/') + if path and resolve_destination(get_media_directory(), path) is None: + raise InvalidConfig(f'Not a folder: {path!r}') + async with PrimaryClient(get_sync_config().primary_url) as client: + entries = await client.list_directory(path) + directories = list() + for entry in entries: + name = (entry.get('name') or '').rstrip('/') + if not entry.get('is_dir') or not name or name.startswith('.') or '/' in name: + continue + if not path and _is_unlisted(name): + continue + directories.append(dict(name=name, path=f'{path}/{name}' if path else name)) + return sorted(directories, key=lambda i: i['name'].lower()) + + +def needs_download(destination: pathlib.Path, file: ManifestFile) -> bool: + try: + stat = destination.stat() + except FileNotFoundError: + return True + return stat.st_size != file.size or abs(stat.st_mtime - file.mtime) > MTIME_TOLERANCE_SECONDS + + +def partial_path(destination: pathlib.Path) -> pathlib.Path: + """Hidden, so a refresh never indexes a file that is still downloading.""" + return destination.with_name(f'.{destination.name}.partial') + + +def _cpu_lacks_aes() -> bool: + """True when the CPU has no AES instructions (a Raspberry Pi 4). ChaCha20 is then about twice as fast.""" + try: + cpuinfo = pathlib.Path('/proc/cpuinfo').read_text() + except OSError: + return False + features = [i for i in cpuinfo.splitlines() if i.lower().startswith(('features', 'flags'))] + return bool(features) and not any(' aes' in f' {i.split(":", 1)[-1]} ' for i in features) + + +def make_sync_ssl_context() -> ssl.SSLContext: + """WROLPi serves HTTPS with its own certificate authority; on a trusted LAN (see the Bookshelf threat model) the + primary is not verified. Pinning it would load the primary's `/ca.crt` here with `load_verify_locations`.""" + context = ssl.create_default_context() + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + if _cpu_lacks_aes(): + # Python cannot order TLS 1.3 ciphers, and TLS 1.3 picks AES-GCM, which a Pi 4 decrypts at half the speed. + context.maximum_version = ssl.TLSVersion.TLSv1_2 + context.set_ciphers('ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE+AESGCM') + return context + + +def normalize_primary_url(url: str) -> str: + """'https://host[:port]' with no path; raises ValueError otherwise.""" + from urllib.parse import urlparse + url = (url or '').strip().rstrip('/') + parsed = urlparse(url) + if parsed.scheme not in ('http', 'https') or not parsed.hostname: + raise ValueError('The primary must be an http:// or https:// address') + if parsed.username or parsed.password: + # The address is logged and shown in the sync's status. + raise ValueError('The primary address must not contain a user name or password') + if parsed.path or parsed.query or parsed.fragment: + raise ValueError('The primary address must not have a path') + return url + + +async def _read_limited(response: aiohttp.ClientResponse, path: str) -> bytes: + data = bytearray() + async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE): + data.extend(chunk) + if len(data) > MAX_CONFIG_BYTES: + raise SyncError(f'{path} is larger than {MAX_CONFIG_BYTES} bytes') + return bytes(data) + + +class PrimaryClient: + """Reads from a primary WROLPi: its sync API, and its media directory through Caddy.""" + + def __init__(self, url: str): + self.url = normalize_primary_url(url) + self.session: Optional[aiohttp.ClientSession] = None + + async def __aenter__(self): + ssl_context = make_sync_ssl_context() if self.url.startswith('https') else False + connector = aiohttp.TCPConnector(ssl=ssl_context, limit=DOWNLOAD_CONCURRENCY * 2) + # A large file takes minutes; only a stalled read is an error. + timeout = aiohttp.ClientTimeout(total=None, connect=30, sock_read=120) + self.session = aiohttp.ClientSession(connector=connector, timeout=timeout, headers=DEFAULT_HTTP_HEADERS) + return self + + async def __aexit__(self, *exc): + await self.session.close() + + def media_url(self, path: str) -> str: + return f'{self.url}/media/{quote(path)}' + + async def get_json(self, path: str, params: dict = None, headers: dict = None): + async with self.session.get(f'{self.url}{path}', params=params, headers=headers) as response: + if response.status != HTTPStatus.OK: + raise SyncError(f'{path} returned HTTP {response.status}') + return json.loads(await _read_limited(response, path)) + + async def get_info(self) -> dict: + return await self.get_json('/api/sync/info') + + async def iter_manifest(self, limit: int = MANIFEST_PAGE_LIMIT) -> AsyncGenerator[dict, None]: + after_id = 0 + while True: + page = await self.get_json('/api/sync/manifest', params=dict(after_id=after_id, limit=limit)) + yield page + if page.get('next_after_id') is None: + return + after_id = page['next_after_id'] + + async def get_bytes(self, path: str) -> Optional[bytes]: + """A file in the primary's media directory, or None if it does not exist.""" + async with self.session.get(self.media_url(path)) as response: + if response.status == HTTPStatus.NOT_FOUND: + return None + if response.status != HTTPStatus.OK: + raise SyncError(f'{path} returned HTTP {response.status}') + return await _read_limited(response, path) + + async def list_directory(self, path: str) -> List[dict]: + """The entries of a directory in the primary's media directory (Caddy's browse listing), [] if missing.""" + url = self.media_url(path.rstrip('/') + '/') + async with self.session.get(url, headers={'Accept': 'application/json'}) as response: + if response.status == HTTPStatus.NOT_FOUND: + return [] + if response.status != HTTPStatus.OK: + raise SyncError(f'{path} returned HTTP {response.status}') + return json.loads(await _read_limited(response, path)) + + @contextlib.asynccontextmanager + async def open_media(self, path: str, offset: int = 0): + # Sizes and ranges count the file's bytes, so the file must not be compressed in transit. + headers = {'Accept-Encoding': 'identity'} + if offset: + headers['Range'] = f'bytes={offset}-' + async with self.session.get(self.media_url(path), headers=headers) as response: + yield response + + +async def download_file(client: PrimaryClient, file: ManifestFile, destination: pathlib.Path, + on_bytes=None) -> int: + """Download one file to `destination` with the primary's modification time. Returns the bytes received. + + A partial file left by an earlier attempt is resumed.""" + partial = partial_path(destination) + destination.parent.mkdir(parents=True, exist_ok=True) + + offset = partial.stat().st_size if partial.is_file() else 0 + if offset > file.size: + offset = 0 + remaining = file.size - offset + if shutil.disk_usage(destination.parent).free - remaining < MIN_FREE_BYTES: + raise SyncDiskFull(f'Not enough free space for {file.path}') + + received = 0 + if remaining > 0 or not partial.is_file(): + async with client.open_media(file.path, offset) as response: + if response.status == HTTPStatus.PARTIAL_CONTENT and offset: + mode = 'ab' + elif response.status == HTTPStatus.OK: + mode, offset = 'wb', 0 + elif response.status == HTTPStatus.REQUESTED_RANGE_NOT_SATISFIABLE: + # The partial does not match the primary's file; start over next time. + partial.unlink(missing_ok=True) + raise SyncError(f'{file.path} changed on the primary') + else: + raise SyncError(f'{file.path} returned HTTP {response.status}') + + if response.content_length is not None and response.content_length != file.size - offset: + # The file changed on the primary after the manifest was read; the next sync gets it. + raise SyncError(f'{file.path} changed on the primary') + + with open(partial, mode) as fh: + async for chunk in response.content.iter_chunked(DOWNLOAD_CHUNK_SIZE): + if offset + received + len(chunk) > file.size: + # Never write past the size the free-space check allowed for. + fh.close() + partial.unlink(missing_ok=True) + raise SyncError(f'{file.path} is larger than the primary said') + await asyncio.to_thread(fh.write, chunk) + received += len(chunk) + if on_bytes: + on_bytes(len(chunk)) + + if (size := partial.stat().st_size) != file.size: + raise SyncError(f'{file.path} is {size} bytes, the primary said {file.size}') + os.utime(partial, (file.mtime, file.mtime)) + os.replace(partial, destination) + return received + + +# --------------------------------------------------------------------------------------------------------------------- +# Secondary: configs. + + +def _import_tags(): + from wrolpi.tags import import_tags_config + import_tags_config() + + +def _import_downloads(): + from wrolpi.downloader import get_download_manager_config + get_download_manager_config().import_config() + + +def _import_channels(): + from modules.videos.lib import import_channels_config + import_channels_config() + + +def _import_domains(): + from modules.archive.lib import import_domains_config + import_domains_config() + + +def _import_playlists(): + from wrolpi.collections.config import playlists_config + from wrolpi.collections.sync import sync_playlists_directory + playlists_config.import_config() + sync_playlists_directory.activate_switch() + + +def _import_zims(): + from modules.zim.lib import import_zims_config + import_zims_config() + + +def _import_plain(file_name: str): + def import_(): + get_config_by_file_name(file_name).import_config() + + return import_ + + +def _import_inventories(): + from modules.inventory.common import import_inventories_config + import_inventories_config() + + +# In the order of `import_all_db_configs`: Collections refer to Tags, and link to Downloads. +SYNCED_CONFIG_IMPORTS = ( + ('tags.yaml', _import_tags), + ('download_manager.yaml', _import_downloads), + ('channels.yaml', _import_channels), + ('domains.yaml', _import_domains), + ('playlists.yaml', _import_playlists), + ('zims.yaml', _import_zims), + ('map_pins.yaml', _import_plain('map_pins.yaml')), + ('bookmarks.yaml', _import_plain('bookmarks.yaml')), + ('videos_downloader.yaml', _import_plain('videos_downloader.yaml')), + ('archives_downloader.yaml', _import_plain('archives_downloader.yaml')), +) +INVENTORY_DIRECTORY = 'config/inventory' +assert {i for i, _ in SYNCED_CONFIG_IMPORTS} | {'inventory/catalog.yaml'} == SYNCED_CONFIG_FILE_NAMES + + +def _parse_config(data: bytes, name: str) -> dict: + config = yaml.safe_load(data) + if not isinstance(config, dict): + raise SyncError(f'{name} from the primary is not a config') + return config + + +def write_config_bytes(file: pathlib.Path, data: bytes, backup: pathlib.Path = None) -> bool: + """Replace `file` with the primary's bytes (keeping its version). Returns False if they are already the same. + + The replaced file is copied to `backup` first (today's config backup), unless today's backup exists: like a save, + the first copy of the day is the one kept, so a sync can be undone with a config backup import.""" + if file.is_file() and file.read_bytes() == data: + return False + if backup and file.is_file() and not backup.exists(): + backup.parent.mkdir(parents=True, exist_ok=True) + shutil.copy2(file, backup) + file.parent.mkdir(parents=True, exist_ok=True) + temporary = file.with_name(f'.{file.name}.sync') + temporary.write_bytes(data) + os.replace(temporary, file) + return True + + +async def sync_configs(client: PrimaryClient, primary_media_directory: str, reimport: bool) -> List[str]: + """Copy the primary's synced configs, then import those that changed (all of them when `reimport`, because newly + downloaded files may let a config import entries it skipped before). Returns the imported configs.""" + imported = list() + for file_name, import_ in SYNCED_CONFIG_IMPORTS: + data = await client.get_bytes(f'config/{file_name}') + if data is None: + continue + config = get_config_by_file_name(file_name) + _parse_config(data, file_name) + changed = write_config_bytes(config.get_file(), data, config._get_backup_filename()) + if changed or reimport: + logger.info(f'Importing {file_name} from the primary') + await asyncio.to_thread(import_) + imported.append(file_name) + + if await _sync_inventory_directory(client) or reimport: + logger.info('Importing inventories from the primary') + await asyncio.to_thread(_import_inventories) + imported.append('inventory') + + await _merge_wrolpi_config(client, primary_media_directory) + return imported + + +async def _sync_inventory_directory(client: PrimaryClient) -> bool: + from modules.inventory.common import get_inventory_configs + inventories = get_inventory_configs() + directory = inventories.get_directory() + changed = False + for entry in await client.list_directory(INVENTORY_DIRECTORY): + name = entry.get('name') or '' + if entry.get('is_dir') or not name.endswith('.yaml') or '/' in name or name.startswith('.'): + continue + data = await client.get_bytes(f'{INVENTORY_DIRECTORY}/{name}') + if data is None: + continue + _parse_config(data, name) + backup = inventories._get_backup_filename(name.removesuffix('.yaml')) + changed = write_config_bytes(directory / name, data, backup) or changed + return changed + + +def merge_wrolpi_config(primary_config: dict, primary_media_directory: str) -> dict: + """The library settings of the primary's wrolpi.yaml that differ from this WROLPi's.""" + wrolpi_config = get_wrolpi_config() + changes = dict() + for key in SYNCED_WROLPI_CONFIG_KEYS: + if key not in primary_config: + continue + value = primary_config[key] + if key == 'ignored_directories': + value = _relative_directories(value or [], primary_media_directory) + if wrolpi_config.dict().get(key) != value: + changes[key] = value + return changes + + +def _relative_directories(directories: List[str], primary_media_directory: str) -> List[str]: + relative = list() + for directory in directories: + path = pathlib.PurePosixPath(directory) + if path.is_absolute(): + try: + path = path.relative_to(primary_media_directory) + except ValueError: + # Outside the primary's media directory; meaningless here. + continue + relative.append(str(path)) + return relative + + +async def _merge_wrolpi_config(client: PrimaryClient, primary_media_directory: str): + data = await client.get_bytes('config/wrolpi.yaml') + if data is None: + return + if changes := merge_wrolpi_config(_parse_config(data, 'wrolpi.yaml'), primary_media_directory): + logger.info(f'Applying library settings from the primary: {sorted(changes)}') + get_wrolpi_config().update(changes) + + +# --------------------------------------------------------------------------------------------------------------------- +# Secondary: the sync. + + +def sync_skip_reason() -> Optional[str]: + """Why a sync cannot start now, or None.""" + config = get_sync_config() + if not is_secondary(): + return 'This WROLPi is not a secondary' + if wrol_mode_enabled(): + return 'WROL Mode is enabled' + if not config.primary_url: + return 'No primary is configured' + if not PYTEST: + if not flags.db_up.is_set(): + return 'The database is not up' + if not flags.refresh_complete.is_set(): + return 'The first refresh has not finished' + if get_wrolpi_config().require_media_mounted and not flags.media_mounted.is_set(): + return 'The media directory is not mounted' + if flags.sync_running.is_set(): + return 'A sync is already running' + return None + + +def request_sync(force: bool = False): + update_sync_status(requested=True, force=force) + + +def request_sync_cancel(): + """Stop a running sync between pages, and forget a requested one.""" + from wrolpi.api_utils import api_app + if 'sync' not in api_app.shared_ctx.status: + return + status = get_sync_status() + if status['state'] == 'running' or status['requested']: + update_sync_status(cancel_requested=status['state'] == 'running', requested=False) + + +def _check_cancel(): + if get_sync_status()['cancel_requested']: + raise SyncCancelled('The sync was cancelled') + if wrol_mode_enabled(): + raise SyncCancelled('WROL Mode was enabled') + + +@contextlib.contextmanager +def lock_api(): + """Refuse API requests (see `check_sync_api_lock`) while the library is replaced.""" + from wrolpi.api_utils import api_app + with flags.sync_running: + api_app.shared_ctx.sync_api_locked.set() + try: + yield + finally: + api_app.shared_ctx.sync_api_locked.clear() + + +async def run_sync(force: bool = False) -> dict: + """Copy the primary's library to this secondary. Returns the final status.""" + if reason := sync_skip_reason(): + logger.info(f'Not syncing: {reason}') + return update_sync_status(last_skip_reason=reason, force=False) + + url = get_sync_config().primary_url + started_at = now() + update_sync_status( + state='running', phase='files', force=False, cancel_requested=False, started_at=started_at.isoformat(), + finished_at=None, last_error=None, last_skip_reason=None, primary_url=url, primary_version=None, + groups_seen=0, max_id=0, files_checked=0, files_to_download=0, files_downloaded=0, bytes_downloaded=0, + files_failed=0, files_excluded=0, configs_imported=[], disk_full=False, + ) + logger.warning(f'Syncing from the primary {url}') + try: + with lock_api(): + async with PrimaryClient(url) as client: + info = await client.get_info() + if info.get('role') != 'primary': + raise SyncError(f'{url} is not a primary WROLPi') + if (info.get('version') or '').strip() != __version__.strip(): + logger.warning(f'The primary runs WROLPi {info.get("version")}, this is {__version__.strip()}') + update_sync_status(primary_version=(info.get('version') or '').strip() or None) + + downloaded = await _sync_files(client) + _check_cancel() + + update_sync_status(phase='configs') + primary_media_directory = info.get('media_directory') or '' + imported = await sync_configs(client, primary_media_directory, reimport=bool(downloaded) or force) + except SyncCancelled as e: + logger.warning(f'Sync stopped: {e}') + return update_sync_status(state='idle', phase=None, finished_at=now().isoformat(), last_error=str(e), + cancel_requested=False) + except Exception as e: + logger.error('Sync failed', exc_info=e) + Events.send_sync_failed(f'Sync from the primary failed: {e}') + return update_sync_status(state='error', phase=None, finished_at=now().isoformat(), last_error=str(e), + cancel_requested=False) + + status = get_sync_status() + finished_at = now() + failed = status['files_failed'] + message = f'Synced {len(downloaded)} files from the primary' + if failed: + message += f'; {failed} failed' + if status['disk_full']: + message += '; the drive is full' + logger.warning(f'{message} in {(finished_at - started_at).total_seconds():.0f}s') + if downloaded or failed or imported: + Events.send_sync_completed(message) + last_success_at = finished_at.isoformat() if not failed and not status['disk_full'] else status['last_success_at'] + return update_sync_status(state='idle', phase=None, finished_at=finished_at.isoformat(), + last_success_at=last_success_at, configs_imported=imported, + last_error=f'{failed} files failed' if failed else None) + + +async def _sync_files(client: PrimaryClient) -> List[pathlib.Path]: + """Download the files this secondary lacks, and refresh them a page at a time. Returns the files downloaded.""" + media_directory = get_media_directory() + downloaded = list() + groups_seen = files_checked = files_to_download = files_failed = files_excluded = 0 + excluded_directories = set(get_sync_config().sync_excluded_directories) + bytes_downloaded = 0 + disk_full = False + + last_report = 0.0 + + def report(force: bool = False): + nonlocal last_report + if force or time.monotonic() - last_report >= PROGRESS_INTERVAL_SECONDS: + last_report = time.monotonic() + update_sync_status(groups_seen=groups_seen, files_checked=files_checked, + files_to_download=files_to_download, files_downloaded=len(downloaded), + bytes_downloaded=bytes_downloaded, files_failed=files_failed, + files_excluded=files_excluded, disk_full=disk_full) + + def count_bytes(n: int): + nonlocal bytes_downloaded + bytes_downloaded += n + report() + + async for page in client.iter_manifest(): + _check_cancel() + wanted = list() + for group in page['groups']: + groups_seen += 1 + for entry in group['paths']: + files_checked += 1 + file = ManifestFile(entry['path'], int(entry['size']), float(entry['mtime'])) + if excluded_directories and is_excluded(file.path, excluded_directories): + files_excluded += 1 + continue + destination = resolve_destination(media_directory, file.path) + if destination is None: + logger.warning(f'Refusing to sync {repr(file.path)}') + files_failed += 1 + continue + if needs_download(destination, file): + wanted.append((file, destination)) + files_to_download += len(wanted) + update_sync_status(max_id=page.get('max_id') or 0) + report(force=True) + + page_downloaded = list() + if wanted and not disk_full: + semaphore = asyncio.Semaphore(DOWNLOAD_CONCURRENCY) + + async def download(file_: ManifestFile, destination_: pathlib.Path): + nonlocal disk_full + async with semaphore: + if disk_full: + return None + try: + await download_file(client, file_, destination_, count_bytes) + downloaded.append(destination_) + return destination_ + except SyncDiskFull as e: + disk_full = True + logger.error(str(e)) + except Exception as e: + logger.error(f'Failed to sync {repr(file_.path)}: {e}') + return False + + results = await asyncio.gather(*(download(f, d) for f, d in wanted)) + page_downloaded = [i for i in results if i] + files_failed += sum(1 for i in results if i is False) + + update_sync_status(max_id=page.get('max_id') or 0) + report(force=True) + + if page_downloaded: + await _refresh(page_downloaded) + + return downloaded + + +async def _refresh(paths: List[pathlib.Path]): + from wrolpi.files.worker import file_worker, FileWorkerJobFailed + job_id = file_worker.queue_refresh(paths, send_events=False) + try: + await file_worker.wait_for_job(job_id, timeout=REFRESH_TIMEOUT_SECONDS) + except (FileWorkerJobFailed, TimeoutError) as e: + # The files are in place; a later refresh indexes them. + logger.error(f'Failed to refresh synced files: {e}') + + +def window_is_due(status: dict, local_now: datetime, start: Optional[str], end: Optional[str], + repeat_seconds: int) -> bool: + """True if the daily window from `start` to `end` is open and the last run (described by `status`) finished + `repeat_seconds` ago. No window: never (only on request).""" + if not start or not end: + return False + if not is_within_daily_window(start, end, local_now.time()): + return False + if not status.get('finished_at'): + return True + finished_at = datetime.fromisoformat(status['finished_at']) + return (local_now - finished_at).total_seconds() >= repeat_seconds + + +def sync_is_due(status: dict, local_now: datetime) -> bool: + """True if the sync window is open and the last sync finished long enough ago.""" + config = get_sync_config() + return window_is_due(status, local_now, config.window_start, config.window_end, SYNC_WINDOW_REPEAT_SECONDS) + + +async def sync_tick(): + """Start a sync when one was requested, or when it is due.""" + if not is_secondary(): + return + status = get_sync_status() + if wrol_mode_enabled(): + if status['requested']: + update_sync_status(requested=False, last_skip_reason='WROL Mode is enabled') + return + from wrolpi.downloader import DownloadManager + if not status['requested'] and not sync_is_due(status, DownloadManager._get_local_now()): + return + force = status['force'] + update_sync_status(requested=False) + await run_sync(force=force) diff --git a/wrolpi/sync/schema.py b/wrolpi/sync/schema.py new file mode 100644 index 00000000..1332bc0a --- /dev/null +++ b/wrolpi/sync/schema.py @@ -0,0 +1,56 @@ +from dataclasses import dataclass +from typing import List, Optional + +from wrolpi.errors import ValidationError + + +@dataclass +class ManifestRequest: + after_id: int = 0 + limit: int = 500 + + def __post_init__(self): + self.after_id = int(self.after_id) + self.limit = int(self.limit) + if self.after_id < 0: + raise ValidationError('after_id must not be negative') + if not 1 <= self.limit <= 5000: + raise ValidationError('limit must be between 1 and 5000') + + +@dataclass +class SyncRunRequest: + # Import every synced config even if the primary's copy did not change. + force: bool = False + + +@dataclass +class SyncDirectoriesRequest: + # The folder whose sub-folders are listed, relative to the media directory ('' is the media directory). + path: str = '' + # 'local' lists this WROLPi's media directory, 'primary' lists its primary's. + source: str = 'local' + + def __post_init__(self): + if self.source not in ('local', 'primary'): + raise ValidationError("source must be 'local' or 'primary'") + + +@dataclass +class SyncSettingsRequest: + role: Optional[str] = None + sync_primary_url: Optional[str] = None + sync_window_start: Optional[str] = None + sync_window_end: Optional[str] = None + # The mount point of the backup drive, and the UUID of the drive that must be mounted there ('' clears both). + backup_directory: Optional[str] = None + backup_device_uuid: Optional[str] = None + backup_window_start: Optional[str] = None + backup_window_end: Optional[str] = None + # Folders, relative to the media directory, never copied ([] clears). + sync_excluded_directories: Optional[List[str]] = None + backup_excluded_directories: Optional[List[str]] = None + + def __post_init__(self): + if self.role is not None and self.role not in ('primary', 'secondary'): + raise ValidationError('role must be "primary" or "secondary"') diff --git a/wrolpi/sync/status.py b/wrolpi/sync/status.py new file mode 100644 index 00000000..845b8e62 --- /dev/null +++ b/wrolpi/sync/status.py @@ -0,0 +1,81 @@ +"""The shared progress records of a secondary WROLPi's sync, `api_app.shared_ctx.status['sync']`, and of a backup to +a second drive, `api_app.shared_ctx.status['backup']`. + +They live in a manager dict, so every read returns a copy: change them only with `update_sync_status` and +`update_backup_status`. +""" + + +def default_sync_status() -> dict: + return dict( + state='idle', + phase=None, + requested=False, + force=False, + cancel_requested=False, + started_at=None, + finished_at=None, + last_success_at=None, + last_error=None, + last_skip_reason=None, + primary_url=None, + primary_version=None, + groups_seen=0, + max_id=0, + files_checked=0, + files_to_download=0, + files_downloaded=0, + bytes_downloaded=0, + files_failed=0, + files_excluded=0, + configs_imported=[], + disk_full=False, + ) + + +def get_sync_status() -> dict: + from wrolpi.api_utils import api_app + return dict(api_app.shared_ctx.status.get('sync') or default_sync_status()) + + +def update_sync_status(**changes) -> dict: + """Merge `changes` into the shared sync status. Only the perpetual process and the sync endpoints write it, and + the endpoints only set the request/cancel keys, so a lost update can only delay a request by one tick.""" + from wrolpi.api_utils import api_app + status = get_sync_status() + status.update(changes) + api_app.shared_ctx.status['sync'] = status + return status + + +def default_backup_status() -> dict: + return dict( + state='idle', + phase=None, + requested=False, + cancel_requested=False, + started_at=None, + finished_at=None, + last_success_at=None, + last_error=None, + last_skip_reason=None, + directory=None, + bytes_transferred=0, + percent=0, + files_transferred=0, + files_to_check=0, + ) + + +def get_backup_status() -> dict: + from wrolpi.api_utils import api_app + return dict(api_app.shared_ctx.status.get('backup') or default_backup_status()) + + +def update_backup_status(**changes) -> dict: + """Merge `changes` into the shared backup status (see `update_sync_status`).""" + from wrolpi.api_utils import api_app + status = get_backup_status() + status.update(changes) + api_app.shared_ctx.status['backup'] = status + return status diff --git a/wrolpi/sync/test/__init__.py b/wrolpi/sync/test/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/wrolpi/sync/test/test_api.py b/wrolpi/sync/test/test_api.py new file mode 100644 index 00000000..f895ef42 --- /dev/null +++ b/wrolpi/sync/test/test_api.py @@ -0,0 +1,216 @@ +import json +from http import HTTPStatus + +import pytest +import yaml + +from wrolpi import flags +from wrolpi.api_utils import api_app +from wrolpi.common import get_wrolpi_config, enable_wrol_mode +from wrolpi.downloader import download_manager +from wrolpi.sync import lib +from wrolpi.sync.config import get_sync_config +from wrolpi.sync.status import get_sync_status, update_sync_status + + +@pytest.mark.asyncio +async def test_info_and_manifest(async_client, test_session, test_wrolpi_config, make_files_structure, + refresh_files): + make_files_structure({'docs/a.txt': 'a', 'docs/b.txt': 'b'}) + await refresh_files() + + request, response = await async_client.get('/api/sync/info') + assert response.status_code == HTTPStatus.OK + assert response.json['role'] == 'primary' + assert response.json['max_file_group_id'] > 0 + + request, response = await async_client.get('/api/sync/manifest?after_id=0&limit=1') + assert response.status_code == HTTPStatus.OK + assert len(response.json['groups']) == 1 + after_id = response.json['next_after_id'] + assert after_id + + request, response = await async_client.get(f'/api/sync/manifest?after_id={after_id}&limit=1') + paths = [p['path'] for g in response.json['groups'] for p in g['paths']] + assert len(paths) == 1 and paths[0].startswith('docs/') + + request, response = await async_client.get('/api/sync/manifest?limit=0') + assert response.status_code == HTTPStatus.BAD_REQUEST + + +@pytest.mark.asyncio +async def test_run(async_client, test_wrolpi_config, make_secondary): + request, response = await async_client.post('/api/sync/run', content='{}') + assert response.status_code == HTTPStatus.FORBIDDEN, 'a primary does not sync' + + make_secondary('https://primary.example') + request, response = await async_client.post('/api/sync/run', content=json.dumps(dict(force=True))) + assert response.status_code == HTTPStatus.ACCEPTED + assert get_sync_status()['requested'] is True + assert get_sync_status()['force'] is True + + request, response = await async_client.get('/api/sync/status') + assert response.json['sync']['requested'] is True + + +@pytest.mark.asyncio +async def test_settings(async_client, test_directory, test_wrolpi_config, test_download_manager, await_switches): + body = dict(role='secondary', sync_primary_url=' https://10.0.0.5:8443/ ', sync_window_start='01:00', + sync_window_end='05:00') + request, response = await async_client.put('/api/sync/settings', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT + config = get_sync_config() + assert config.role == 'secondary' + assert config.primary_url == 'https://10.0.0.5:8443' + assert (config.window_start, config.window_end) == ('01:00', '05:00') + # Saved to sync.yaml, not wrolpi.yaml. + await await_switches() + saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text()) + assert (saved['role'], saved['primary_url']) == ('secondary', 'https://10.0.0.5:8443') + assert 'role' not in get_wrolpi_config().dict() + assert download_manager.is_disabled, 'a secondary does not download' + + request, response = await async_client.get('/api/settings') + assert response.json['role'] == 'secondary' + assert response.json['sync_primary_url'] == 'https://10.0.0.5:8443' + + request, response = await async_client.post('/api/download/enable') + assert response.status_code == HTTPStatus.FORBIDDEN + + # The backup drive has its own window. + body = dict(backup_window_start='02:00', backup_window_end='04:00') + request, response = await async_client.put('/api/sync/settings', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT + assert (config.backup_window_start, config.backup_window_end) == ('02:00', '04:00') + assert (config.window_start, config.window_end) == ('01:00', '05:00') + request, response = await async_client.get('/api/settings') + assert (response.json['backup_window_start'], response.json['backup_window_end']) == ('02:00', '04:00') + + # Clearing the window leaves only "Sync now". + body = dict(sync_window_start='', sync_window_end='') + request, response = await async_client.put('/api/sync/settings', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT + assert config.window_start is None + + for body in (dict(role='tertiary'), dict(sync_primary_url='ftp://primary'), + dict(sync_primary_url='https://user:password@primary'), + dict(sync_primary_url='https://primary/media'), dict(sync_window_start='25:00'), + dict(backup_window_end='5pm'), dict()): + request, response = await async_client.put('/api/sync/settings', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_REQUEST, body + + +@pytest.mark.asyncio +async def test_settings_excluded_directories(async_client, test_directory, test_wrolpi_config, await_switches): + """Each panel has its own excluded folders, normalized and saved to sync.yaml.""" + body = dict(sync_excluded_directories=['/Movies/', 'Movies/deep'], backup_excluded_directories=['videos/Movies']) + request, response = await async_client.put('/api/sync/settings', content=json.dumps(body)) + assert response.status_code == HTTPStatus.NO_CONTENT + config = get_sync_config() + assert config.sync_excluded_directories == ['Movies'] + assert config.backup_excluded_directories == ['videos/Movies'] + await await_switches() + saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text()) + assert saved['sync_excluded_directories'] == ['Movies'] + assert saved['backup_excluded_directories'] == ['videos/Movies'] + + request, response = await async_client.get('/api/settings') + assert response.json['sync_excluded_directories'] == ['Movies'] + assert response.json['backup_excluded_directories'] == ['videos/Movies'] + + # Changing one list leaves the other alone; [] clears. + request, response = await async_client.put('/api/sync/settings', + content=json.dumps(dict(backup_excluded_directories=[]))) + assert response.status_code == HTTPStatus.NO_CONTENT + assert config.backup_excluded_directories == [] + assert config.sync_excluded_directories == ['Movies'] + + for value in (['../etc'], ['config'], ['.hidden'], [''], 'Movies'): + body = dict(sync_excluded_directories=value) + request, response = await async_client.put('/api/sync/settings', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_REQUEST, value + assert config.sync_excluded_directories == ['Movies'] + + +@pytest.mark.asyncio +async def test_directories(async_client, test_directory, test_wrolpi_config): + for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'): + (test_directory / i).mkdir(parents=True, exist_ok=True) + + request, response = await async_client.post('/api/sync/directories', content=json.dumps(dict(path=''))) + assert response.status_code == HTTPStatus.OK + assert response.json['directories'] == [dict(name='Movies', path='Movies'), dict(name='videos', path='videos')] + + body = dict(path='videos', source='local') + request, response = await async_client.post('/api/sync/directories', content=json.dumps(body)) + assert response.json['directories'] == [dict(name='channel', path='videos/channel')] + + for body in (dict(path='../'), dict(path='missing'), dict(source='elsewhere'), + dict(source='primary')): # A primary has no primary. + request, response = await async_client.post('/api/sync/directories', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_REQUEST, body + + +@pytest.mark.asyncio +async def test_directories_primary_unreachable(async_client, test_wrolpi_config, make_secondary): + """A primary that does not answer is reported, not raised.""" + make_secondary('https://127.0.0.1:1') + body = dict(path='', source='primary') + request, response = await async_client.post('/api/sync/directories', content=json.dumps(body)) + assert response.status_code == HTTPStatus.BAD_GATEWAY + assert response.json['error'] + + +@pytest.mark.asyncio +async def test_api_locked_while_syncing(async_client, test_wrolpi_config, flags_lock, make_secondary): + """While a secondary syncs, its API refuses everything but the sync's own progress.""" + make_secondary() + with lib.lock_api(): + request, response = await async_client.get('/api/files/search?search_str=foo') + assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE + assert response.json['code'] == 'SYNCING' + request, response = await async_client.post('/api/tag', content=json.dumps(dict(name='one'))) + assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE + + for path in ('/api/status', '/api/sync/status', '/api/settings', '/api/echo'): + request, response = await async_client.get(path) + assert response.status_code == HTTPStatus.OK, path + request, response = await async_client.post('/api/sync/cancel') + assert response.status_code == HTTPStatus.ACCEPTED + # Nothing may change the sync while it runs, except stopping it. + request, response = await async_client.put('/api/sync/settings', content=json.dumps(dict(role='primary'))) + assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE + request, response = await async_client.post('/api/sync/run', content='{}') + assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE + + request, response = await async_client.get('/api/status') + assert response.json['flags']['sync_running'] is True + + request, response = await async_client.get('/api/settings') + assert response.status_code == HTTPStatus.OK + assert not api_app.shared_ctx.sync_api_locked.is_set() + assert not flags.sync_running.is_set() + + +@pytest.mark.asyncio +async def test_stale_lock_event_does_not_lock(async_client, test_wrolpi_config, flags_lock): + """The Event outlives a perpetual process that died mid-sync; the cleared Flag unlocks the API.""" + api_app.shared_ctx.sync_api_locked.set() + try: + request, response = await async_client.get('/api/settings') + assert response.status_code == HTTPStatus.OK + finally: + api_app.shared_ctx.sync_api_locked.clear() + + +@pytest.mark.asyncio +async def test_wrol_mode_cancels_sync(async_client, test_wrolpi_config, test_download_manager, make_secondary): + """WROL Mode means the secondary is in use: a running sync stops, and none start.""" + make_secondary() + update_sync_status(state='running') + enable_wrol_mode() + assert get_sync_status()['cancel_requested'] is True + assert lib.sync_skip_reason() == 'WROL Mode is enabled' + + request, response = await async_client.post('/api/sync/run', content='{}') + assert response.status_code == HTTPStatus.FORBIDDEN diff --git a/wrolpi/sync/test/test_backup.py b/wrolpi/sync/test/test_backup.py new file mode 100644 index 00000000..ba75143a --- /dev/null +++ b/wrolpi/sync/test/test_backup.py @@ -0,0 +1,467 @@ +import contextlib +import os +import pathlib +import re +import shutil +import sqlite3 +import subprocess +import tempfile +from datetime import datetime, timezone + +import pytest +import yaml + +from wrolpi.common import get_wrolpi_config +from wrolpi.sync import backup +from wrolpi.sync.config import get_sync_config +from wrolpi.sync.status import get_backup_status, update_backup_status + + +def _rsync_supports_progress2() -> bool: + """rsync 3.1+ (Raspberry Pi OS, Debian); macOS ships openrsync, which has no `--info=progress2`.""" + if not shutil.which('rsync'): + return False + output = subprocess.run(['rsync', '--version'], capture_output=True, text=True).stdout + match = re.search(r'rsync\s+version\s+(\d+)\.(\d+)', output) + return bool(match) and (int(match.group(1)), int(match.group(2))) >= (3, 1) + + +requires_rsync = pytest.mark.skipif(not _rsync_supports_progress2(), reason='rsync 3.1 or newer is not installed') + + +@pytest.fixture +def backup_drive(test_directory, test_wrolpi_config, monkeypatch): + """A temporary directory that the target checks accept as the mounted backup drive `uuid-1`.""" + with tempfile.TemporaryDirectory() as tmp_dir: + drive = pathlib.Path(tmp_dir).resolve() + monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: None) + get_sync_config()._config.update(backup_directory=str(drive), backup_device_uuid='uuid-1') + yield drive + + +def test_build_rsync_command(monkeypatch): + monkeypatch.setattr(backup.shutil, 'which', lambda name: f'/usr/bin/{name}') + command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), 'playlists') + assert command[:5] == ['/usr/bin/ionice', '-c3', '/usr/bin/nice', '-n', '19'] + assert command[-2:] == ['/media/wrolpi/', '/media/backup/'], 'trailing slashes copy the contents, not the folder' + assert not any(i.startswith('--delete') for i in command), 'a backup never deletes' + assert '-H' not in command and '--hard-links' not in command + for exclude in ('/config/wrolpi.db', '/config/wrolpi.db-*', '/tags/', '/playlists/', '.*.partial', + '.rsync-partial/', '/lost+found/'): + assert f'--exclude={exclude}' in command, exclude + + # ionice and nice are optional. + monkeypatch.setattr(backup.shutil, 'which', lambda name: '/usr/bin/rsync' if name == 'rsync' else None) + command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), None) + assert command[0] == '/usr/bin/rsync' + assert not any(i.startswith('--exclude=/playlists') for i in command) + + +def test_build_rsync_command_excluded_directories(monkeypatch): + """Excluded folders are anchored at the media directory and matched literally.""" + monkeypatch.setattr(backup.shutil, 'which', lambda name: '/usr/bin/rsync' if name == 'rsync' else None) + command = backup.build_rsync_command(pathlib.Path('/media/wrolpi'), pathlib.Path('/media/backup'), None, + ['Movies', 'videos/Movies [HD]', 'a*b?']) + assert '--exclude=/Movies/' in command + assert '--exclude=/videos/Movies \\[HD]/' in command + assert '--exclude=/a\\*b\\?/' in command + assert command[-2:] == ['/media/wrolpi/', '/media/backup/'], 'excludes come before the paths' + + +@pytest.mark.parametrize('line, expected', [ + (' 32,768 0% 0.00kB/s 0:00:00 (xfr#1, ir-chk=1018/1023)', + dict(bytes_transferred=32768, percent=0, files_transferred=1, files_to_check=1023)), + (' 1234567890 45% 10.50MB/s 0:01:23 (xfr#12, to-chk=100/2000)', + dict(bytes_transferred=1234567890, percent=45, files_transferred=12, files_to_check=2000)), + (' 1234567890 45% 10.50MB/s 0:01:23', dict(bytes_transferred=1234567890, percent=45)), + ('Number of regular files transferred: 1,234', dict(files_transferred=1234)), + ('sending incremental file list', None), + ('', None), +]) +def test_parse_progress(line, expected): + assert backup.parse_progress(line) == expected + + +@pytest.fixture +def fake_mounts(test_directory, monkeypatch, tmp_path): + """Pretend `/backup` is a mounted drive with UUID `uuid-1` (device /dev/sdb1). The mount root is a + temporary directory standing in for /media, so folders can be made on the "drive".""" + mount_root = (tmp_path / 'media').resolve() + drive = mount_root / 'backup' + drive.mkdir(parents=True) + media = test_directory.resolve() + monkeypatch.setattr(backup, 'MOUNT_ROOT', mount_root) + + state = dict(mounted=True, device=2, uuid_device='/dev/sdb1', source='/dev/sdb1', nested=None) + monkeypatch.setattr(backup, 'is_mount', lambda path: (path == drive and state['mounted']) + or path == state['nested'] or path == pathlib.Path('/')) + monkeypatch.setattr(backup, 'device_id', lambda path: {drive: state['device'], media: 1}.get( + pathlib.Path(path), 3 if pathlib.Path(path) == state['nested'] else 0)) + monkeypatch.setattr(backup, 'uuid_device', lambda uuid: state['uuid_device'] if uuid == 'uuid-1' else None) + monkeypatch.setattr(backup, 'mount_source', lambda path: {str(drive): state['source']}.get( + path, '/dev/sdz1' if path == str(state['nested']) else None)) + state['drive'] = drive + state['root'] = mount_root + return state + + +def test_check_backup_target(test_directory, fake_mounts): + drive, root = str(fake_mounts['drive']), fake_mounts['root'] + assert backup.check_backup_target(drive, 'uuid-1') is None + + assert backup.check_backup_target(None, None) == 'No backup drive is configured' + assert backup.check_backup_target('backup', 'uuid-1') == 'backup is not a valid directory' + assert backup.check_backup_target(f'{drive}/../backup', 'uuid-1') == f'{drive}/../backup is not a valid directory' + assert backup.check_backup_target('/opt/backup', 'uuid-1') == f'/opt/backup is not under {root}' + (root / 'wrolpi_temp_onboarding').mkdir() + fake_mounts['nested'] = root / 'wrolpi_temp_onboarding' + assert backup.check_backup_target(str(root / 'wrolpi_temp_onboarding'), 'uuid-1') \ + == f'{root}/wrolpi_temp_onboarding cannot be a backup drive' + fake_mounts['nested'] = None + + fake_mounts['mounted'] = False + assert backup.check_backup_target(drive, 'uuid-1') == f'No drive is mounted at {drive}', \ + 'an unplugged drive leaves a directory on the system drive' + fake_mounts['mounted'] = True + + fake_mounts['device'] = 1 + assert backup.check_backup_target(drive, 'uuid-1') == 'The backup drive is the media drive' + fake_mounts['device'] = 0 + assert backup.check_backup_target(drive, 'uuid-1') == 'The backup drive is the system drive' + fake_mounts['device'] = 2 + + fake_mounts['source'] = '/dev/sdc1' + assert backup.check_backup_target(drive, 'uuid-1') \ + == f'The chosen drive is not mounted at {drive}', 'another drive mounted at the same place' + fake_mounts['source'] = '/dev/sdb1' + assert backup.check_backup_target(drive, 'uuid-2') \ + == f'The chosen drive is not mounted at {drive}', 'the drive is not plugged in' + + +def test_check_backup_target_folder(test_directory, fake_mounts): + """The backup may go to a folder on the chosen drive; the drive is checked through the folder's mount point.""" + drive = fake_mounts['drive'] + (drive / 'wrolpi').mkdir() + assert backup.check_backup_target(str(drive / 'wrolpi'), 'uuid-1') is None + assert backup.check_backup_target(str(drive / 'not' / 'made' / 'yet'), 'uuid-1') is None, \ + 'the backup creates the folder' + + # The folder is checked through the drive: wrong drive, unplugged drive. + assert backup.check_backup_target(str(drive / 'wrolpi'), 'uuid-2') \ + == f'The chosen drive is not mounted at {drive}' + fake_mounts['mounted'] = False + assert backup.check_backup_target(str(drive / 'wrolpi'), 'uuid-1') == f'No drive is mounted at {drive}/wrolpi', \ + 'a folder left on the system drive by an unplugged drive is not written' + fake_mounts['mounted'] = True + + # Another drive mounted inside the chosen drive is not the chosen drive. + (drive / 'other').mkdir() + fake_mounts['nested'] = drive / 'other' + assert backup.check_backup_target(str(drive / 'other' / 'wrolpi'), 'uuid-1') \ + == f'The chosen drive is not mounted at {drive}/other' + + +def test_mount_source(monkeypatch, tmp_path): + mounts = tmp_path / 'mounts' + mounts.write_text('/dev/sda1 /media/wrolpi ext4 rw 0 0\n' + '/dev/sdb1 /media/my\\040backup ext4 rw 0 0\n' + '/dev/sdc1 /media/stacked ext4 rw 0 0\n' + '/dev/sdd1 /media/stacked ext4 rw 0 0\n') + monkeypatch.setattr(backup, 'PROC_MOUNTS', mounts) + assert backup.mount_source('/media/my backup') == '/dev/sdb1' + assert backup.mount_source('/media/stacked') == '/dev/sdd1', 'the last mount is the visible one' + assert backup.mount_source('/media/missing') is None + + +def test_snapshot_database(test_directory, tmp_path): + """The snapshot is a complete, consistent database, even with writes still in the live database's WAL.""" + (test_directory / 'config').mkdir(exist_ok=True) + live = sqlite3.connect(test_directory / 'config/wrolpi.db') + live.execute('PRAGMA journal_mode=WAL') + live.execute('CREATE TABLE t (v TEXT)') + live.execute("INSERT INTO t VALUES ('in the WAL')") + live.commit() + assert (test_directory / 'config/wrolpi.db-wal').stat().st_size > 0 + + drive = tmp_path / 'drive' + (drive / 'config').mkdir(parents=True) + # A stale WAL from an older copy must not be applied to the new snapshot. + (drive / 'config/wrolpi.db-wal').write_bytes(b'stale') + try: + backup.snapshot_database(drive) + finally: + live.close() + + assert not (drive / 'config/wrolpi.db-wal').exists() + assert not (drive / 'config/.wrolpi.db.backup').exists() + with sqlite3.connect(drive / 'config/wrolpi.db') as copy: + assert copy.execute('SELECT v FROM t').fetchall() == [('in the WAL',)] + + +def test_snapshot_database_only_when_changed(async_client, test_directory, tmp_path): + """The 2.5 GB snapshot is written again only when the live database, or the drive's copy, has changed.""" + (test_directory / 'config').mkdir(exist_ok=True) + database = test_directory / 'config/wrolpi.db' + with contextlib.closing(sqlite3.connect(database)) as live: + live.execute('PRAGMA journal_mode=WAL') + live.execute('CREATE TABLE t (v TEXT)') + live.commit() + drive = tmp_path / 'drive' + copy = drive / 'config/wrolpi.db' + + assert backup.snapshot_database_if_changed(drive) is True + written = copy.stat().st_mtime_ns + assert backup.snapshot_database_if_changed(drive) is False + assert copy.stat().st_mtime_ns == written + + # Reading opens an empty WAL; nothing changed. + with contextlib.closing(sqlite3.connect(database)) as reader: + reader.execute('SELECT * FROM t').fetchall() + assert backup.snapshot_database_if_changed(drive) is False + + # A write still in the WAL is a change. + with contextlib.closing(sqlite3.connect(database)) as live: + live.execute("INSERT INTO t VALUES ('new')") + live.commit() + assert (test_directory / 'config/wrolpi.db-wal').stat().st_size > 0 + assert backup.snapshot_database_if_changed(drive) is True + with contextlib.closing(sqlite3.connect(copy)) as db: + assert db.execute('SELECT v FROM t').fetchall() == [('new',)] + + # The live database changed when the WAL was checkpointed on close. + backup.snapshot_database_if_changed(drive) + assert backup.snapshot_database_if_changed(drive) is False + + # The drive's copy is missing, or replaced by something else. + copy.unlink() + assert backup.snapshot_database_if_changed(drive) is True + copy.write_bytes(b'not the snapshot') + assert backup.snapshot_database_if_changed(drive) is True + # Another drive has no snapshot yet. + assert backup.snapshot_database_if_changed(tmp_path / 'other') is True + + +@requires_rsync +@pytest.mark.asyncio +async def test_run_backup(async_client, test_directory, backup_drive, events_fixture): + """Files and configs are copied with their modification times; the database is a snapshot; hard-link directories + are not copied; nothing on the backup drive is deleted; a second backup copies nothing.""" + files = { + 'videos/channel/video.mp4': b'video', + 'videos/channel/video.info.json': b'{}', + 'config/tags.yaml': b'tags: {}\n', + 'tags/one/video.mp4': b'video', + 'playlists/0001_video.mp4': b'video', + 'videos/channel/.video.mp4.partial': b'vid', + } + for path, content in files.items(): + (test_directory / path).parent.mkdir(parents=True, exist_ok=True) + (test_directory / path).write_bytes(content) + os.utime(test_directory / path, (1_700_000_000, 1_700_000_000)) + with sqlite3.connect(test_directory / 'config/wrolpi.db') as db: + db.execute('CREATE TABLE t (v TEXT)') + db.execute("INSERT INTO t VALUES ('row')") + # Only on the backup drive: never deleted. + (backup_drive / 'old.txt').write_text('deleted from the media directory long ago') + + status = await backup.run_backup() + + assert status['state'] == 'idle', status['last_error'] + assert status['last_success_at'] + for path in ('videos/channel/video.mp4', 'videos/channel/video.info.json', 'config/tags.yaml'): + assert (backup_drive / path).read_bytes() == files[path] + assert (backup_drive / path).stat().st_mtime == 1_700_000_000 + assert not (backup_drive / 'tags').exists() + assert not (backup_drive / 'playlists').exists() + assert not (backup_drive / 'videos/channel/.video.mp4.partial').exists() + assert (backup_drive / 'old.txt').exists() + with sqlite3.connect(backup_drive / 'config/wrolpi.db') as copy: + assert copy.execute('SELECT v FROM t').fetchall() == [('row',)] + assert status['files_transferred'] == 3 + assert status['cancel_requested'] is False + events_fixture.assert_has_event('backup_completed') + + status = await backup.run_backup() + assert status['state'] == 'idle', status['last_error'] + assert status['files_transferred'] == 0 + + +@requires_rsync +@pytest.mark.asyncio +async def test_run_backup_excluded_directories(async_client, test_directory, backup_drive): + """Excluded folders are not copied, even when their names hold rsync wildcards; what the drive already has in + them is kept.""" + files = ['Movies/a.mp4', 'Movies [HD]/b.mp4', 'Movies2/c.mp4', 'videos/Movies/d.mp4', 'Mo*/e.mp4'] + for path in files: + (test_directory / path).parent.mkdir(parents=True, exist_ok=True) + (test_directory / path).write_bytes(b'file') + (test_directory / 'config').mkdir(exist_ok=True) + with sqlite3.connect(test_directory / 'config/wrolpi.db') as db: + db.execute('CREATE TABLE t (v TEXT)') + (backup_drive / 'Movies').mkdir() + (backup_drive / 'Movies/old.mp4').write_bytes(b'old') + get_sync_config()._config['backup_excluded_directories'] = ['Movies', 'Movies [HD]', 'Mo*'] + + status = await backup.run_backup() + + assert status['state'] == 'idle', status['last_error'] + assert not (backup_drive / 'Movies/a.mp4').exists() + assert not (backup_drive / 'Movies [HD]').exists() + assert not (backup_drive / 'Mo*').exists() + assert (backup_drive / 'Movies2/c.mp4').read_bytes() == b'file', 'only the folder itself is excluded' + assert (backup_drive / 'videos/Movies/d.mp4').read_bytes() == b'file', 'excludes are anchored' + assert (backup_drive / 'Movies/old.mp4').read_bytes() == b'old', 'nothing is deleted' + + +@requires_rsync +@pytest.mark.asyncio +async def test_run_backup_to_folder(async_client, test_directory, backup_drive): + """A backup to a folder on the drive creates the folder and mirrors the media directory inside it.""" + (test_directory / 'videos').mkdir() + (test_directory / 'config').mkdir(exist_ok=True) + (test_directory / 'videos/video.mp4').write_bytes(b'video') + with sqlite3.connect(test_directory / 'config/wrolpi.db') as db: + db.execute('CREATE TABLE t (v TEXT)') + get_sync_config()._config.update(backup_directory=str(backup_drive / 'wrolpi' / 'library')) + + status = await backup.run_backup() + + assert status['state'] == 'idle', status['last_error'] + assert (backup_drive / 'wrolpi/library/videos/video.mp4').read_bytes() == b'video' + assert (backup_drive / 'wrolpi/library/config/wrolpi.db').is_file() + assert not (backup_drive / 'videos').exists() + + +@requires_rsync +@pytest.mark.asyncio +async def test_run_backup_stops_when_drive_disappears(async_client, test_directory, backup_drive, monkeypatch): + """A drive that is unplugged during a backup is not written to again: rsync is stopped.""" + (test_directory / 'a.txt').write_text('a') + monkeypatch.setattr(backup, 'MOUNT_CHECK_SECONDS', 0) + monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: 'No drive is mounted at /media/x') + monkeypatch.setattr(backup, 'backup_skip_reason', lambda: None) + + status = await backup.run_backup() + + assert status['state'] == 'error' + assert 'No drive is mounted' in status['last_error'] + + +@pytest.mark.asyncio +async def test_run_backup_skips(async_client, test_wrolpi_config): + status = await backup.run_backup() + assert status['last_skip_reason'] == 'No backup drive is configured' + assert status['started_at'] is None + + get_sync_config()._config.update(backup_directory='/media/backup', backup_device_uuid='uuid-1') + get_wrolpi_config()._config.update(wrol_mode=True) + status = await backup.run_backup() + assert status['last_skip_reason'] == 'WROL Mode is enabled' + + +@pytest.mark.asyncio +async def test_backup_tick(async_client, test_wrolpi_config, monkeypatch): + calls = list() + + async def fake_run_backup(): + calls.append(True) + + monkeypatch.setattr(backup, 'run_backup', fake_run_backup) + + backup.request_backup() + await backup.backup_tick() + assert calls == [], 'no backup drive' + assert get_backup_status()['last_skip_reason'] == 'No backup drive is configured' + + get_sync_config()._config.update(backup_directory='/media/backup', backup_device_uuid='uuid-1') + await backup.backup_tick() + assert calls == [], 'nothing requested and no window' + + backup.request_backup() + await backup.backup_tick() + assert calls == [True] + assert get_backup_status()['requested'] is False + + get_wrolpi_config()._config.update(wrol_mode=True) + backup.request_backup() + await backup.backup_tick() + assert calls == [True] + assert get_backup_status()['last_skip_reason'] == 'WROL Mode is enabled' + + +def test_backup_is_due_hourly_in_window(test_wrolpi_config): + """Within the backup window, a backup repeats an hour after the last one finished.""" + get_sync_config()._config.update(backup_window_start='01:00', backup_window_end='05:00') + local_now = datetime(2026, 11, 5, 3, 0, tzinfo=timezone.utc) + never = dict(finished_at=None) + recent = dict(finished_at=local_now.replace(hour=2, minute=30).isoformat()) + earlier = dict(finished_at=local_now.replace(hour=1, minute=55).isoformat()) + assert backup.backup_is_due(never, local_now) + assert not backup.backup_is_due(recent, local_now) + assert backup.backup_is_due(earlier, local_now) + assert not backup.backup_is_due(never, local_now.replace(hour=6)), 'outside the window' + + +def test_backup_and_sync_windows_are_separate(test_wrolpi_config): + """The sync window (a secondary's sync) does not start backups, and the backup window does not start syncs.""" + from wrolpi.sync.lib import sync_is_due + local_now = datetime(2026, 11, 5, 3, 0, tzinfo=timezone.utc) + never = dict(finished_at=None) + + get_sync_config()._config.update(window_start='01:00', window_end='05:00') + assert sync_is_due(never, local_now) + assert not backup.backup_is_due(never, local_now) + + get_sync_config()._config.update(window_start=None, window_end=None, + backup_window_start='02:00', backup_window_end='04:00') + assert not sync_is_due(never, local_now) + assert backup.backup_is_due(never, local_now) + + +def test_request_backup_cancel(async_client): + update_backup_status(state='running') + backup.request_backup_cancel() + assert get_backup_status()['cancel_requested'] is True + + +@pytest.mark.asyncio +async def test_last_success_is_kept(async_client, test_directory, test_wrolpi_config, monkeypatch, await_switches): + """When the last backup succeeded is saved in sync.yaml, and known again after a restart (its status is in + memory).""" + async def fake_rsync(command, target, uuid): + return 0, [] + + monkeypatch.setattr(backup, 'backup_skip_reason', lambda: None) + monkeypatch.setattr(backup, 'check_backup_target', lambda directory, uuid: None) + monkeypatch.setattr(backup, '_run_rsync', fake_rsync) + monkeypatch.setattr(backup, 'snapshot_database', lambda target: None) + drive = test_directory / 'drive' + get_sync_config()._config.update(backup_directory=str(drive), backup_device_uuid='uuid-1') + + status = await backup.run_backup() + await await_switches() + + assert status['last_success_at'] + saved = yaml.safe_load((test_directory / 'config/sync.yaml').read_text()) + assert saved['backup_last_success_at'] == status['last_success_at'] + + # A restart: the status is new, the config is imported. + update_backup_status(last_success_at=None) + get_sync_config().import_config() + assert get_backup_status()['last_success_at'] == status['last_success_at'] + + +@pytest.mark.asyncio +async def test_failed_backup_keeps_last_success(async_client, test_directory, test_wrolpi_config, monkeypatch): + async def failing_rsync(command, target, uuid): + return 12, ['rsync: connection unexpectedly closed'] + + monkeypatch.setattr(backup, 'backup_skip_reason', lambda: None) + monkeypatch.setattr(backup, '_run_rsync', failing_rsync) + get_sync_config()._config.update(backup_directory=str(test_directory / 'drive'), backup_device_uuid='uuid-1', + backup_last_success_at='2026-11-01T02:00:00+00:00') + + status = await backup.run_backup() + + assert status['state'] == 'error' + assert get_sync_config().backup_last_success_at == '2026-11-01T02:00:00+00:00' diff --git a/wrolpi/sync/test/test_engine.py b/wrolpi/sync/test/test_engine.py new file mode 100644 index 00000000..218d2294 --- /dev/null +++ b/wrolpi/sync/test/test_engine.py @@ -0,0 +1,531 @@ +import os +import pathlib +from collections import namedtuple + +import pytest +import yaml + +from wrolpi.common import get_wrolpi_config, set_test_media_directory +from wrolpi.db import get_db_session +from wrolpi.errors import InvalidConfig +from wrolpi.files.models import FileGroup +from wrolpi.sync import lib +from wrolpi.sync.config import get_sync_config +from wrolpi.sync.conftest import manifest_page +from wrolpi.sync.status import get_sync_status, update_sync_status +from wrolpi.tags import get_tags_config, Tag, TagFile + + +@pytest.mark.parametrize('path, allowed', [ + ('videos/channel/video.mp4', True), + ('a.txt', True), + ('../escape.txt', False), + ('videos/../../escape.txt', False), + ('/etc/passwd', False), + ('videos/.hidden/video.mp4', False), + ('.hidden.txt', False), + ('config/wrolpi.db', False), + ('config/tags.yaml', False), + ('tags/one/video.mp4', False), + # A drive that ignores case (exFAT, NTFS, macOS) would write these into the real config and tags folders. + ('Config/wrolpi.db', False), + ('CONFIG/tags.yaml', False), + ('Tags/one/video.mp4', False), + ('videos\\video.mp4', False), + ('', False), +]) +def test_resolve_destination(test_directory, path, allowed): + destination = lib.resolve_destination(test_directory, path) + if allowed: + assert destination == test_directory / path + else: + assert destination is None + + +def test_resolve_destination_refuses_symlinks(test_directory): + """A sync never writes through a symlink; it could point outside the media directory.""" + outside = test_directory / 'outside' + outside.mkdir() + (test_directory / 'videos').symlink_to(outside) + assert lib.resolve_destination(test_directory, 'videos/video.mp4') is None + + (test_directory / 'docs').mkdir() + (test_directory / 'docs/link.txt').symlink_to(outside / 'target.txt') + assert lib.resolve_destination(test_directory, 'docs/link.txt') is None + + +def test_needs_download(test_directory): + path = test_directory / 'a.txt' + file = lib.ManifestFile('a.txt', 5, 1_700_000_000.0) + assert lib.needs_download(path, file), 'a missing file is downloaded' + + path.write_text('12345') + os.utime(path, (file.mtime, file.mtime)) + assert not lib.needs_download(path, file) + + os.utime(path, (file.mtime + 1.5, file.mtime + 1.5)) + assert not lib.needs_download(path, file), 'within the 2 second tolerance of exFAT' + + os.utime(path, (file.mtime + 3, file.mtime + 3)) + assert lib.needs_download(path, file), 'a different modification time is downloaded' + + os.utime(path, (file.mtime, file.mtime)) + path.write_text('1234') + os.utime(path, (file.mtime, file.mtime)) + assert lib.needs_download(path, file), 'a different size is downloaded' + + +Primary = namedtuple('Primary', 'directory url state') + + +@pytest.fixture +def primary(test_directory, secondary_directory, primary_media_server) -> Primary: + """A primary whose media is `test_directory`, served like Caddy.""" + url, state = primary_media_server(test_directory) + return Primary(test_directory, url, state) + + +def write_primary_file(primary: Primary, path: str, content: bytes, mtime: float = 1_700_000_000.0) \ + -> lib.ManifestFile: + file = primary.directory / path + file.parent.mkdir(parents=True, exist_ok=True) + file.write_bytes(content) + os.utime(file, (mtime, mtime)) + return lib.ManifestFile(path, len(content), mtime) + + +@pytest.mark.asyncio +async def test_download_file(primary, secondary_directory): + """A downloaded file has the primary's bytes and modification time, and no partial is left.""" + file = write_primary_file(primary, 'videos/channel/video name.mp4', b'video bytes' * 1000) + destination = secondary_directory / file.path + + async with lib.PrimaryClient(primary.url) as client: + received = await lib.download_file(client, file, destination) + + assert received == file.size + assert destination.read_bytes() == (primary.directory / file.path).read_bytes() + assert destination.stat().st_mtime == file.mtime + assert not lib.partial_path(destination).exists() + assert not lib.needs_download(destination, file) + + +@pytest.mark.asyncio +async def test_download_file_resumes(primary, secondary_directory): + """A partial left by an interrupted download is resumed with a Range request.""" + content = bytes(range(256)) * 400 + file = write_primary_file(primary, 'big.bin', content) + destination = secondary_directory / file.path + lib.partial_path(destination).write_bytes(content[:30_000]) + + async with lib.PrimaryClient(primary.url) as client: + received = await lib.download_file(client, file, destination) + + assert received == len(content) - 30_000 + assert ('/media/big.bin', 'bytes=30000-') in primary.state['requests'] + assert destination.read_bytes() == content + + +@pytest.mark.asyncio +async def test_download_file_restarts_without_range(primary, secondary_directory): + """A server that ignores Range sends the whole file; the partial is replaced, not appended to.""" + primary.state['honor_range'] = False + content = b'abcdefghij' * 100 + file = write_primary_file(primary, 'a.bin', content) + destination = secondary_directory / file.path + lib.partial_path(destination).write_bytes(content[:500]) + + async with lib.PrimaryClient(primary.url) as client: + await lib.download_file(client, file, destination) + + assert destination.read_bytes() == content + + +@pytest.mark.asyncio +async def test_download_file_changed_on_primary(primary, secondary_directory): + """A file whose size changed after the manifest was read is not written; the next sync gets it.""" + file = write_primary_file(primary, 'a.txt', b'old') + (primary.directory / 'a.txt').write_bytes(b'new and longer') + destination = secondary_directory / file.path + + async with lib.PrimaryClient(primary.url) as client: + with pytest.raises(lib.SyncError): + await lib.download_file(client, file, destination) + + assert not destination.exists() + + +@pytest.mark.asyncio +async def test_download_file_larger_than_manifest(primary, secondary_directory): + """A primary that sends more bytes than its manifest said (no Content-Length to catch it) is not written past + the manifest's size, which is what the free-space check allowed for.""" + primary.state['omit_length'] = True + write_primary_file(primary, 'a.bin', b'x' * 5 * lib.DOWNLOAD_CHUNK_SIZE) + file = lib.ManifestFile('a.bin', 3000, 1_700_000_000.0) + destination = secondary_directory / file.path + + async with lib.PrimaryClient(primary.url) as client: + with pytest.raises(lib.SyncError, match='larger than the primary said'): + await lib.download_file(client, file, destination) + + assert not destination.exists() + assert not lib.partial_path(destination).exists() + + +@pytest.mark.asyncio +async def test_download_file_disk_full(primary, secondary_directory, monkeypatch): + """A file that would leave less than the minimum free space is not downloaded; its partial is kept.""" + file = write_primary_file(primary, 'a.bin', b'x' * 1000) + destination = secondary_directory / file.path + destination.parent.mkdir(parents=True, exist_ok=True) + lib.partial_path(destination).write_bytes(b'x' * 10) + + usage = namedtuple('usage', 'total used free') + monkeypatch.setattr(lib.shutil, 'disk_usage', lambda _: usage(10, 10, lib.MIN_FREE_BYTES)) + + async with lib.PrimaryClient(primary.url) as client: + with pytest.raises(lib.SyncDiskFull): + await lib.download_file(client, file, destination) + + assert lib.partial_path(destination).read_bytes() == b'x' * 10 + assert primary.state['requests'] == [] + + +def capture_manifest() -> list: + pages = list() + after_id = 0 + with get_db_session() as session: + while True: + page = lib.build_manifest_page(session, after_id, limit=2) + pages.append(page) + if page['next_after_id'] is None: + return pages + after_id = page['next_after_id'] + + +@pytest.mark.asyncio +async def test_run_sync(async_client, test_session, primary, secondary_directory, make_secondary, fake_primary_api, + refresh_files, tag_factory, test_tags_config, events_fixture, await_switches): + """A secondary gets the primary's files, then its tags.yaml, and its FileGroups are tagged like the primary's.""" + write_primary_file(primary, 'docs/notes.txt', b'some notes', mtime=1_700_000_001.0) + write_primary_file(primary, 'docs/more notes.txt', b'more notes', mtime=1_700_000_002.0) + write_primary_file(primary, 'videos/channel/video.mp4', b'not really a video', mtime=1_700_000_003.0) + write_primary_file(primary, 'videos/channel/video.info.json', b'{}', mtime=1_700_000_004.0) + await refresh_files() + + tag = await tag_factory('one') + notes = test_session.query(FileGroup).filter_by(primary_path=primary.directory / 'docs/notes.txt').one() + notes.add_tag(test_session, tag.id) + test_session.commit() + get_tags_config().dump_config() + await await_switches() + primary_tags = (primary.directory / 'config/tags.yaml').read_bytes() + assert b'docs/notes.txt' in primary_tags + + pages = capture_manifest() + assert len(pages) >= 2, 'the manifest is paged' + fake_primary_api(pages, primary.directory) + + # The secondary is another machine with its own (empty) database. + test_session.query(TagFile).delete() + test_session.query(Tag).delete() + test_session.query(FileGroup).delete() + test_session.commit() + + # This WROLPi becomes the secondary; it has its own (empty) tags.yaml. + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + (secondary_directory / 'config/tags.yaml').write_text('tags: {}\ntag_files: []\nversion: 1\n') + + status = await lib.run_sync() + + assert status['state'] == 'idle', status['last_error'] + assert status['files_failed'] == 0 + assert status['files_downloaded'] == 4 + assert status['last_success_at'] + for path in ('docs/notes.txt', 'docs/more notes.txt', 'videos/channel/video.mp4', + 'videos/channel/video.info.json'): + copy = secondary_directory / path + assert copy.read_bytes() == (primary.directory / path).read_bytes() + assert copy.stat().st_mtime == (primary.directory / path).stat().st_mtime + + # The secondary's own tags.yaml is kept in today's backup. + backups = list((secondary_directory / 'config/backup').glob('tags-*.yaml')) + assert len(backups) == 1 and b'docs/notes.txt' not in backups[0].read_bytes() + + # The primary's tags.yaml is copied byte for byte, and imported. + assert (secondary_directory / 'config/tags.yaml').read_bytes() == primary_tags + assert 'tags.yaml' in status['configs_imported'] + with get_db_session() as session: + copy = session.query(FileGroup).filter_by(primary_path=secondary_directory / 'docs/notes.txt').one() + assert copy.tag_names == ['one'] + # Files of a group arrive together, and are grouped again on the secondary. + video = session.query(FileGroup).filter_by(directory=secondary_directory / 'videos/channel').one() + assert {pathlib.Path(i['path']).name for i in video.files} == {'video.mp4', 'video.info.json'} + + events_fixture.assert_has_event('sync_completed') + + # Nothing changed, so nothing is downloaded or imported again. + primary.state['requests'].clear() + status = await lib.run_sync() + assert status['files_to_download'] == 0 + assert status['configs_imported'] == [] + assert not [i for i in primary.state['requests'] if not i[0].startswith('/media/config/')] + + +@pytest.mark.asyncio +async def test_run_sync_refuses_unsafe_paths(async_client, primary, secondary_directory, make_secondary, + fake_primary_api): + """A manifest cannot make a secondary write outside its library, or over its configs and database.""" + good = write_primary_file(primary, 'good.txt', b'good') + write_primary_file(primary, 'config/wrolpi.db', b'not a database') + pages = [manifest_page( + (good.path, good.size, good.mtime), + ('config/wrolpi.db', 14, 1_700_000_000.0), + ('../escape.txt', 4, 1_700_000_000.0), + ('tags/one/good.txt', 4, 1_700_000_000.0), + )] + fake_primary_api(pages, primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + status = await lib.run_sync() + + assert status['files_failed'] == 3 + assert (secondary_directory / 'good.txt').read_bytes() == b'good' + assert not (secondary_directory / 'config/wrolpi.db').exists() + assert not (secondary_directory.parent / 'escape.txt').exists() + assert not (secondary_directory / 'tags').exists() + + +@pytest.mark.asyncio +async def test_run_sync_excluded_directories(async_client, primary, secondary_directory, make_secondary, + fake_primary_api): + """A secondary never downloads files inside its excluded folders, and keeps what it already has in them.""" + wanted = [write_primary_file(primary, i, b'wanted') for i in ('Movies2/a.mp4', 'docs/Movies/b.txt', 'c.txt')] + excluded = [write_primary_file(primary, i, b'excluded') for i in ('Movies/a.mp4', 'Movies/deep/b.mp4', + 'Movies [HD]/c.mp4')] + pages = [manifest_page(*((i.path, i.size, i.mtime) for i in wanted + excluded))] + fake_primary_api(pages, primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + get_sync_config()._config['sync_excluded_directories'] = ['Movies', 'Movies [HD]'] + # Copied before the folder was excluded. + (secondary_directory / 'Movies').mkdir() + (secondary_directory / 'Movies/old.mp4').write_bytes(b'old') + + status = await lib.run_sync() + + assert status['last_error'] is None + assert status['files_excluded'] == 3 + assert status['files_downloaded'] == 3 + for file in wanted: + assert (secondary_directory / file.path).read_bytes() == b'wanted' + for file in excluded: + assert not (secondary_directory / file.path).exists() + assert (secondary_directory / 'Movies/old.mp4').read_bytes() == b'old', 'nothing is deleted' + + +@pytest.mark.parametrize('values, expected', [ + ([], []), + (['Movies'], ['Movies']), + (['/Movies/', ' Movies ', 'Movies//'], ['Movies']), + (['videos/Movies', 'videos'], ['videos']), + (['Movies', 'Movies2', 'Movies/deep'], ['Movies', 'Movies2']), + (['a//b'], ['a/b']), + (['Movies [HD]'], ['Movies [HD]']), +]) +def test_normalize_excluded_directories(values, expected): + assert lib.normalize_excluded_directories(values) == expected + + +@pytest.mark.parametrize('value', ['', '/', '..', 'a/../b', '../a', 'a/./b', '.hidden', 'a/.hidden', 'config', + 'config/x', 'tags', 'Config', 'TAGS/x', 'a\\b', None]) +def test_normalize_excluded_directories_rejects(value): + with pytest.raises(InvalidConfig): + lib.normalize_excluded_directories([value]) + + +def test_is_excluded(): + assert lib.is_excluded('Movies/a.mp4', {'Movies'}) + assert lib.is_excluded('Movies', {'Movies'}) + assert not lib.is_excluded('Movies2/a.mp4', {'Movies'}) + assert not lib.is_excluded('docs/Movies/a.mp4', {'Movies'}) + assert lib.is_excluded('docs/Movies/a.mp4', {'docs/Movies'}) + + +def test_list_local_directories(test_directory): + for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'zims', 'lost+found', 'Tags'): + try: + (test_directory / i).mkdir(parents=True) + except FileExistsError: + pass # 'Tags' is 'tags' on a filesystem that ignores case. + (test_directory / 'file.txt').write_text('a file') + (test_directory / 'link').symlink_to(test_directory / 'Movies') + + assert lib.list_local_directories('') == [ + dict(name='Movies', path='Movies'), dict(name='videos', path='videos'), dict(name='zims', path='zims')] + assert lib.list_local_directories('/videos/') == [dict(name='channel', path='videos/channel')] + for path in ('../', 'videos/../..', 'file.txt', 'missing', 'link'): + with pytest.raises(InvalidConfig): + lib.list_local_directories(path) + + +@pytest.mark.asyncio +async def test_list_primary_directories(primary, secondary_directory, make_secondary): + for i in ('Movies', 'videos/channel', 'config', 'tags', '.hidden', 'lost+found'): + (primary.directory / i).mkdir(parents=True) + (primary.directory / 'file.txt').write_text('a file') + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + assert await lib.list_primary_directories('') == [ + dict(name='Movies', path='Movies'), dict(name='videos', path='videos')] + assert await lib.list_primary_directories('videos') == [dict(name='channel', path='videos/channel')] + with pytest.raises(InvalidConfig): + await lib.list_primary_directories('../etc') + + +@pytest.mark.asyncio +async def test_run_sync_only_on_a_secondary(async_client, test_wrolpi_config, fake_primary_api): + status = await lib.run_sync() + assert status['state'] == 'idle' + assert status['last_skip_reason'] == 'This WROLPi is not a secondary' + assert status['started_at'] is None + + +@pytest.mark.asyncio +async def test_run_sync_refuses_a_secondary_primary(async_client, primary, secondary_directory, make_secondary, + fake_primary_api): + """Secondaries do not chain; a secondary's library may be stale.""" + fake_primary_api([manifest_page()], primary.directory, role='secondary') + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + status = await lib.run_sync() + assert status['state'] == 'error' + assert 'is not a primary' in status['last_error'] + + +@pytest.mark.asyncio +async def test_run_sync_cancel(async_client, primary, secondary_directory, make_secondary, fake_primary_api): + """A cancelled sync stops between pages; the files of finished pages stay.""" + first = write_primary_file(primary, 'first.txt', b'first') + second = write_primary_file(primary, 'second.txt', b'second') + pages = [manifest_page((first.path, first.size, first.mtime), next_after_id=1, max_id=2), + manifest_page((second.path, second.size, second.mtime), max_id=2)] + fake_primary_api(pages, primary.directory) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + original = lib._refresh + + async def refresh_then_cancel(paths): + await original(paths) + lib.request_sync_cancel() + + lib._refresh = refresh_then_cancel + try: + status = await lib.run_sync() + finally: + lib._refresh = original + + assert status['last_error'] == 'The sync was cancelled' + assert status['state'] == 'idle' + assert (secondary_directory / 'first.txt').exists() + assert not (secondary_directory / 'second.txt').exists() + + +@pytest.mark.asyncio +async def test_sync_configs_wrolpi_yaml(async_client, primary, secondary_directory, make_secondary): + """Only the library settings of the primary's wrolpi.yaml are applied; the secondary keeps its own role, hotspot, + and the like.""" + primary_config = dict( + videos_destination='my videos/%(channel_name)s', + ignored_directories=['config', 'tags', str(primary.directory / 'private'), '/elsewhere/directory'], + hotspot_ssid='primary hotspot', + role='primary', + nav_color='red', + version=12, + ) + (primary.directory / 'config').mkdir(exist_ok=True) + (primary.directory / 'config/wrolpi.yaml').write_text(yaml.dump(primary_config)) + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + async with lib.PrimaryClient(primary.url) as client: + await lib.sync_configs(client, str(primary.directory), reimport=False) + + config = get_wrolpi_config() + assert config.videos_destination == 'my videos/%(channel_name)s' + assert config.ignored_directories == ['config', 'tags', 'private'] + assert get_sync_config().role == 'secondary', 'the role is in sync.yaml, which is never synced' + assert config.hotspot_ssid == 'WROLPi' + assert config.nav_color == 'violet' + + +@pytest.mark.asyncio +async def test_sync_configs_inventories(async_client, primary, secondary_directory, make_secondary): + """Inventories are a directory of configs; each one is copied.""" + from modules.inventory.common import get_inventory_configs + + inventory_directory = primary.directory / 'config/inventory' + inventory_directory.mkdir(parents=True) + pantry = dict(version=3, slug='pantry', name='Pantry', type='food', fields=[], items=[]) + (inventory_directory / 'pantry.yaml').write_text(yaml.dump(pantry)) + (inventory_directory / '.hidden.yaml').write_text('ignored') + set_test_media_directory(secondary_directory) + make_secondary(primary.url) + + async with lib.PrimaryClient(primary.url) as client: + changed = await lib._sync_inventory_directory(client) + + assert changed + copy = get_inventory_configs().get_directory() / 'pantry.yaml' + assert yaml.safe_load(copy.read_text())['name'] == 'Pantry' + assert not (copy.parent / '.hidden.yaml').exists() + + +@pytest.mark.asyncio +async def test_sync_tick(async_client, test_wrolpi_config, make_secondary, monkeypatch): + """A requested sync runs; WROL Mode drops the request.""" + calls = list() + + async def fake_run_sync(force=False): + calls.append(force) + + monkeypatch.setattr(lib, 'run_sync', fake_run_sync) + + await lib.sync_tick() + assert calls == [], 'a primary never syncs' + + make_secondary() + await lib.sync_tick() + assert calls == [], 'nothing requested, no window' + + lib.request_sync(force=True) + await lib.sync_tick() + assert calls == [True] + assert get_sync_status()['requested'] is False + + get_wrolpi_config()._config['wrol_mode'] = True + lib.request_sync() + await lib.sync_tick() + assert calls == [True] + assert get_sync_status()['requested'] is False + assert get_sync_status()['last_skip_reason'] == 'WROL Mode is enabled' + + +def test_request_sync_cancel(async_client): + update_sync_status(state='running') + lib.request_sync_cancel() + assert get_sync_status()['cancel_requested'] is True + + update_sync_status(state='idle', cancel_requested=False, requested=True) + lib.request_sync_cancel() + assert get_sync_status()['requested'] is False + assert get_sync_status()['cancel_requested'] is False + + +def test_partial_path(): + assert lib.partial_path(pathlib.Path('/a/b/video.mp4')) == pathlib.Path('/a/b/.video.mp4.partial') diff --git a/wrolpi/sync/test/test_manifest.py b/wrolpi/sync/test/test_manifest.py new file mode 100644 index 00000000..7416dadc --- /dev/null +++ b/wrolpi/sync/test/test_manifest.py @@ -0,0 +1,67 @@ +import os + +import pytest + +from wrolpi.files.models import FileGroup +from wrolpi.sync import lib + + +@pytest.mark.asyncio +async def test_manifest_pages(async_client, test_session, test_directory, make_files_structure, refresh_files): + """The manifest lists every file of every FileGroup, a page of FileGroups at a time, in id order.""" + make_files_structure({f'docs/{i}.txt': f'file {i}' for i in range(7)}) + await refresh_files() + + pages = list() + after_id = 0 + while after_id is not None: + page = lib.build_manifest_page(test_session, after_id, limit=3) + pages.append(page) + after_id = page['next_after_id'] + + assert [len(i['groups']) for i in pages] == [3, 3, 1] + ids = [g['id'] for page in pages for g in page['groups']] + assert ids == sorted(ids) + assert pages[0]['max_id'] == max(ids) + + files = {p['path']: p for page in pages for g in page['groups'] for p in g['paths']} + assert set(files) == {f'docs/{i}.txt' for i in range(7)} + stat = (test_directory / 'docs/3.txt').stat() + assert files['docs/3.txt']['size'] == stat.st_size + assert files['docs/3.txt']['mtime'] == stat.st_mtime + + +def test_manifest_skips(test_session, test_directory): + """Files that are still downloading, hidden, missing, or outside the library are not listed.""" + directory = test_directory / 'videos' + directory.mkdir() + for name in ('video.mp4', 'video.mp4.part', 'video.f137.mp4', 'video.mp4.aria2', '.video.mp4.partial', + 'video.en.vtt'): + (directory / name).write_text(name) + outside = test_directory.parent / f'{test_directory.name}-outside.txt' + outside.write_text('outside') + try: + group = FileGroup.from_paths(test_session, directory / 'video.mp4', directory / 'video.en.vtt') + group.files = [ + *group.files, + dict(path='video.mp4.part'), dict(path='video.f137.mp4'), dict(path='video.mp4.aria2'), + dict(path='.video.mp4.partial'), dict(path='deleted.mp4'), + # Older rows stored absolute paths. + dict(path=str(directory / 'video.en.vtt')), + dict(path=str(outside)), + {}, + ] + test_session.commit() + + page = lib.build_manifest_page(test_session) + finally: + os.remove(outside) + + paths = [p['path'] for g in page['groups'] for p in g['paths']] + assert sorted(set(paths)) == ['videos/video.en.vtt', 'videos/video.mp4'] + assert page['next_after_id'] is None + + +def test_manifest_empty(test_session, test_directory): + assert lib.build_manifest_page(test_session) == dict(groups=[], next_after_id=None, max_id=0, + limit=lib.MANIFEST_PAGE_LIMIT) diff --git a/wrolpi/sync/test/test_role.py b/wrolpi/sync/test/test_role.py new file mode 100644 index 00000000..2fc36922 --- /dev/null +++ b/wrolpi/sync/test/test_role.py @@ -0,0 +1,105 @@ +"""A secondary WROLPi never saves the configs it copies from its primary, and never downloads.""" +from datetime import datetime, time, timezone + +import pytest +import yaml + +from wrolpi.common import get_wrolpi_config, SYNCED_CONFIG_FILE_NAMES, get_all_configs +from wrolpi.config_version import IMPORT, SAVE, get_config_version, set_config_version +from wrolpi.dates import is_within_daily_window +from wrolpi.db import get_db_session +from wrolpi.downloader import download_manager +from wrolpi.sync import lib +from wrolpi.sync.config import get_sync_config +from wrolpi.tags import get_tags_config + + +def test_synced_configs_exist(test_channels_config): + """Every synced config name is a real config (a typo would sync nothing, and save on a secondary).""" + assert SYNCED_CONFIG_FILE_NAMES - {'inventory/catalog.yaml'} <= set(get_all_configs()) + assert 'repos.yaml' not in SYNCED_CONFIG_FILE_NAMES, 'git clones are not synced' + assert 'wrolpi.yaml' not in SYNCED_CONFIG_FILE_NAMES, 'wrolpi.yaml is merged, it holds local settings' + assert 'sync.yaml' not in SYNCED_CONFIG_FILE_NAMES, "a secondary's role is its own" + assert 'sync.yaml' in get_all_configs() + + +@pytest.mark.asyncio +async def test_secondary_does_not_save_synced_configs(async_client, test_session, test_tags_config, tag_factory, + make_secondary, await_switches): + await tag_factory('one') + get_tags_config().dump_config() + await await_switches() + primary_copy = test_tags_config.read_text() + assert 'one' in primary_copy + + make_secondary() + await tag_factory('two') + get_tags_config().dump_config() + await await_switches() + assert test_tags_config.read_text() == primary_copy, 'a secondary must not save tags.yaml' + + # wrolpi.yaml holds the secondary's own settings, and is saved. + get_wrolpi_config().update(dict(nav_color='red')) + await await_switches() + assert yaml.safe_load(get_wrolpi_config().get_file().read_text())['nav_color'] == 'red' + + +@pytest.mark.asyncio +async def test_secondary_does_not_save_inventories(async_client, test_inventory_configs, make_secondary, + await_switches): + from modules.inventory.common import get_inventory_configs + config = get_inventory_configs() + slug = config.create_inventory('Pantry', 'food')['slug'] + pantry = config.get_file(slug) + before = pantry.read_text() + + make_secondary() + config.replace(slug, dict(name='Changed')) + await await_switches() + assert pantry.read_text() == before, 'a secondary must not save inventories' + + +def test_secondary_imports_synced_configs(async_client, test_tags_config, make_secondary): + """A secondary's database never outranks the primary's copy of a config: every import applies the file.""" + with get_db_session(commit=True) as session: + set_config_version(session, 'tags.yaml', 10) + config = get_tags_config() + assert config.version_decision(file_version=3)[0] == SAVE + + make_secondary() + assert config.version_decision(file_version=3)[0] == IMPORT + + +def test_secondary_never_downloads(async_client, test_wrolpi_config, make_secondary): + assert download_manager.blocked_download_reason() != 'secondary role' + make_secondary() + assert download_manager.blocked_download_reason() == 'secondary role' + + +@pytest.mark.parametrize('start, end, current, expected', [ + ('01:00', '05:00', time(0, 59), False), + ('01:00', '05:00', time(1, 0), True), + ('01:00', '05:00', time(4, 59), True), + ('01:00', '05:00', time(5, 0), False), + ('22:00', '06:00', time(23, 0), True), + ('22:00', '06:00', time(3, 0), True), + ('22:00', '06:00', time(12, 0), False), +]) +def test_is_within_daily_window(start, end, current, expected): + assert is_within_daily_window(start, end, current) is expected + + +def test_sync_is_due(test_wrolpi_config): + config = get_sync_config() + local_now = datetime(2026, 11, 5, 2, 0, tzinfo=timezone.utc) + never = dict(finished_at=None) + assert not lib.sync_is_due(never, local_now), 'no window is "Sync now" only' + + config._config.update(window_start='01:00', window_end='05:00') + assert lib.sync_is_due(never, local_now) + assert not lib.sync_is_due(never, local_now.replace(hour=6)), 'outside the window' + + recent = dict(finished_at=local_now.replace(hour=1, minute=50).isoformat()) + assert not lib.sync_is_due(recent, local_now), 'synced 10 minutes ago' + earlier = dict(finished_at=local_now.replace(hour=1, minute=40).isoformat()) + assert lib.sync_is_due(earlier, local_now), 'synced 20 minutes ago' diff --git a/wrolpi/test/test_api_fixtures.py b/wrolpi/test/test_api_fixtures.py index b68c47fd..d76504d1 100644 --- a/wrolpi/test/test_api_fixtures.py +++ b/wrolpi/test/test_api_fixtures.py @@ -218,6 +218,28 @@ async def test_downloads_summary_fixture_matches_the_download_manager(async_clie ) +@pytest.mark.parametrize('fixture_name, seed_name, response_name', [ + ('syncStatusFixture', 'default_sync_status', 'SyncStatusResponse'), + ('backupStatusFixture', 'default_backup_status', 'BackupStatusResponse'), +]) +def test_sync_status_fixture_matches_the_seed(fixture_name, seed_name, response_name): + """statusFixture's `sync` and `backup` are seeded by wrolpi.sync.status and declared in wrolpi.schema; the + fixture, the seed and the declaration must name the same keys.""" + from dataclasses import fields + from wrolpi import schema + from wrolpi.sync import status + + seeded = set(getattr(status, seed_name)()) + declared = {i.name for i in fields(getattr(schema, response_name))} + assert seeded == declared, f'seed and {response_name} differ: {sorted(seeded ^ declared)}' + + source = TEST_FIXTURES_JS.read_text() + block = re.search(rf'^export const {fixture_name} = .*?\n(.*?)\n \.\.\.overrides', source, re.S | re.M) + assert block, f'{fixture_name} is no longer in the expected form' + fixture = set(re.findall(r'^ ([a-z0-9_]+):', block.group(1), re.M)) + assert seeded == fixture, f'{fixture_name} differs from the seed: {sorted(seeded ^ fixture)}' + + @pytest.mark.asyncio async def test_which_fixtures_the_spec_can_vouch_for(async_client): """Name the API-shaped fixtures this file does not cover, and why. @@ -249,9 +271,10 @@ async def test_which_fixtures_the_spec_can_vouch_for(async_client): source = TEST_FIXTURES_JS.read_text() api_fixtures = set(re.findall(r'^export const ([a-zA-Z]+Fixture) =', source, re.M)) - # domainsFixture is a list built from domainFixture, and the *Context* fixtures are held to + # domainsFixture is a list built from domainFixture, syncStatusFixture and backupStatusFixture are + # statusFixture's `sync` and `backup` (test_sync_status_fixture_matches_the_seed holds them), and the *Context* fixtures are held to # their React contexts by the frontend's own test-fixtures.test.js. - api_fixtures -= {'domainsFixture'} + api_fixtures -= {'domainsFixture', 'syncStatusFixture', 'backupStatusFixture'} api_fixtures = {n for n in api_fixtures if 'Context' not in n} assert api_fixtures == set(checked) | set(unchecked), (