Backup configs can be restored in the UI. User can choose between Merge or Overwrite, the UI displays a comparison. Configs are compared when user opens the restore modal, configs which won't cause changes are marked.

This commit is contained in:
Roland 2026-03-09 16:11:44 -06:00 • committed by lrnselfreliance
parent 1cf402821c
commit da6e62ee45
15 changed files with 1870 additions and 17 deletions

View file

@ -382,6 +382,12 @@ tr.drag-deselecting td {
-ms-user-select: none;
}
@media only screen and (max-width: 700px) {
.hide-on-mobile {
display: none !important;
}
}
/* Ensure step icons are always visible, even when disabled */
.ui.steps .step.disabled > .icon,
.ui.steps .step.disabled > i.icon,

View file

@ -417,6 +417,35 @@ export async function getConfig(fileName) {
return (await response.json())['config'];
}
export async function getConfigBackups(fileName) {
const response = await apiGet(`${API_URI}/config/backups?file_name=${fileName}`);
if (response.ok) {
return await response.json();
}
return {dates: []};
}
export async function postConfigBackupPreview(fileName, backupDate, mode) {
const body = {file_name: fileName, backup_date: backupDate, mode};
const response = await apiPost(`${API_URI}/config/backup/preview`, body);
if (response.ok) {
return await response.json();
}
const message = await getErrorMessage(response, 'Could not preview backup.');
toast({type: 'error', title: 'Preview failed', description: message, time: 5000});
return null;
}
export async function postConfigBackupImport(fileName, backupDate, mode) {
const body = {file_name: fileName, backup_date: backupDate, mode};
const response = await apiPost(`${API_URI}/config/backup/import`, body);
if (!response.ok) {
const message = await getErrorMessage(response, 'Could not import backup.');
toast({type: 'error', title: 'Backup import failed', description: message, time: 5000});
}
return response;
}
export async function fetchVideoDownloaderConfig() {
return await getConfig('videos_downloader.yaml');
}

View file

@ -1,9 +1,11 @@
import {APIButton} from "../Common";
import {Icon, Popup, Table} from "../Theme";
import React from "react";
import {Button, Header, Icon, Loader, Modal, Popup, Table} from "../Theme";
import React, {useContext, useEffect, useState} from "react";
import {TableBody, TableCell, TableHeader, TableHeaderCell, TableRow} from "semantic-ui-react";
import {TableRowPlaceholder} from "../Placeholder";
import {useNavigate} from "react-router";
import {getConfigBackups, postConfigBackupImport, postConfigBackupPreview} from "../../api";
import {ThemeContext} from "../../contexts/contexts";
function WarningIcon({ok}) {
// null is empty, true is valid, false is invalid.
@ -15,9 +17,285 @@ function WarningIcon({ok}) {
return <Popup trigger={trigger} content={content}/>
}
const TYPE_LABELS = {
tag: 'Tags',
tag_file: 'Tag Files',
tag_zim: 'Tag Zims',
channel: 'Channels',
domain: 'Domains',
download: 'Downloads',
inventory: 'Inventories',
};
const TYPE_COLUMNS = {
tag: [{key: 'name', header: 'Name'}],
tag_file: [{key: 'tag', header: 'Tag'}, {key: 'path', header: 'Path'}],
tag_zim: [{key: 'tag', header: 'Tag'}, {key: 'zim', header: 'Zim'}, {key: 'entry', header: 'Entry'}],
channel: [{key: 'name', header: 'Name'}, {key: 'directory', header: 'Directory'}],
domain: [{key: 'name', header: 'Name'}, {key: 'directory', header: 'Directory'}],
download: [{key: 'url', header: 'URL'}, {key: 'downloader', header: 'Downloader'}],
inventory: [{key: 'name', header: 'Name'}],
};
function groupItemsByType(items) {
const groups = {};
for (const item of items) {
if (!groups[item.type]) {
groups[item.type] = [];
}
groups[item.type].push(item);
}
return Object.entries(groups).map(([type, items]) => ({
type, label: TYPE_LABELS[type] || type, items,
}));
}
function PreviewItemTable({items, negative, inverted}) {
if (!items.length) return null;
const columns = TYPE_COLUMNS[items[0].type] || [{key: 'type', header: 'Item'}];
return <Table compact basic='very' size='small' inverted={!!inverted} style={{marginBottom: '0.5em'}}>
<TableHeader>
<TableRow>
{columns.map(col => <TableHeaderCell key={col.key}>{col.header}</TableHeaderCell>)}
</TableRow>
</TableHeader>
<TableBody>
{items.map((item, i) => <TableRow key={i} negative={negative}>
{columns.map(col =>
<TableCell key={col.key}>{item[col.key] || ''}</TableCell>
)}
</TableRow>)}
</TableBody>
</Table>;
}
const formatDate = (d) => d ? `${d.substring(0, 4)}-${d.substring(4, 6)}-${d.substring(6, 8)}` : '';
function BackupDateRow({date, previews, previewsLoaded, onSelect}) {
const mergeLoaded = previewsLoaded?.merge;
const overwriteLoaded = previewsLoaded?.overwrite;
const mergePreview = previews?.merge;
const overwritePreview = previews?.overwrite;
const hasChanges = (p) => p && (p.add.length > 0 || p.remove.length > 0);
const mergeHasChanges = hasChanges(mergePreview);
const overwriteHasChanges = hasChanges(overwritePreview);
let mergeCell;
if (!mergeLoaded) {
mergeCell = <Loader size='mini' active inline/>;
} else if (mergePreview === null) {
mergeCell = <Popup trigger={<Icon name='warning sign' color='red'/>} content='Failed to load preview'/>;
} else if (mergeHasChanges) {
mergeCell = <Button size='mini' color='green' onClick={() => onSelect(date, 'merge', mergePreview)}>
<Icon name='plus'/> Merge
</Button>;
} else {
mergeCell = <span style={{color: 'grey', fontStyle: 'italic'}}>No changes</span>;
}
let overwriteCell;
if (!overwriteLoaded) {
overwriteCell = <Loader size='mini' active inline/>;
} else if (overwritePreview === null) {
overwriteCell = <Popup trigger={<Icon name='warning sign' color='red'/>} content='Failed to load preview'/>;
} else if (overwriteHasChanges) {
overwriteCell = <Button size='mini' color='orange' onClick={() => onSelect(date, 'overwrite', overwritePreview)}>
<Icon name='refresh'/> Overwrite
</Button>;
} else {
overwriteCell = <span style={{color: 'grey', fontStyle: 'italic'}}>No changes</span>;
}
return <TableRow>
<TableCell>{formatDate(date)}</TableCell>
<TableCell>{mergeCell}</TableCell>
<TableCell>{overwriteCell}</TableCell>
</TableRow>;
}
function BackupsModal({open, onClose, fileName, fetchConfigs}) {
const {inverted} = useContext(ThemeContext);
const [dates, setDates] = useState([]);
const [datesLoading, setDatesLoading] = useState(true);
const [previews, setPreviews] = useState({});
const [previewsLoaded, setPreviewsLoaded] = useState({});
const [selectedAction, setSelectedAction] = useState(null); // {date, mode, preview}
const [applying, setApplying] = useState(false);
// Fetch backup dates on open.
useEffect(() => {
if (!open) return;
let cancelled = false;
setDatesLoading(true);
setDates([]);
setPreviews({});
setPreviewsLoaded({});
setSelectedAction(null);
setApplying(false);
getConfigBackups(fileName).then(result => {
if (cancelled) return;
setDates(result?.dates || []);
setDatesLoading(false);
});
return () => { cancelled = true; };
}, [open, fileName]);
// Progressively fetch previews for each date, newest first.
useEffect(() => {
if (!open || dates.length === 0) return;
let cancelled = false;
const loadPreviews = async () => {
for (const date of dates) {
if (cancelled) break;
const [mergeResult, overwriteResult] = await Promise.all([
postConfigBackupPreview(fileName, date, 'merge'),
postConfigBackupPreview(fileName, date, 'overwrite'),
]);
if (cancelled) break;
setPreviews(prev => ({
...prev,
[date]: {merge: mergeResult?.preview || null, overwrite: overwriteResult?.preview || null}
}));
setPreviewsLoaded(prev => ({...prev, [date]: {merge: true, overwrite: true}}));
}
};
loadPreviews();
return () => { cancelled = true; };
}, [open, dates, fileName]);
const handleSelect = (date, mode, preview) => {
setSelectedAction({date, mode, preview});
};
const handleApply = async () => {
if (!selectedAction) return;
setApplying(true);
await postConfigBackupImport(fileName, selectedAction.date, selectedAction.mode);
await fetchConfigs();
setApplying(false);
handleClose();
};
const handleBack = () => {
setSelectedAction(null);
};
const handleClose = () => {
onClose();
};
// Confirmation view: show the diff for the selected action.
if (selectedAction) {
const {date, mode, preview} = selectedAction;
const modeLabel = mode === 'merge' ? 'Merge' : 'Overwrite';
return <Modal open={open} onClose={handleClose} closeIcon size='large'>
<Modal.Header>
{modeLabel}: {fileName} ({formatDate(date)})
</Modal.Header>
<Modal.Content>
{applying
? <div style={{textAlign: 'center', padding: '2em'}}>
<Loader active inline='centered'>Applying backup...</Loader>
</div>
: <>
{preview.add.length > 0 && <>
<Header as='h4' color='green'>
<Icon name='plus'/> {preview.add.length} item{preview.add.length !== 1 ? 's' : ''} to add
</Header>
<div style={{maxHeight: '300px', overflowY: 'auto', marginBottom: '1em'}}>
{groupItemsByType(preview.add).map(group =>
<React.Fragment key={group.type}>
<Header as='h5' inverted={!!inverted}
style={{marginBottom: '0.3em'}}>{group.label}</Header>
<PreviewItemTable items={group.items} inverted={inverted}/>
</React.Fragment>
)}
</div>
</>}
{preview.remove.length > 0 && <>
<Header as='h4' color='red'>
<Icon name='minus'/> {preview.remove.length} item{preview.remove.length !== 1 ? 's' : ''} to
remove
</Header>
<div style={{maxHeight: '300px', overflowY: 'auto', marginBottom: '1em'}}>
{groupItemsByType(preview.remove).map(group =>
<React.Fragment key={group.type}>
<Header as='h5' inverted={!!inverted}
style={{marginBottom: '0.3em'}}>{group.label}</Header>
<PreviewItemTable items={group.items} negative inverted={inverted}/>
</React.Fragment>
)}
</div>
</>}
<p style={{color: 'grey'}}>{preview.unchanged} unchanged
item{preview.unchanged !== 1 ? 's' : ''}</p>
</>
}
</Modal.Content>
<Modal.Actions>
{!applying && <>
<Button onClick={handleBack}>Back</Button>
<Button color='green' onClick={handleApply}>
<Icon name='check'/> Apply
</Button>
</>}
</Modal.Actions>
</Modal>;
}
// Backups list view.
return <Modal open={open} onClose={handleClose} closeIcon size='small'>
<Modal.Header>Restore Backup: {fileName}</Modal.Header>
<Modal.Content>
{datesLoading
? <div style={{textAlign: 'center', padding: '2em'}}>
<Loader active inline='centered'>Loading backups...</Loader>
</div>
: dates.length === 0
? <p>No backups available for this config.</p>
: <div style={{maxHeight: '400px', overflowY: 'auto'}}>
<Table compact unstackable>
<TableHeader>
<TableRow>
<TableHeaderCell>Date</TableHeaderCell>
<TableHeaderCell>Merge</TableHeaderCell>
<TableHeaderCell>Overwrite</TableHeaderCell>
</TableRow>
</TableHeader>
<TableBody>
{dates.map(date =>
<BackupDateRow
key={date}
date={date}
previews={previews[date]}
previewsLoaded={previewsLoaded[date]}
onSelect={handleSelect}
/>
)}
</TableBody>
</Table>
</div>
}
</Modal.Content>
<Modal.Actions>
<Button onClick={handleClose}>Close</Button>
</Modal.Actions>
</Modal>;
}
function ConfigTableRow({config, importConfig, saveConfig, fetchConfigs}) {
const {file_name, rel_path, valid, successful_import} = config;
const {file_name, rel_path, valid, successful_import, has_backup_import} = config;
const navigate = useNavigate();
const [backupsModalOpen, setBackupsModalOpen] = useState(false);
const localImportConfig = async () => {
await importConfig(file_name);
@ -57,25 +335,39 @@ function ConfigTableRow({config, importConfig, saveConfig, fetchConfigs}) {
<TableCell><span onClick={() => navigate(`?preview=${rel_path}`)}>{file_name}</span></TableCell>
<TableCell><WarningIcon ok={successful_import}/></TableCell>
<TableCell><WarningIcon ok={valid}/></TableCell>
<TableCell className="hide-on-mobile">
{has_backup_import
? <Button icon size='mini' onClick={() => setBackupsModalOpen(true)}>
<Icon name='undo'/>
</Button>
: null
}
</TableCell>
<TableCell>{importButton}</TableCell>
<TableCell>{saveButton}</TableCell>
{backupsModalOpen && <BackupsModal
open={backupsModalOpen}
onClose={() => setBackupsModalOpen(false)}
fileName={file_name}
fetchConfigs={fetchConfigs}
/>}
</TableRow>
}
export function ConfigsTable({configs, loading, importConfig, saveConfig, fetchConfigs}) {
let body = <TableRow>
<TableCell colSpan={4}><TableRowPlaceholder/></TableCell>
<TableCell colSpan={6}><TableRowPlaceholder/></TableCell>
</TableRow>
if (loading) {
// Configs are being fetched again.
} else if (configs === undefined) {
body = <TableRow>
<TableCell colSpan={2}>Failed to fetch configs</TableCell>
<TableCell colSpan={6}>Failed to fetch configs</TableCell>
</TableRow>;
} else if (configs && Object.keys(configs).length === 0) {
body = <TableRow>
<TableCell colSpan={2}>No configs exist. Try tagging some files.</TableCell>
<TableCell colSpan={6}>No configs exist. Try tagging some files.</TableCell>
</TableRow>;
} else if (configs) {
body = Object.entries(configs).map(([key, value]) =>
@ -89,12 +381,13 @@ export function ConfigsTable({configs, loading, importConfig, saveConfig, fetchC
);
}
return <Table unstackable striped>
return <Table unstackable striped compact>
<TableHeader>
<TableRow>
<TableHeaderCell width={10}>File Name</TableHeaderCell>
<TableHeaderCell width={8}>File Name</TableHeaderCell>
<TableHeaderCell width={1}>Imported</TableHeaderCell>
<TableHeaderCell width={1}>Valid</TableHeaderCell>
<TableHeaderCell width={3} className="hide-on-mobile">Restore</TableHeaderCell>
<TableHeaderCell width={1}>Import</TableHeaderCell>
<TableHeaderCell width={1}>Save</TableHeaderCell>
</TableRow>

View file

@ -1893,7 +1893,10 @@
}
const checkUpgradeComplete = async () => {
try {
const resp = await fetch('/api/ready', {
// WARNING: This must be a relative URL (no leading slash) so it resolves to
// /controller/api/ready behind Caddy, or /api/ready when accessed directly.
// An absolute '/api/ready' would be routed to Sanic by Caddy, not the Controller.
const resp = await fetch('api/ready', {
method: 'GET',
headers: {'Accept': 'application/json'},
signal: AbortSignal.timeout(5000)
@ -1936,7 +1939,10 @@
// Poll controller's readiness endpoint (same-origin, avoids CORS/cert issues).
const checkMainApp = async () => {
try {
const resp = await fetch('/api/ready', {
// WARNING: This must be a relative URL (no leading slash) so it resolves to
// /controller/api/ready behind Caddy, or /api/ready when accessed directly.
// An absolute '/api/ready' would be routed to Sanic by Caddy, not the Controller.
const resp = await fetch('api/ready', {
method: 'GET',
headers: {'Accept': 'application/json'},
signal: AbortSignal.timeout(5000)

View file

@ -180,6 +180,70 @@ class DomainsConfig(ConfigFile):
logger.info(f'Dumping {len(collections_data)} domain collections to config')
self.save(file, send_events, overwrite)
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(collections=[], version=0)
backup_collections = backup_data.get('collections', [])
current_collections = current_data.get('collections', [])
# Identity key for domains is the directory (if present) or name
def get_key(c):
return c.get('directory') or c.get('name', '')
current_keys = {get_key(c) for c in current_collections}
backup_keys = {get_key(c) for c in backup_collections}
add = []
remove = []
unchanged = 0
for c in backup_collections:
key = get_key(c)
if key not in current_keys:
add.append(dict(type='domain', name=c.get('name', ''), directory=c.get('directory', '')))
else:
unchanged += 1
if mode == 'overwrite':
current_key_to_coll = {get_key(c): c for c in current_collections}
for key in sorted(current_keys - backup_keys):
c = current_key_to_coll[key]
remove.append(dict(type='domain', name=c.get('name', ''), directory=c.get('directory', '')))
return dict(mode=mode, add=add, remove=remove, unchanged=unchanged)
def import_backup(self, backup_date: str, mode: str, send_events: bool = False):
self._preserve_current_config()
import shutil
backup_file = self._get_backup_file(backup_date)
config_file = self.get_file()
if mode == 'overwrite':
config_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(collections=[], version=0)
def get_key(c):
return c.get('directory') or c.get('name', '')
current_keys = {get_key(c) for c in current_data.get('collections', [])}
merged_collections = list(current_data.get('collections', []))
for c in backup_data.get('collections', []):
if get_key(c) not in current_keys:
merged_collections.append(c)
merged_data = dict(
collections=merged_collections,
version=current_data.get('version', 0) + 1,
)
config_file.parent.mkdir(parents=True, exist_ok=True)
self.write_config_data(merged_data, config_file)
self.import_config(send_events=send_events)
# Global instance
domains_config = DomainsConfig()

View file

@ -212,6 +212,61 @@ class InventoriesConfig(ConfigFile):
Events.send_config_import_failed(message)
raise
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(inventories=[], version=0)
backup_inventories = backup_data.get('inventories', [])
current_inventories = current_data.get('inventories', [])
current_names = {inv.get('name') for inv in current_inventories if inv.get('name')}
backup_names = {inv.get('name') for inv in backup_inventories if inv.get('name')}
add = []
remove = []
unchanged = 0
for inv in backup_inventories:
name = inv.get('name')
if name and name not in current_names:
add.append(dict(type='inventory', name=name))
elif name:
unchanged += 1
if mode == 'overwrite':
for name in sorted(current_names - backup_names):
remove.append(dict(type='inventory', name=name))
return dict(mode=mode, add=add, remove=remove, unchanged=unchanged)
def import_backup(self, backup_date: str, mode: str, send_events: bool = False):
self._preserve_current_config()
import shutil
backup_file = self._get_backup_file(backup_date)
config_file = self.get_file()
if mode == 'overwrite':
config_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(inventories=[], version=0)
current_names = {inv.get('name') for inv in current_data.get('inventories', []) if inv.get('name')}
merged_inventories = list(current_data.get('inventories', []))
for inv in backup_data.get('inventories', []):
if inv.get('name') and inv['name'] not in current_names:
merged_inventories.append(inv)
merged_data = dict(
inventories=merged_inventories,
version=current_data.get('version', 0) + 1,
)
config_file.parent.mkdir(parents=True, exist_ok=True)
self.write_config_data(merged_data, config_file)
self.import_config(send_events=send_events)
INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig()
TEST_INVENTORIES_CONFIG: InventoriesConfig = None

View file

@ -452,6 +452,62 @@ class ChannelsConfig(ConfigFile):
channel_import_logger.info(f'Dumping {len(channels_data)} channels to config')
self.save(file, send_events, overwrite)
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(channels=[], version=0)
backup_channels = backup_data.get('channels', [])
current_channels = current_data.get('channels', [])
current_dirs = {ch.get('directory') for ch in current_channels if ch.get('directory')}
backup_dirs = {ch.get('directory') for ch in backup_channels if ch.get('directory')}
add = []
remove = []
unchanged = 0
for ch in backup_channels:
d = ch.get('directory')
if d and d not in current_dirs:
add.append(dict(type='channel', name=ch.get('name', ''), directory=d))
elif d:
unchanged += 1
if mode == 'overwrite':
backup_dir_to_name = {ch.get('directory'): ch.get('name', '') for ch in current_channels}
for d in sorted(current_dirs - backup_dirs):
remove.append(dict(type='channel', name=backup_dir_to_name.get(d, ''), directory=d))
return dict(mode=mode, add=add, remove=remove, unchanged=unchanged)
def import_backup(self, backup_date: str, mode: str, send_events: bool = False):
self._preserve_current_config()
import shutil
backup_file = self._get_backup_file(backup_date)
config_file = self.get_file()
if mode == 'overwrite':
config_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(channels=[], version=0)
current_dirs = {ch.get('directory') for ch in current_data.get('channels', []) if ch.get('directory')}
merged_channels = list(current_data.get('channels', []))
for ch in backup_data.get('channels', []):
if ch.get('directory') and ch['directory'] not in current_dirs:
merged_channels.append(ch)
merged_data = dict(
channels=merged_channels,
version=current_data.get('version', 0) + 1,
)
config_file.parent.mkdir(parents=True, exist_ok=True)
self.write_config_data(merged_data, config_file)
self.import_config(send_events=send_events)
CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig()
TEST_CHANNELS_CONFIG: ChannelsConfig = None

View file

@ -173,7 +173,7 @@ async def test_import_channel_downloads(await_switches, test_session, channel_fa
channels_config = get_channels_config()
update_channel_config(channels_config, 'foo',
{'downloads': [
{'url': 'https://example.com/channel1', 'frequency': DownloadFrequency.weekly}
{'url': 'https://example.com/channel1', 'frequency': DownloadFrequency.weekly.value}
]})
await await_switches()

View file

@ -449,6 +449,9 @@ def get_media_directory() -> Path:
return MEDIA_DIRECTORY
DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'inventories.yaml'}
class ConfigFile:
"""
This is used to keep the DB and config files in sync. Importing the config reads the config from file, then
@ -491,11 +494,13 @@ class ConfigFile:
return f'<{self.__class__.__name__} file={self.get_file()}>'
def config_status(self) -> dict:
has_backup_import = self.file_name in DB_CONFIG_FILE_NAMES
d = dict(
file_name=self.file_name,
rel_path=get_relative_to_media_directory(self.get_file()),
successful_import=self.successful_import,
valid=self.is_valid(),
has_backup_import=has_backup_import,
)
return d
@ -532,7 +537,7 @@ class ConfigFile:
def read_config_file(self, file: pathlib.Path = None) -> dict:
file = file or self.get_file()
with file.open('rt') as fh:
config_data = yaml.load(fh, Loader=yaml.Loader)
config_data = yaml.load(fh, Loader=yaml.CSafeLoader)
if not isinstance(config_data, dict):
raise InvalidConfig(f'Config file is invalid: {file}')
return config_data
@ -546,6 +551,53 @@ class ConfigFile:
path = path.with_name(name)
return path
def get_backup_dates(self) -> list:
"""Scan config/backup/ for backup files matching this config, return sorted dates (newest first)."""
backup_dir = get_media_directory() / 'config/backup'
if not backup_dir.is_dir():
return []
stem = pathlib.Path(self.file_name).stem
suffix = pathlib.Path(self.file_name).suffix
dates = []
for f in backup_dir.iterdir():
if f.is_file() and f.name.startswith(f'{stem}-') and f.name.endswith(suffix):
# Extract date string between stem- and suffix
date_str = f.name[len(stem) + 1:-len(suffix)]
if len(date_str) == 8 and date_str.isdigit():
dates.append(date_str)
dates.sort(reverse=True)
return dates
def _get_backup_file(self, backup_date: str) -> pathlib.Path:
"""Return path to a specific backup file."""
stem = pathlib.Path(self.file_name).stem
suffix = pathlib.Path(self.file_name).suffix
return get_media_directory() / f'config/backup/{stem}-{backup_date}{suffix}'
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
"""Preview what a backup import would do. Subclasses override this."""
raise NotImplementedError(f'{self.__class__.__name__} does not support backup import')
def _preserve_current_config(self):
"""Back up the current config file to today's backup slot before a restore overwrites it.
This ensures the user can undo a backup restore by restoring today's backup.
If today's backup already exists (from an earlier save), it is not overwritten.
"""
config_file = self.get_file()
if not config_file.is_file():
return
todays_backup = self._get_backup_filename()
if todays_backup.is_file():
return
todays_backup.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(config_file, todays_backup)
def import_backup(self, backup_date: str, mode: str, send_events: bool = False):
"""Import a backup file. Subclasses override this."""
raise NotImplementedError(f'{self.__class__.__name__} does not support backup import')
def save(self, file: pathlib.Path = None, send_events: bool = False, overwrite: bool = False):
"""
Write this config to its file.
@ -612,8 +664,19 @@ class ConfigFile:
raise e
def write_config_data(self, config: dict, config_file: pathlib.Path):
from decimal import Decimal
class _ConfigDumper(yaml.CSafeDumper):
"""CSafeDumper that converts Decimal to str. All other Python objects are rejected."""
pass
_ConfigDumper.add_representer(Decimal, lambda dumper, data: dumper.represent_str(str(data)))
with config_file.open('wt') as fh:
yaml.dump(config, fh, width=self.width, sort_keys=True)
try:
yaml.dump(config, fh, Dumper=_ConfigDumper, width=self.width, sort_keys=True)
except yaml.representer.RepresenterError as e:
raise ValueError(f'Config contains a Python object that cannot be serialized: {e}') from e
# Wait for data to be written before releasing lock.
fh.flush()
os.fsync(fh.fileno())

View file

@ -4,7 +4,7 @@ from sanic_ext.extensions.openapi import openapi
from wrolpi import config_schema
from wrolpi.api_utils import json_response
from wrolpi.common import get_all_configs, get_config_by_file_name
from wrolpi.common import get_all_configs, get_config_by_file_name, DB_CONFIG_FILE_NAMES
from wrolpi.common import logger
from wrolpi.errors import InvalidConfig
@ -70,3 +70,60 @@ def post_config_dump(_: Request, body: config_schema.ConfigsRequest):
raise InvalidConfig(f'Failed to dump config {body.file_name}')
return response.empty()
@config_bp.get('/backups')
@openapi.description('Get available backup dates for a config file.')
@openapi.parameter('file_name', str, location='query')
def get_config_backups(request: Request):
file_name = request.args.get('file_name')
if not file_name:
raise InvalidConfig('file_name is required')
config = get_config_by_file_name(file_name)
if file_name not in DB_CONFIG_FILE_NAMES:
raise InvalidConfig(f'{file_name} does not support backup import')
dates = config.get_backup_dates()
return json_response(dict(dates=dates))
@config_bp.post('/backup/preview')
@openapi.definition(
description='Preview what a backup import would change.',
body=config_schema.ConfigBackupPreviewRequest,
validate=True,
)
def post_config_backup_preview(_: Request, body: config_schema.ConfigBackupPreviewRequest):
if body.mode not in ('merge', 'overwrite'):
raise InvalidConfig('mode must be "merge" or "overwrite"')
config = get_config_by_file_name(body.file_name)
if body.file_name not in DB_CONFIG_FILE_NAMES:
raise InvalidConfig(f'{body.file_name} does not support backup import')
backup_file = config._get_backup_file(body.backup_date)
if not backup_file.is_file():
raise InvalidConfig(f'Backup file not found for date {body.backup_date}')
preview = config.preview_backup_import(body.backup_date, body.mode)
return json_response(dict(preview=preview))
@config_bp.post('/backup/import')
@openapi.definition(
description='Import a backup config file.',
body=config_schema.ConfigBackupImportRequest,
validate=True,
)
def post_config_backup_import(_: Request, body: config_schema.ConfigBackupImportRequest):
if body.mode not in ('merge', 'overwrite'):
raise InvalidConfig('mode must be "merge" or "overwrite"')
config = get_config_by_file_name(body.file_name)
if body.file_name not in DB_CONFIG_FILE_NAMES:
raise InvalidConfig(f'{body.file_name} does not support backup import')
backup_file = config._get_backup_file(body.backup_date)
if not backup_file.is_file():
raise InvalidConfig(f'Backup file not found for date {body.backup_date}')
try:
config.import_backup(body.backup_date, body.mode, send_events=True)
except Exception as e:
logger.error(f'Failed to import backup: {body.file_name} {body.backup_date}', exc_info=e)
raise InvalidConfig(f'Failed to import backup {body.file_name}')
return response.empty()

View file

@ -20,3 +20,17 @@ class ConfigsRequest:
@dataclass
class ConfigUpdateRequest:
config: dict
@dataclass
class ConfigBackupPreviewRequest:
file_name: str
backup_date: str
mode: str
@dataclass
class ConfigBackupImportRequest:
file_name: str
backup_date: str
mode: str

View file

@ -1552,6 +1552,78 @@ class DownloadManagerConfig(ConfigFile):
self.successful_import = True
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(downloads=[], version=0)
backup_downloads = backup_data.get('downloads', [])
current_downloads = current_data.get('downloads', [])
current_urls = {dl.get('url') for dl in current_downloads if dl.get('url')}
backup_urls = {dl.get('url') for dl in backup_downloads if dl.get('url')}
add = []
remove = []
unchanged = 0
for dl in backup_downloads:
url = dl.get('url')
if not url:
continue
# In merge mode, skip once-downloads (no frequency)
if mode == 'merge' and not dl.get('frequency'):
continue
if url not in current_urls:
add.append(dict(type='download', url=url, downloader=dl.get('downloader', '')))
else:
unchanged += 1
if mode == 'overwrite':
current_url_to_dl = {dl.get('url'): dl for dl in current_downloads}
for url in sorted(current_urls - backup_urls):
dl = current_url_to_dl[url]
remove.append(dict(type='download', url=url, downloader=dl.get('downloader', '')))
return dict(mode=mode, add=add, remove=remove, unchanged=unchanged)
def import_backup(self, backup_date: str, mode: str, send_events: bool = False):
self._preserve_current_config()
import shutil
backup_file = self._get_backup_file(backup_date)
config_file = self.get_file()
if mode == 'overwrite':
config_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[], version=0)
current_urls = {dl.get('url') for dl in current_data.get('downloads', []) if dl.get('url')}
merged_downloads = list(current_data.get('downloads', []))
for dl in backup_data.get('downloads', []):
url = dl.get('url')
# Only merge recurring downloads
if url and url not in current_urls and dl.get('frequency'):
merged_downloads.append(dl)
# Merge skip_urls
current_skip = set(current_data.get('skip_urls', []))
merged_skip = list(current_data.get('skip_urls', []))
for url in backup_data.get('skip_urls', []):
if url not in current_skip:
merged_skip.append(url)
merged_data = dict(
downloads=merged_downloads,
skip_urls=merged_skip,
version=current_data.get('version', 0) + 1,
)
config_file.parent.mkdir(parents=True, exist_ok=True)
self.write_config_data(merged_data, config_file)
self.import_config(send_events=send_events)
DOWNLOAD_MANAGER_CONFIG: DownloadManagerConfig = DownloadManagerConfig()
TEST_DOWNLOAD_MANAGER_CONFIG: DownloadManagerConfig = None

View file

@ -514,6 +514,111 @@ class TagsConfig(ConfigFile):
Events.send_config_import_failed(message)
raise
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[], tag_zims=[])
backup_tags = backup_data.get('tags', {})
current_tags = current_data.get('tags', {})
backup_tag_files = backup_data.get('tag_files', [])
current_tag_files = current_data.get('tag_files', [])
backup_tag_zims = backup_data.get('tag_zims', [])
current_tag_zims = current_data.get('tag_zims', [])
# Build identity sets
current_tag_names = set(current_tags.keys())
current_tf_keys = {(tf[0], tf[1]) for tf in current_tag_files if len(tf) >= 2}
current_tz_keys = {(tz[0], tz[1], tz[2]) for tz in current_tag_zims if len(tz) >= 3}
backup_tag_names = set(backup_tags.keys())
backup_tf_keys = {(tf[0], tf[1]) for tf in backup_tag_files if len(tf) >= 2}
backup_tz_keys = {(tz[0], tz[1], tz[2]) for tz in backup_tag_zims if len(tz) >= 3}
add = []
remove = []
unchanged = 0
# Tags
for name in sorted(backup_tag_names):
if name not in current_tag_names:
add.append(dict(type='tag', name=name))
else:
unchanged += 1
if mode == 'overwrite':
for name in sorted(current_tag_names - backup_tag_names):
remove.append(dict(type='tag', name=name))
# Tag files
for tf in backup_tag_files:
if len(tf) >= 2:
key = (tf[0], tf[1])
if key not in current_tf_keys:
add.append(dict(type='tag_file', tag=tf[0], path=tf[1]))
else:
unchanged += 1
if mode == 'overwrite':
for key in sorted(current_tf_keys - backup_tf_keys):
remove.append(dict(type='tag_file', tag=key[0], path=key[1]))
# Tag zims
for tz in backup_tag_zims:
if len(tz) >= 3:
key = (tz[0], tz[1], tz[2])
if key not in current_tz_keys:
add.append(dict(type='tag_zim', tag=tz[0], zim=tz[1], entry=tz[2]))
else:
unchanged += 1
if mode == 'overwrite':
for key in sorted(current_tz_keys - backup_tz_keys):
remove.append(dict(type='tag_zim', tag=key[0], zim=key[1], entry=key[2]))
return dict(mode=mode, add=add, remove=remove, unchanged=unchanged)
def import_backup(self, backup_date: str, mode: str, send_events: bool = False):
self._preserve_current_config()
import shutil
backup_file = self._get_backup_file(backup_date)
config_file = self.get_file()
if mode == 'overwrite':
config_file.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(backup_file, config_file)
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[], tag_zims=[], version=0)
# Merge tags: add missing tags only
merged_tags = dict(current_data.get('tags', {}))
for name, value in backup_data.get('tags', {}).items():
if name not in merged_tags:
merged_tags[name] = value
# Merge tag_files: add missing by (tag_name, path)
current_tf_keys = {(tf[0], tf[1]) for tf in current_data.get('tag_files', []) if len(tf) >= 2}
merged_tag_files = list(current_data.get('tag_files', []))
for tf in backup_data.get('tag_files', []):
if len(tf) >= 2 and (tf[0], tf[1]) not in current_tf_keys:
merged_tag_files.append(tf)
# Merge tag_zims: add missing by (tag_name, zim_path, entry)
current_tz_keys = {(tz[0], tz[1], tz[2]) for tz in current_data.get('tag_zims', []) if len(tz) >= 3}
merged_tag_zims = list(current_data.get('tag_zims', []))
for tz in backup_data.get('tag_zims', []):
if len(tz) >= 3 and (tz[0], tz[1], tz[2]) not in current_tz_keys:
merged_tag_zims.append(tz)
merged_data = dict(
tags=merged_tags,
tag_files=merged_tag_files,
tag_zims=merged_tag_zims,
version=current_data.get('version', 0) + 1,
)
config_file.parent.mkdir(parents=True, exist_ok=True)
self.write_config_data(merged_data, config_file)
self.import_config(send_events=send_events)
TAGS_CONFIG: TagsConfig = TagsConfig()
TEST_TAGS_CONFIG: TagsConfig = None # noqa

File diff suppressed because it is too large Load diff

View file

@ -237,8 +237,8 @@ async def test_import_tags_config(async_client, test_session, test_directory, te
# Tag names can contain Unicode characters.
tags={'🔫': {'color': '#123456'}},
tag_files=[
('🔫', str(example_singlefile.relative_to(test_directory)), '2000-01-01 01:01:01'),
('🔫', 'does not exist', None),
['🔫', str(example_singlefile.relative_to(test_directory)), '2000-01-01 01:01:01'],
['🔫', 'does not exist', None],
],
)
yaml.dump(data, fh)
@ -270,7 +270,7 @@ async def test_import_tags_config_missing_file(test_session, test_directory, tes
# Tag names can contain Unicode characters.
tags={'🦄': {'color': '#654321'}},
tag_files=[
('🦄', str(example_singlefile.relative_to(test_directory)), '2000-01-01 01:01:01'),
['🦄', str(example_singlefile.relative_to(test_directory)), '2000-01-01 01:01:01'],
],
)
yaml.dump(data, fh)