A malformed entry stops a backup preview or merge with an error naming it (#896)

* A malformed entry stops a backup preview or merge with an error naming it

Backup previews and merges read every config entry as well-formed: a malformed one (an entry that is not a
mapping, a missing or non-text identity, a short tag_files line) raised an AttributeError or TypeError, so the
preview was a 500 and a merge failed with a generic message.  `ConfigFile.backup_entries` now checks each entry
a preview or merge reads, in the backup and in the current config, and raises InvalidConfig naming the file and
the entry; the merge endpoint passes that message on.  A malformed entry may be corrupt, so it is not skipped.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* A falsy "tags" that is not a mapping is an error in a backup preview or merge

`tags: []`, `''`, `0` or `false` became no tags through `or dict()` before the mapping check.  Only a missing
or null `tags` means no tags now.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-27 20:59:49 -06:00 • committed by GitHub
parent 3c815245b4
commit 98ceb67321
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
9 changed files with 190 additions and 45 deletions

View file

@ -129,6 +129,10 @@ class DomainsConfigValidator:
collections: List[dict] = field(default_factory=list)
# The fields a domains.yaml entry must have (see `DomainsSection`).
DOMAIN_FIELDS = dict(required=('name',), optional=('directory',))
class DomainsConfig(ConfigFile):
"""
Config file for Domain Collections.
@ -229,8 +233,8 @@ class DomainsConfig(ConfigFile):
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(collections=[], version=0)
backup_collections = backup_data.get('collections', [])
current_collections = current_data.get('collections', [])
backup_collections = self.backup_entries(backup_data, 'collections', backup_file, **DOMAIN_FIELDS)
current_collections = self.backup_entries(current_data, 'collections', self.get_file(), **DOMAIN_FIELDS)
# Identity key for domains is the directory (if present) or name
def get_key(c):
@ -273,9 +277,11 @@ class DomainsConfig(ConfigFile):
def get_key(c):
return c.get('directory') or c.get('name', '')
current_keys = {get_key(c) for c in current_data.get('collections', [])}
merged_collections = list(current_data.get('collections', []))
for c in backup_data.get('collections', []):
current_collections = self.backup_entries(current_data, 'collections', config_file, **DOMAIN_FIELDS)
backup_collections = self.backup_entries(backup_data, 'collections', backup_file, **DOMAIN_FIELDS)
current_keys = {get_key(c) for c in current_collections}
merged_collections = list(current_collections)
for c in backup_collections:
if get_key(c) not in current_keys:
merged_collections.append(c)

View file

@ -1542,11 +1542,14 @@ class ReposConfig(ConfigFile):
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
"""Preview which Repos a backup import would add/remove (keyed by URL)."""
backup_data = self.read_config_file(self._get_backup_file(backup_date))
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(repos=[], version=0)
backup_repos = {repo_url_key(i['url']): i for i in backup_data.get('repos', []) if i.get('url')}
current_repos = {repo_url_key(i['url']): i for i in current_data.get('repos', []) if i.get('url')}
backup_repos = {repo_url_key(i['url']): i for i in
self.backup_entries(backup_data, 'repos', backup_file, required=('url',))}
current_repos = {repo_url_key(i['url']): i for i in
self.backup_entries(current_data, 'repos', self.get_file(), required=('url',))}
add = [dict(type='repo', name=i.get('name', ''), url=i['url'])
for key, i in backup_repos.items() if key not in current_repos]
@ -1569,10 +1572,11 @@ class ReposConfig(ConfigFile):
elif mode == 'merge':
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(repos=[], version=0)
current_keys = {repo_url_key(i['url']) for i in current_data.get('repos', []) if i.get('url')}
merged = list(current_data.get('repos', []))
merged.extend(i for i in backup_data.get('repos', [])
if i.get('url') and repo_url_key(i['url']) not in current_keys)
current_repos = self.backup_entries(current_data, 'repos', config_file, required=('url',))
backup_repos = self.backup_entries(backup_data, 'repos', backup_file, required=('url',))
current_keys = {repo_url_key(i['url']) for i in current_repos}
merged = list(current_repos)
merged.extend(i for i in backup_repos if repo_url_key(i['url']) not in current_keys)
self.write_config_data(dict(repos=merged, version=current_data.get('version', 0) + 1), config_file)
# Importing a backup is the user's choice: apply it whatever the versions.

View file

@ -451,6 +451,10 @@ class ChannelsConfigValidator:
channels: list[ChannelDictValidator] = dataclasses.field(default_factory=list)
# The fields a channels.yaml entry must have (see `ChannelsSection`).
CHANNEL_KEYS = ('directory', 'name')
class ChannelsConfig(ConfigFile):
file_name = 'channels.yaml'
sections = (ChannelsSection(), ChannelDownloadsSection())
@ -609,8 +613,8 @@ class ChannelsConfig(ConfigFile):
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(channels=[], version=0)
backup_channels = backup_data.get('channels', [])
current_channels = current_data.get('channels', [])
backup_channels = self.backup_entries(backup_data, 'channels', backup_file, required=CHANNEL_KEYS)
current_channels = self.backup_entries(current_data, 'channels', self.get_file(), required=CHANNEL_KEYS)
current_dirs = {ch.get('directory') for ch in current_channels if ch.get('directory')}
backup_dirs = {ch.get('directory') for ch in backup_channels if ch.get('directory')}
@ -645,9 +649,11 @@ class ChannelsConfig(ConfigFile):
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(channels=[], version=0)
current_dirs = {ch.get('directory') for ch in current_data.get('channels', []) if ch.get('directory')}
merged_channels = list(current_data.get('channels', []))
for ch in backup_data.get('channels', []):
current_channels = self.backup_entries(current_data, 'channels', config_file, required=CHANNEL_KEYS)
backup_channels = self.backup_entries(backup_data, 'channels', backup_file, required=CHANNEL_KEYS)
current_dirs = {ch.get('directory') for ch in current_channels if ch.get('directory')}
merged_channels = list(current_channels)
for ch in backup_channels:
if ch.get('directory') and ch['directory'] not in current_dirs:
merged_channels.append(ch)

View file

@ -240,6 +240,10 @@ class PlaylistsConfigValidator:
playlists: List[dict] = field(default_factory=list)
# The fields a playlists.yaml entry must have (see `PlaylistsSection`).
PLAYLIST_FIELDS = dict(required=('name',), lists=('items',))
class PlaylistsConfig(ConfigFile):
"""Config file for playlists (Collections with kind='playlist').
@ -361,8 +365,8 @@ class PlaylistsConfig(ConfigFile):
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(playlists=[], version=0)
backup_playlists = backup_data.get('playlists', [])
current_playlists = current_data.get('playlists', [])
backup_playlists = self.backup_entries(backup_data, 'playlists', backup_file, **PLAYLIST_FIELDS)
current_playlists = self.backup_entries(current_data, 'playlists', self.get_file(), **PLAYLIST_FIELDS)
current_names = {p.get('name') for p in current_playlists if p.get('name')}
backup_names = {p.get('name') for p in backup_playlists if p.get('name')}
@ -400,9 +404,11 @@ class PlaylistsConfig(ConfigFile):
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if config_file.is_file() else dict(playlists=[], version=0)
current_names = {p.get('name') for p in current_data.get('playlists', []) if p.get('name')}
merged_playlists = list(current_data.get('playlists', []))
for playlist in backup_data.get('playlists', []):
current_playlists = self.backup_entries(current_data, 'playlists', config_file, **PLAYLIST_FIELDS)
backup_playlists = self.backup_entries(backup_data, 'playlists', backup_file, **PLAYLIST_FIELDS)
current_names = {p.get('name') for p in current_playlists if p.get('name')}
merged_playlists = list(current_playlists)
for playlist in backup_playlists:
if playlist.get('name') and playlist['name'] not in current_names:
merged_playlists.append(playlist)

View file

@ -635,6 +635,49 @@ class ConfigFile:
suffix = pathlib.Path(self.file_name).suffix
return get_media_directory() / f'config/backup/{stem}-{backup_date}{suffix}'
@staticmethod
def backup_entries(data: dict, key: str, file: pathlib.Path, required: Tuple[str, ...] = (),
optional: Tuple[str, ...] = (), lists: Tuple[str, ...] = (), items: int = 0,
text: bool = False) -> list:
"""The entries of `data[key]`, as a backup preview or merge reads them. A malformed entry may be corrupt, so
it is an error naming the file and the entry, not skipped.
@param required: Fields each entry must have, as text.
@param optional: Fields that, when present, must be text.
@param lists: Fields that, when present, must be lists.
@param items: Each entry is a list starting with this many text values (instead of a mapping).
@param text: Each entry is text (instead of a mapping).
"""
where = get_relative_to_media_directory(file)
entries = data.get(key)
if entries is None:
return []
if not isinstance(entries, list):
raise InvalidConfig(f'{where}: "{key}" is not a list')
for number, entry in enumerate(entries, 1):
entry_where = f'{where}: {key} entry {number}'
shown = repr(entry)[:80]
if items:
if not isinstance(entry, list) or len(entry) < items \
or not all(isinstance(i, str) for i in entry[:items]):
raise InvalidConfig(f'{entry_where} must be a list starting with {items} text values: {shown}')
elif text:
if not isinstance(entry, str):
raise InvalidConfig(f'{entry_where} is not text: {shown}')
elif not isinstance(entry, dict):
raise InvalidConfig(f'{entry_where} is not a mapping: {shown}')
else:
for field in required:
if not entry.get(field) or not isinstance(entry[field], str):
raise InvalidConfig(f'{entry_where} needs "{field}" as text: {shown}')
for field in optional:
if entry.get(field) is not None and not isinstance(entry[field], str):
raise InvalidConfig(f'{entry_where} has a "{field}" that is not text: {shown}')
for field in lists:
if entry.get(field) is not None and not isinstance(entry[field], list):
raise InvalidConfig(f'{entry_where} has "{field}" that is not a list: {shown}')
return entries
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
"""Preview what a backup import would do. Subclasses override this."""
raise NotImplementedError(f'{self.__class__.__name__} does not support backup import')

View file

@ -127,6 +127,10 @@ async def post_config_backup_import(_: Request, body: config_schema.ConfigBackup
raise InvalidConfig(f'Backup file not found for date {body.backup_date}')
try:
await asyncio.to_thread(config.import_backup, body.backup_date, body.mode, send_events=True)
except InvalidConfig as e:
# e.g. a malformed entry: the message says which file and entry.
logger.error(f'Failed to import backup: {body.file_name} {body.backup_date}: {e}')
raise
except Exception as e:
logger.error(f'Failed to import backup: {body.file_name} {body.backup_date}', exc_info=e)
raise InvalidConfig(f'Failed to import backup {body.file_name}')

View file

@ -2316,8 +2316,10 @@ class DownloadManagerConfig(ConfigFile):
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(downloads=[], version=0)
backup_downloads = backup_data.get('downloads', [])
current_downloads = current_data.get('downloads', [])
backup_downloads = self.backup_entries(backup_data, 'downloads', backup_file, required=('url',))
current_downloads = self.backup_entries(current_data, 'downloads', self.get_file(), required=('url',))
self.backup_entries(backup_data, 'skip_urls', backup_file, text=True)
self.backup_entries(current_data, 'skip_urls', self.get_file(), text=True)
current_urls = {dl.get('url') for dl in current_downloads if dl.get('url')}
backup_urls = {dl.get('url') for dl in backup_downloads if dl.get('url')}
@ -2359,18 +2361,22 @@ class DownloadManagerConfig(ConfigFile):
current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[],
version=0)
current_urls = {dl.get('url') for dl in current_data.get('downloads', []) if dl.get('url')}
merged_downloads = list(current_data.get('downloads', []))
for dl in backup_data.get('downloads', []):
current_downloads = self.backup_entries(current_data, 'downloads', config_file, required=('url',))
backup_downloads = self.backup_entries(backup_data, 'downloads', backup_file, required=('url',))
current_skip_urls = self.backup_entries(current_data, 'skip_urls', config_file, text=True)
backup_skip_urls = self.backup_entries(backup_data, 'skip_urls', backup_file, text=True)
current_urls = {dl.get('url') for dl in current_downloads if dl.get('url')}
merged_downloads = list(current_downloads)
for dl in backup_downloads:
url = dl.get('url')
# Only merge recurring downloads
if url and url not in current_urls and dl.get('frequency'):
merged_downloads.append(dl)
# Merge skip_urls
current_skip = set(current_data.get('skip_urls', []))
merged_skip = list(current_data.get('skip_urls', []))
for url in backup_data.get('skip_urls', []):
current_skip = set(current_skip_urls)
merged_skip = list(current_skip_urls)
for url in backup_skip_urls:
if url not in current_skip:
merged_skip.append(url)

View file

@ -19,7 +19,7 @@ from wrolpi.config_version import count_config_change, register_ownership
from wrolpi.dates import TZDateTime
from wrolpi.db import get_db_session, named_placeholders, session_curs, serializer
from wrolpi.downloader import save_downloads_config
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile, InvalidConfig
from wrolpi.events import Events
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import PYTEST
@ -765,18 +765,24 @@ class TagsConfig(ConfigFile):
if self._config_outdated:
self.background_dump.activate_switch()
def _backup_parts(self, data: dict, file: pathlib.Path) -> Tuple[dict, list, list]:
"""The tags, tag_files and tag_zims a backup preview or merge reads (see `ConfigFile.backup_entries`)."""
tags = data.get('tags')
if tags is None:
tags = dict()
elif not isinstance(tags, dict):
raise InvalidConfig(f'{get_relative_to_media_directory(file)}: "tags" is not a mapping')
return tags, self.backup_entries(data, 'tag_files', file, items=2), \
self.backup_entries(data, 'tag_zims', file, items=3)
def preview_backup_import(self, backup_date: str, mode: str) -> dict:
backup_file = self._get_backup_file(backup_date)
backup_data = self.read_config_file(backup_file)
current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[],
tag_zims=[])
backup_tags = backup_data.get('tags', {})
current_tags = current_data.get('tags', {})
backup_tag_files = backup_data.get('tag_files', [])
current_tag_files = current_data.get('tag_files', [])
backup_tag_zims = backup_data.get('tag_zims', [])
current_tag_zims = current_data.get('tag_zims', [])
backup_tags, backup_tag_files, backup_tag_zims = self._backup_parts(backup_data, backup_file)
current_tags, current_tag_files, current_tag_zims = self._backup_parts(current_data, self.get_file())
# Build identity sets
current_tag_names = set(current_tags.keys())
@ -841,23 +847,26 @@ class TagsConfig(ConfigFile):
current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[],
tag_zims=[], version=0)
current_tags, current_tag_files, current_tag_zims = self._backup_parts(current_data, config_file)
backup_tags, backup_tag_files, backup_tag_zims = self._backup_parts(backup_data, backup_file)
# Merge tags: add missing tags only
merged_tags = dict(current_data.get('tags', {}))
for name, value in backup_data.get('tags', {}).items():
merged_tags = dict(current_tags)
for name, value in backup_tags.items():
if name not in merged_tags:
merged_tags[name] = value
# Merge tag_files: add missing by (tag_name, path)
current_tf_keys = {(tf[0], tf[1]) for tf in current_data.get('tag_files', []) if len(tf) >= 2}
merged_tag_files = list(current_data.get('tag_files', []))
for tf in backup_data.get('tag_files', []):
current_tf_keys = {(tf[0], tf[1]) for tf in current_tag_files if len(tf) >= 2}
merged_tag_files = list(current_tag_files)
for tf in backup_tag_files:
if len(tf) >= 2 and (tf[0], tf[1]) not in current_tf_keys:
merged_tag_files.append(tf)
# Merge tag_zims: add missing by (tag_name, zim_path, entry)
current_tz_keys = {(tz[0], tz[1], tz[2]) for tz in current_data.get('tag_zims', []) if len(tz) >= 3}
merged_tag_zims = list(current_data.get('tag_zims', []))
for tz in backup_data.get('tag_zims', []):
current_tz_keys = {(tz[0], tz[1], tz[2]) for tz in current_tag_zims if len(tz) >= 3}
merged_tag_zims = list(current_tag_zims)
for tz in backup_tag_zims:
if len(tz) >= 3 and (tz[0], tz[1], tz[2]) not in current_tz_keys:
merged_tag_zims.append(tz)

View file

@ -903,6 +903,67 @@ class TestPreviewWithRealisticData:
assert preview['unchanged'] == 2
MALFORMED = [
# (config, fixture, a valid file, a malformed file, the error names this)
('channels.yaml', 'test_channels_config', dict(channels=[]),
dict(channels=[dict(name='one', directory='videos/one'), 'not a mapping']), 'channels entry 2 is not a mapping'),
('channels.yaml', 'test_channels_config', dict(channels=[]),
dict(channels=[dict(name='one', directory=['videos', 'one'])]), 'channels entry 1 needs "directory"'),
('domains.yaml', None, dict(collections=[]), dict(collections=[5]), 'collections entry 1 is not a mapping'),
('domains.yaml', None, dict(collections=[]), dict(collections=[dict(name='example.com', directory=1)]),
'collections entry 1 has a "directory" that is not text'),
('playlists.yaml', None, dict(playlists=[]), dict(playlists=[dict(name='one', items=5)]),
'playlists entry 1 has "items" that is not a list'),
('download_manager.yaml', 'test_download_manager_config', dict(downloads=[], skip_urls=[]),
dict(downloads=[dict(downloader='archive')], skip_urls=[]), 'downloads entry 1 needs "url"'),
('download_manager.yaml', 'test_download_manager_config', dict(downloads=[], skip_urls=[]),
dict(downloads=[], skip_urls=[['https://example.com']]), 'skip_urls entry 1 is not text'),
('repos.yaml', None, dict(repos=[]), dict(repos='https://example.com/a/b'), '"repos" is not a list'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags=['news'], tag_files=[], tag_zims=[]), '"tags" is not a mapping'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags=[], tag_files=[], tag_zims=[]), '"tags" is not a mapping'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags='', tag_files=[], tag_zims=[]), '"tags" is not a mapping'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags=0, tag_files=[], tag_zims=[]), '"tags" is not a mapping'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags=False, tag_files=[], tag_zims=[]), '"tags" is not a mapping'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags={}, tag_files=[['news']], tag_zims=[]), 'tag_files entry 1 must be a list starting with 2 text'),
('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]),
dict(tags={}, tag_files=[], tag_zims=[['news', 'wiki.zim', 5]]),
'tag_zims entry 1 must be a list starting with 3 text'),
]
@pytest.mark.asyncio
@pytest.mark.parametrize('file_name,fixture,valid,malformed,error', MALFORMED)
@pytest.mark.parametrize('corrupt', ['backup', 'current'])
@pytest.mark.parametrize('endpoint', ['preview', 'import'])
async def test_malformed_entry_is_an_error(request, async_client, test_directory, file_name, fixture, valid,
malformed, error, corrupt, endpoint):
"""A malformed entry, in the backup or the current config, may be corrupt: the preview and the merge refuse it
with an error naming the file and the entry, rather than skip it. The current config is not changed."""
from wrolpi.common import get_config_by_file_name
if fixture:
request.getfixturevalue(fixture)
config = get_config_by_file_name(file_name)
backup, current = (malformed, valid) if corrupt == 'backup' else (valid, malformed)
_make_backup(test_directory, file_name, '20260301', dict(backup, version=1))
_write_yaml(config.get_file(), dict(current, version=1))
before = config.get_file().read_text()
body = dict(file_name=file_name, backup_date='20260301', mode='merge')
_, response = await async_client.post(f'/api/config/backup/{endpoint}', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
stem = pathlib.Path(file_name).stem
name = f'config/backup/{stem}-20260301.yaml' if corrupt == 'backup' else f'config/{file_name}'
assert f'{name}: {error}' in response.json['error']
assert config.get_file().read_text() == before
class TestWriteConfigData:
def test_rejects_python_objects(self, test_directory, test_tags_config):