diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 76640fa4..f45b3a42 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -129,6 +129,10 @@ class DomainsConfigValidator: collections: List[dict] = field(default_factory=list) +# The fields a domains.yaml entry must have (see `DomainsSection`). +DOMAIN_FIELDS = dict(required=('name',), optional=('directory',)) + + class DomainsConfig(ConfigFile): """ Config file for Domain Collections. @@ -229,8 +233,8 @@ class DomainsConfig(ConfigFile): backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if self.get_file().is_file() else dict(collections=[], version=0) - backup_collections = backup_data.get('collections', []) - current_collections = current_data.get('collections', []) + backup_collections = self.backup_entries(backup_data, 'collections', backup_file, **DOMAIN_FIELDS) + current_collections = self.backup_entries(current_data, 'collections', self.get_file(), **DOMAIN_FIELDS) # Identity key for domains is the directory (if present) or name def get_key(c): @@ -273,9 +277,11 @@ class DomainsConfig(ConfigFile): def get_key(c): return c.get('directory') or c.get('name', '') - current_keys = {get_key(c) for c in current_data.get('collections', [])} - merged_collections = list(current_data.get('collections', [])) - for c in backup_data.get('collections', []): + current_collections = self.backup_entries(current_data, 'collections', config_file, **DOMAIN_FIELDS) + backup_collections = self.backup_entries(backup_data, 'collections', backup_file, **DOMAIN_FIELDS) + current_keys = {get_key(c) for c in current_collections} + merged_collections = list(current_collections) + for c in backup_collections: if get_key(c) not in current_keys: merged_collections.append(c) diff --git a/modules/repos/lib.py b/modules/repos/lib.py index 2b2f8126..0d7dd77b 100644 --- a/modules/repos/lib.py +++ b/modules/repos/lib.py @@ -1542,11 +1542,14 @@ class ReposConfig(ConfigFile): def preview_backup_import(self, backup_date: str, mode: str) -> dict: """Preview which Repos a backup import would add/remove (keyed by URL).""" - backup_data = self.read_config_file(self._get_backup_file(backup_date)) + backup_file = self._get_backup_file(backup_date) + backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if self.get_file().is_file() else dict(repos=[], version=0) - backup_repos = {repo_url_key(i['url']): i for i in backup_data.get('repos', []) if i.get('url')} - current_repos = {repo_url_key(i['url']): i for i in current_data.get('repos', []) if i.get('url')} + backup_repos = {repo_url_key(i['url']): i for i in + self.backup_entries(backup_data, 'repos', backup_file, required=('url',))} + current_repos = {repo_url_key(i['url']): i for i in + self.backup_entries(current_data, 'repos', self.get_file(), required=('url',))} add = [dict(type='repo', name=i.get('name', ''), url=i['url']) for key, i in backup_repos.items() if key not in current_repos] @@ -1569,10 +1572,11 @@ class ReposConfig(ConfigFile): elif mode == 'merge': backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if config_file.is_file() else dict(repos=[], version=0) - current_keys = {repo_url_key(i['url']) for i in current_data.get('repos', []) if i.get('url')} - merged = list(current_data.get('repos', [])) - merged.extend(i for i in backup_data.get('repos', []) - if i.get('url') and repo_url_key(i['url']) not in current_keys) + current_repos = self.backup_entries(current_data, 'repos', config_file, required=('url',)) + backup_repos = self.backup_entries(backup_data, 'repos', backup_file, required=('url',)) + current_keys = {repo_url_key(i['url']) for i in current_repos} + merged = list(current_repos) + merged.extend(i for i in backup_repos if repo_url_key(i['url']) not in current_keys) self.write_config_data(dict(repos=merged, version=current_data.get('version', 0) + 1), config_file) # Importing a backup is the user's choice: apply it whatever the versions. diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 38fc58c7..22101b35 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -451,6 +451,10 @@ class ChannelsConfigValidator: channels: list[ChannelDictValidator] = dataclasses.field(default_factory=list) +# The fields a channels.yaml entry must have (see `ChannelsSection`). +CHANNEL_KEYS = ('directory', 'name') + + class ChannelsConfig(ConfigFile): file_name = 'channels.yaml' sections = (ChannelsSection(), ChannelDownloadsSection()) @@ -609,8 +613,8 @@ class ChannelsConfig(ConfigFile): backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if self.get_file().is_file() else dict(channels=[], version=0) - backup_channels = backup_data.get('channels', []) - current_channels = current_data.get('channels', []) + backup_channels = self.backup_entries(backup_data, 'channels', backup_file, required=CHANNEL_KEYS) + current_channels = self.backup_entries(current_data, 'channels', self.get_file(), required=CHANNEL_KEYS) current_dirs = {ch.get('directory') for ch in current_channels if ch.get('directory')} backup_dirs = {ch.get('directory') for ch in backup_channels if ch.get('directory')} @@ -645,9 +649,11 @@ class ChannelsConfig(ConfigFile): backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if config_file.is_file() else dict(channels=[], version=0) - current_dirs = {ch.get('directory') for ch in current_data.get('channels', []) if ch.get('directory')} - merged_channels = list(current_data.get('channels', [])) - for ch in backup_data.get('channels', []): + current_channels = self.backup_entries(current_data, 'channels', config_file, required=CHANNEL_KEYS) + backup_channels = self.backup_entries(backup_data, 'channels', backup_file, required=CHANNEL_KEYS) + current_dirs = {ch.get('directory') for ch in current_channels if ch.get('directory')} + merged_channels = list(current_channels) + for ch in backup_channels: if ch.get('directory') and ch['directory'] not in current_dirs: merged_channels.append(ch) diff --git a/wrolpi/collections/config.py b/wrolpi/collections/config.py index 0d99a248..1ca470eb 100644 --- a/wrolpi/collections/config.py +++ b/wrolpi/collections/config.py @@ -240,6 +240,10 @@ class PlaylistsConfigValidator: playlists: List[dict] = field(default_factory=list) +# The fields a playlists.yaml entry must have (see `PlaylistsSection`). +PLAYLIST_FIELDS = dict(required=('name',), lists=('items',)) + + class PlaylistsConfig(ConfigFile): """Config file for playlists (Collections with kind='playlist'). @@ -361,8 +365,8 @@ class PlaylistsConfig(ConfigFile): backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if self.get_file().is_file() else dict(playlists=[], version=0) - backup_playlists = backup_data.get('playlists', []) - current_playlists = current_data.get('playlists', []) + backup_playlists = self.backup_entries(backup_data, 'playlists', backup_file, **PLAYLIST_FIELDS) + current_playlists = self.backup_entries(current_data, 'playlists', self.get_file(), **PLAYLIST_FIELDS) current_names = {p.get('name') for p in current_playlists if p.get('name')} backup_names = {p.get('name') for p in backup_playlists if p.get('name')} @@ -400,9 +404,11 @@ class PlaylistsConfig(ConfigFile): backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if config_file.is_file() else dict(playlists=[], version=0) - current_names = {p.get('name') for p in current_data.get('playlists', []) if p.get('name')} - merged_playlists = list(current_data.get('playlists', [])) - for playlist in backup_data.get('playlists', []): + current_playlists = self.backup_entries(current_data, 'playlists', config_file, **PLAYLIST_FIELDS) + backup_playlists = self.backup_entries(backup_data, 'playlists', backup_file, **PLAYLIST_FIELDS) + current_names = {p.get('name') for p in current_playlists if p.get('name')} + merged_playlists = list(current_playlists) + for playlist in backup_playlists: if playlist.get('name') and playlist['name'] not in current_names: merged_playlists.append(playlist) diff --git a/wrolpi/common.py b/wrolpi/common.py index e08721a9..ea7475af 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -635,6 +635,49 @@ class ConfigFile: suffix = pathlib.Path(self.file_name).suffix return get_media_directory() / f'config/backup/{stem}-{backup_date}{suffix}' + @staticmethod + def backup_entries(data: dict, key: str, file: pathlib.Path, required: Tuple[str, ...] = (), + optional: Tuple[str, ...] = (), lists: Tuple[str, ...] = (), items: int = 0, + text: bool = False) -> list: + """The entries of `data[key]`, as a backup preview or merge reads them. A malformed entry may be corrupt, so + it is an error naming the file and the entry, not skipped. + + @param required: Fields each entry must have, as text. + @param optional: Fields that, when present, must be text. + @param lists: Fields that, when present, must be lists. + @param items: Each entry is a list starting with this many text values (instead of a mapping). + @param text: Each entry is text (instead of a mapping). + """ + where = get_relative_to_media_directory(file) + entries = data.get(key) + if entries is None: + return [] + if not isinstance(entries, list): + raise InvalidConfig(f'{where}: "{key}" is not a list') + for number, entry in enumerate(entries, 1): + entry_where = f'{where}: {key} entry {number}' + shown = repr(entry)[:80] + if items: + if not isinstance(entry, list) or len(entry) < items \ + or not all(isinstance(i, str) for i in entry[:items]): + raise InvalidConfig(f'{entry_where} must be a list starting with {items} text values: {shown}') + elif text: + if not isinstance(entry, str): + raise InvalidConfig(f'{entry_where} is not text: {shown}') + elif not isinstance(entry, dict): + raise InvalidConfig(f'{entry_where} is not a mapping: {shown}') + else: + for field in required: + if not entry.get(field) or not isinstance(entry[field], str): + raise InvalidConfig(f'{entry_where} needs "{field}" as text: {shown}') + for field in optional: + if entry.get(field) is not None and not isinstance(entry[field], str): + raise InvalidConfig(f'{entry_where} has a "{field}" that is not text: {shown}') + for field in lists: + if entry.get(field) is not None and not isinstance(entry[field], list): + raise InvalidConfig(f'{entry_where} has "{field}" that is not a list: {shown}') + return entries + def preview_backup_import(self, backup_date: str, mode: str) -> dict: """Preview what a backup import would do. Subclasses override this.""" raise NotImplementedError(f'{self.__class__.__name__} does not support backup import') diff --git a/wrolpi/config_api.py b/wrolpi/config_api.py index 7564c49e..2a5ebd0f 100644 --- a/wrolpi/config_api.py +++ b/wrolpi/config_api.py @@ -127,6 +127,10 @@ async def post_config_backup_import(_: Request, body: config_schema.ConfigBackup raise InvalidConfig(f'Backup file not found for date {body.backup_date}') try: await asyncio.to_thread(config.import_backup, body.backup_date, body.mode, send_events=True) + except InvalidConfig as e: + # e.g. a malformed entry: the message says which file and entry. + logger.error(f'Failed to import backup: {body.file_name} {body.backup_date}: {e}') + raise except Exception as e: logger.error(f'Failed to import backup: {body.file_name} {body.backup_date}', exc_info=e) raise InvalidConfig(f'Failed to import backup {body.file_name}') diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index a89c61ee..cd99de5f 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -2316,8 +2316,10 @@ class DownloadManagerConfig(ConfigFile): backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if self.get_file().is_file() else dict(downloads=[], version=0) - backup_downloads = backup_data.get('downloads', []) - current_downloads = current_data.get('downloads', []) + backup_downloads = self.backup_entries(backup_data, 'downloads', backup_file, required=('url',)) + current_downloads = self.backup_entries(current_data, 'downloads', self.get_file(), required=('url',)) + self.backup_entries(backup_data, 'skip_urls', backup_file, text=True) + self.backup_entries(current_data, 'skip_urls', self.get_file(), text=True) current_urls = {dl.get('url') for dl in current_downloads if dl.get('url')} backup_urls = {dl.get('url') for dl in backup_downloads if dl.get('url')} @@ -2359,18 +2361,22 @@ class DownloadManagerConfig(ConfigFile): current_data = self.read_config_file() if config_file.is_file() else dict(downloads=[], skip_urls=[], version=0) - current_urls = {dl.get('url') for dl in current_data.get('downloads', []) if dl.get('url')} - merged_downloads = list(current_data.get('downloads', [])) - for dl in backup_data.get('downloads', []): + current_downloads = self.backup_entries(current_data, 'downloads', config_file, required=('url',)) + backup_downloads = self.backup_entries(backup_data, 'downloads', backup_file, required=('url',)) + current_skip_urls = self.backup_entries(current_data, 'skip_urls', config_file, text=True) + backup_skip_urls = self.backup_entries(backup_data, 'skip_urls', backup_file, text=True) + current_urls = {dl.get('url') for dl in current_downloads if dl.get('url')} + merged_downloads = list(current_downloads) + for dl in backup_downloads: url = dl.get('url') # Only merge recurring downloads if url and url not in current_urls and dl.get('frequency'): merged_downloads.append(dl) # Merge skip_urls - current_skip = set(current_data.get('skip_urls', [])) - merged_skip = list(current_data.get('skip_urls', [])) - for url in backup_data.get('skip_urls', []): + current_skip = set(current_skip_urls) + merged_skip = list(current_skip_urls) + for url in backup_skip_urls: if url not in current_skip: merged_skip.append(url) diff --git a/wrolpi/tags.py b/wrolpi/tags.py index a858e1f5..9d05b5fa 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -19,7 +19,7 @@ from wrolpi.config_version import count_config_change, register_ownership from wrolpi.dates import TZDateTime from wrolpi.db import get_db_session, named_placeholders, session_curs, serializer from wrolpi.downloader import save_downloads_config -from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile +from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileWorkerConflict, NoPrimaryFile, InvalidConfig from wrolpi.events import Events from wrolpi.switches import register_switch_handler, ActivateSwitchMethod from wrolpi.vars import PYTEST @@ -765,18 +765,24 @@ class TagsConfig(ConfigFile): if self._config_outdated: self.background_dump.activate_switch() + def _backup_parts(self, data: dict, file: pathlib.Path) -> Tuple[dict, list, list]: + """The tags, tag_files and tag_zims a backup preview or merge reads (see `ConfigFile.backup_entries`).""" + tags = data.get('tags') + if tags is None: + tags = dict() + elif not isinstance(tags, dict): + raise InvalidConfig(f'{get_relative_to_media_directory(file)}: "tags" is not a mapping') + return tags, self.backup_entries(data, 'tag_files', file, items=2), \ + self.backup_entries(data, 'tag_zims', file, items=3) + def preview_backup_import(self, backup_date: str, mode: str) -> dict: backup_file = self._get_backup_file(backup_date) backup_data = self.read_config_file(backup_file) current_data = self.read_config_file() if self.get_file().is_file() else dict(tags={}, tag_files=[], tag_zims=[]) - backup_tags = backup_data.get('tags', {}) - current_tags = current_data.get('tags', {}) - backup_tag_files = backup_data.get('tag_files', []) - current_tag_files = current_data.get('tag_files', []) - backup_tag_zims = backup_data.get('tag_zims', []) - current_tag_zims = current_data.get('tag_zims', []) + backup_tags, backup_tag_files, backup_tag_zims = self._backup_parts(backup_data, backup_file) + current_tags, current_tag_files, current_tag_zims = self._backup_parts(current_data, self.get_file()) # Build identity sets current_tag_names = set(current_tags.keys()) @@ -841,23 +847,26 @@ class TagsConfig(ConfigFile): current_data = self.read_config_file() if config_file.is_file() else dict(tags={}, tag_files=[], tag_zims=[], version=0) + current_tags, current_tag_files, current_tag_zims = self._backup_parts(current_data, config_file) + backup_tags, backup_tag_files, backup_tag_zims = self._backup_parts(backup_data, backup_file) + # Merge tags: add missing tags only - merged_tags = dict(current_data.get('tags', {})) - for name, value in backup_data.get('tags', {}).items(): + merged_tags = dict(current_tags) + for name, value in backup_tags.items(): if name not in merged_tags: merged_tags[name] = value # Merge tag_files: add missing by (tag_name, path) - current_tf_keys = {(tf[0], tf[1]) for tf in current_data.get('tag_files', []) if len(tf) >= 2} - merged_tag_files = list(current_data.get('tag_files', [])) - for tf in backup_data.get('tag_files', []): + current_tf_keys = {(tf[0], tf[1]) for tf in current_tag_files if len(tf) >= 2} + merged_tag_files = list(current_tag_files) + for tf in backup_tag_files: if len(tf) >= 2 and (tf[0], tf[1]) not in current_tf_keys: merged_tag_files.append(tf) # Merge tag_zims: add missing by (tag_name, zim_path, entry) - current_tz_keys = {(tz[0], tz[1], tz[2]) for tz in current_data.get('tag_zims', []) if len(tz) >= 3} - merged_tag_zims = list(current_data.get('tag_zims', [])) - for tz in backup_data.get('tag_zims', []): + current_tz_keys = {(tz[0], tz[1], tz[2]) for tz in current_tag_zims if len(tz) >= 3} + merged_tag_zims = list(current_tag_zims) + for tz in backup_tag_zims: if len(tz) >= 3 and (tz[0], tz[1], tz[2]) not in current_tz_keys: merged_tag_zims.append(tz) diff --git a/wrolpi/test/test_config_backup_import.py b/wrolpi/test/test_config_backup_import.py index 09ef6dc2..29d9f896 100644 --- a/wrolpi/test/test_config_backup_import.py +++ b/wrolpi/test/test_config_backup_import.py @@ -903,6 +903,67 @@ class TestPreviewWithRealisticData: assert preview['unchanged'] == 2 +MALFORMED = [ + # (config, fixture, a valid file, a malformed file, the error names this) + ('channels.yaml', 'test_channels_config', dict(channels=[]), + dict(channels=[dict(name='one', directory='videos/one'), 'not a mapping']), 'channels entry 2 is not a mapping'), + ('channels.yaml', 'test_channels_config', dict(channels=[]), + dict(channels=[dict(name='one', directory=['videos', 'one'])]), 'channels entry 1 needs "directory"'), + ('domains.yaml', None, dict(collections=[]), dict(collections=[5]), 'collections entry 1 is not a mapping'), + ('domains.yaml', None, dict(collections=[]), dict(collections=[dict(name='example.com', directory=1)]), + 'collections entry 1 has a "directory" that is not text'), + ('playlists.yaml', None, dict(playlists=[]), dict(playlists=[dict(name='one', items=5)]), + 'playlists entry 1 has "items" that is not a list'), + ('download_manager.yaml', 'test_download_manager_config', dict(downloads=[], skip_urls=[]), + dict(downloads=[dict(downloader='archive')], skip_urls=[]), 'downloads entry 1 needs "url"'), + ('download_manager.yaml', 'test_download_manager_config', dict(downloads=[], skip_urls=[]), + dict(downloads=[], skip_urls=[['https://example.com']]), 'skip_urls entry 1 is not text'), + ('repos.yaml', None, dict(repos=[]), dict(repos='https://example.com/a/b'), '"repos" is not a list'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags=['news'], tag_files=[], tag_zims=[]), '"tags" is not a mapping'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags=[], tag_files=[], tag_zims=[]), '"tags" is not a mapping'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags='', tag_files=[], tag_zims=[]), '"tags" is not a mapping'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags=0, tag_files=[], tag_zims=[]), '"tags" is not a mapping'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags=False, tag_files=[], tag_zims=[]), '"tags" is not a mapping'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags={}, tag_files=[['news']], tag_zims=[]), 'tag_files entry 1 must be a list starting with 2 text'), + ('tags.yaml', 'test_tags_config', dict(tags={}, tag_files=[], tag_zims=[]), + dict(tags={}, tag_files=[], tag_zims=[['news', 'wiki.zim', 5]]), + 'tag_zims entry 1 must be a list starting with 3 text'), +] + + +@pytest.mark.asyncio +@pytest.mark.parametrize('file_name,fixture,valid,malformed,error', MALFORMED) +@pytest.mark.parametrize('corrupt', ['backup', 'current']) +@pytest.mark.parametrize('endpoint', ['preview', 'import']) +async def test_malformed_entry_is_an_error(request, async_client, test_directory, file_name, fixture, valid, + malformed, error, corrupt, endpoint): + """A malformed entry, in the backup or the current config, may be corrupt: the preview and the merge refuse it + with an error naming the file and the entry, rather than skip it. The current config is not changed.""" + from wrolpi.common import get_config_by_file_name + if fixture: + request.getfixturevalue(fixture) + config = get_config_by_file_name(file_name) + backup, current = (malformed, valid) if corrupt == 'backup' else (valid, malformed) + _make_backup(test_directory, file_name, '20260301', dict(backup, version=1)) + _write_yaml(config.get_file(), dict(current, version=1)) + before = config.get_file().read_text() + + body = dict(file_name=file_name, backup_date='20260301', mode='merge') + _, response = await async_client.post(f'/api/config/backup/{endpoint}', json=body) + + assert response.status_code == HTTPStatus.BAD_REQUEST + stem = pathlib.Path(file_name).stem + name = f'config/backup/{stem}-20260301.yaml' if corrupt == 'backup' else f'config/{file_name}' + assert f'{name}: {error}' in response.json['error'] + assert config.get_file().read_text() == before + + class TestWriteConfigData: def test_rejects_python_objects(self, test_directory, test_tags_config):