A malformed config entry is skipped and counted instead of failing the import (#875)

* A malformed config entry is skipped and counted instead of failing the import

An entry that is not a mapping (or whose key is not text, or a list that is not a list) made the whole file fail to
import in playlists.yaml, repos.yaml, download_manager.yaml and tags.yaml (tag_files/tag_zims), and a non-text name
did in domains.yaml.  Each is now skipped and counted in import_skipped, so the config is not saved over it, and the
rest of the file imports.  A Channel whose `downloads` is not a list keeps its Downloads.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* An unreadable entry keeps no row and does not make a section present

The key of an entry too malformed to name a row is now `Unreadable(position)`.  It is still counted in
import_skipped, but `present` ignores it, so a section whose entries are all unreadable removes nothing (as an empty
section), instead of deleting every row.  A playlist whose `items` is not a list keeps its items, and a Channel
Download whose url is not text is counted (its Channel keeps its Downloads) instead of failing the import.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-25 16:28:30 -06:00 • committed by Roland
parent 0dd61b8e64
commit 79526afebe
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
10 changed files with 279 additions and 35 deletions

View file

@ -28,7 +28,7 @@ from wrolpi.common import get_media_directory, get_relative_to_media_directory,
escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name, chunks
from wrolpi.collections.models import validate_collection_directory
from wrolpi.config_import import ConfigSection, media_relative
from wrolpi.config_import import ConfigSection, Unreadable, media_relative
from wrolpi.dates import now, Seconds
from wrolpi.db import get_db_session, get_db_curs, session_curs
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
@ -65,10 +65,10 @@ class DomainsSection(ConfigSection):
rows, invalid = dict(), set()
for position, data in enumerate(config.get('collections') or []):
name = data.get('name') if isinstance(data, dict) else None
if not name:
if not name or not isinstance(name, str):
logger.error(f'Domain collection at index {position} has no name, skipping: {data!r}')
# Counted as skipped; never a domain's name.
invalid.add(f'\0{position}')
# Counted as skipped (see `Unreadable`).
invalid.add(Unreadable(position))
continue
try:
if not Collection.is_valid_domain_name(name):

View file

@ -25,7 +25,7 @@ from wrolpi.cmd import GIT_BIN, run_command
from wrolpi.collections.models import Collection
from wrolpi.common import ConfigFile, get_media_directory, get_relative_to_media_directory, logger, \
escape_file_name, background_task, get_wrolpi_config
from wrolpi.config_import import ConfigSection, media_relative
from wrolpi.config_import import ConfigSection, Unreadable, media_relative
from wrolpi.config_version import register_ownership
from wrolpi.db import get_db_session
from wrolpi.downloader import Download, DownloadFrequency, download_manager, save_downloads_config
@ -1397,11 +1397,11 @@ class ReposSection(ConfigSection):
media_directory = get_media_directory()
rows, invalid = dict(), set()
for position, data in enumerate(config.get('repos') or []):
url = data.get('url')
if not url:
logger.error(f'Repo config entry has no url, skipping: {data}')
# Counted as skipped; never a Repo's key.
invalid.add(f'\0{position}')
url = data.get('url') if isinstance(data, dict) else None
if not url or not isinstance(url, str):
logger.error(f'Repo config entry has no url, skipping: {data!r}')
# Counted as skipped (see `Unreadable`).
invalid.add(Unreadable(position))
continue
try:
parsed = parse_repo_url(url)

View file

@ -102,6 +102,34 @@ async def test_invalid_entry_is_kept(test_session, saved_repos):
assert (found[TWO].mode, found[ONE].collection.description) == ('snapshot', 'Changed')
@pytest.mark.asyncio
async def test_malformed_entry_is_skipped(test_session, saved_repos):
"""An entry that is not a mapping is skipped and counted; the rest of the file still imports."""
edit_repo(saved_repos, ONE, description='Changed')
edit_config(saved_repos, lambda data: data['repos'].insert(0, 'not a mapping'))
config = lib.get_repos_config()
config.import_config()
assert config.successful_import is True
assert config.import_skipped == 1
found = repos(test_session)
assert set(found) == {ONE, TWO}
assert found[ONE].collection.description == 'Changed'
@pytest.mark.asyncio
async def test_only_malformed_entries_delete_nothing(test_session, saved_repos):
"""A file whose entries are all unreadable deletes no Repo (like an empty list)."""
edit_config(saved_repos, lambda data: data.update(repos=['not a mapping', dict(url=['a', 'list'])]))
config = lib.get_repos_config()
config.import_config()
assert config.import_skipped == 2
assert set(repos(test_session)) == {ONE, TWO}
@pytest.mark.asyncio
async def test_failed_import_creates_nothing(test_session, saved_repos, monkeypatch):
"""A new Repo is created in the import's one write: a failure later in the write leaves nothing (each new Repo

View file

@ -20,7 +20,7 @@ from wrolpi.common import ConfigFile, extract_domain, logger, \
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config, trim_file_name
from wrolpi.collections import Collection
from wrolpi.collections.models import validate_collection_directory
from wrolpi.config_import import ConfigSection, chunks, media_relative
from wrolpi.config_import import ConfigSection, Unreadable, chunks, media_relative
from wrolpi.config_version import register_ownership
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session, session_curs
@ -295,8 +295,8 @@ class ChannelsSection(ConfigSection):
key = _channel_key(data)
except Exception as e:
channel_import_logger.error(f'Skipping invalid Channel in config: {data!r}', exc_info=e)
# Counted as skipped; never a directory. (The entry's name keeps its Channel, see `prepare`.)
invalid.add(f'\0{position}')
# Counted as skipped (see `Unreadable`). The entry's name keeps its Channel (see `prepare`).
invalid.add(Unreadable(position))
continue
if key in rows or key in invalid:
raise RuntimeError('Refusing to import channels config because it contains duplicate directories!')
@ -394,8 +394,14 @@ class ChannelDownloadsSection(ConfigSection):
key = _channel_key(data)
except Exception:
continue
for position, download in enumerate(data.get('downloads') or []):
if isinstance(download, dict) and download.get('url'):
downloads = data.get('downloads') or []
if not isinstance(downloads, list):
channel_import_logger.error(f'Skipping the Downloads of Channel {key!r}, they are not a list')
# Counted as skipped; the Channel keeps its Downloads (see `ChannelsConfig.prepare`).
invalid.add((key, '\0downloads'))
continue
for position, download in enumerate(downloads):
if isinstance(download, dict) and download.get('url') and isinstance(download['url'], str):
rows[(key, download['url'])] = dict(frequency=download.get('frequency'))
else:
channel_import_logger.error(f'Skipping invalid Download of Channel {key!r}: {download!r}')