mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
An unreadable entry keeps no row and does not make a section present
The key of an entry too malformed to name a row is now `Unreadable(position)`. It is still counted in import_skipped, but `present` ignores it, so a section whose entries are all unreadable removes nothing (as an empty section), instead of deleting every row. A playlist whose `items` is not a list keeps its items, and a Channel Download whose url is not text is counted (its Channel keeps its Downloads) instead of failing the import. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
a787c02908
commit
96d5aa3a99
10 changed files with 134 additions and 24 deletions
|
|
@ -28,7 +28,7 @@ from wrolpi.common import get_media_directory, get_relative_to_media_directory,
|
|||
escape_file_name, aiohttp_post, format_html_string, split_lines_by_length, get_html_soup, get_title_from_html, \
|
||||
get_wrolpi_config, html_screenshot, html_file_screenshot, ConfigFile, trim_file_name, chunks
|
||||
from wrolpi.collections.models import validate_collection_directory
|
||||
from wrolpi.config_import import ConfigSection, media_relative
|
||||
from wrolpi.config_import import ConfigSection, Unreadable, media_relative
|
||||
from wrolpi.dates import now, Seconds
|
||||
from wrolpi.db import get_db_session, get_db_curs, session_curs
|
||||
from wrolpi.errors import UnknownArchive, InvalidOrderBy, InvalidDatetime
|
||||
|
|
@ -67,8 +67,8 @@ class DomainsSection(ConfigSection):
|
|||
name = data.get('name') if isinstance(data, dict) else None
|
||||
if not name or not isinstance(name, str):
|
||||
logger.error(f'Domain collection at index {position} has no name, skipping: {data!r}')
|
||||
# Counted as skipped; never a domain's name.
|
||||
invalid.add(f'\0{position}')
|
||||
# Counted as skipped (see `Unreadable`).
|
||||
invalid.add(Unreadable(position))
|
||||
continue
|
||||
try:
|
||||
if not Collection.is_valid_domain_name(name):
|
||||
|
|
|
|||
|
|
@ -25,7 +25,7 @@ from wrolpi.cmd import GIT_BIN, run_command
|
|||
from wrolpi.collections.models import Collection
|
||||
from wrolpi.common import ConfigFile, get_media_directory, get_relative_to_media_directory, logger, \
|
||||
escape_file_name, background_task, get_wrolpi_config
|
||||
from wrolpi.config_import import ConfigSection, media_relative
|
||||
from wrolpi.config_import import ConfigSection, Unreadable, media_relative
|
||||
from wrolpi.config_version import register_ownership
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import Download, DownloadFrequency, download_manager, save_downloads_config
|
||||
|
|
@ -1400,8 +1400,8 @@ class ReposSection(ConfigSection):
|
|||
url = data.get('url') if isinstance(data, dict) else None
|
||||
if not url or not isinstance(url, str):
|
||||
logger.error(f'Repo config entry has no url, skipping: {data!r}')
|
||||
# Counted as skipped; never a Repo's key.
|
||||
invalid.add(f'\0{position}')
|
||||
# Counted as skipped (see `Unreadable`).
|
||||
invalid.add(Unreadable(position))
|
||||
continue
|
||||
try:
|
||||
parsed = parse_repo_url(url)
|
||||
|
|
|
|||
|
|
@ -118,6 +118,18 @@ async def test_malformed_entry_is_skipped(test_session, saved_repos):
|
|||
assert found[ONE].collection.description == 'Changed'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_only_malformed_entries_delete_nothing(test_session, saved_repos):
|
||||
"""A file whose entries are all unreadable deletes no Repo (like an empty list)."""
|
||||
edit_config(saved_repos, lambda data: data.update(repos=['not a mapping', dict(url=['a', 'list'])]))
|
||||
|
||||
config = lib.get_repos_config()
|
||||
config.import_config()
|
||||
|
||||
assert config.import_skipped == 2
|
||||
assert set(repos(test_session)) == {ONE, TWO}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_failed_import_creates_nothing(test_session, saved_repos, monkeypatch):
|
||||
"""A new Repo is created in the import's one write: a failure later in the write leaves nothing (each new Repo
|
||||
|
|
|
|||
|
|
@ -20,7 +20,7 @@ from wrolpi.common import ConfigFile, extract_domain, logger, \
|
|||
escape_file_name, get_media_directory, background_task, Base, get_wrolpi_config, trim_file_name
|
||||
from wrolpi.collections import Collection
|
||||
from wrolpi.collections.models import validate_collection_directory
|
||||
from wrolpi.config_import import ConfigSection, chunks, media_relative
|
||||
from wrolpi.config_import import ConfigSection, Unreadable, chunks, media_relative
|
||||
from wrolpi.config_version import register_ownership
|
||||
from wrolpi.dates import Seconds, from_timestamp
|
||||
from wrolpi.db import get_db_curs, get_db_session, session_curs
|
||||
|
|
@ -295,8 +295,8 @@ class ChannelsSection(ConfigSection):
|
|||
key = _channel_key(data)
|
||||
except Exception as e:
|
||||
channel_import_logger.error(f'Skipping invalid Channel in config: {data!r}', exc_info=e)
|
||||
# Counted as skipped; never a directory. (The entry's name keeps its Channel, see `prepare`.)
|
||||
invalid.add(f'\0{position}')
|
||||
# Counted as skipped (see `Unreadable`). The entry's name keeps its Channel (see `prepare`).
|
||||
invalid.add(Unreadable(position))
|
||||
continue
|
||||
if key in rows or key in invalid:
|
||||
raise RuntimeError('Refusing to import channels config because it contains duplicate directories!')
|
||||
|
|
@ -401,7 +401,7 @@ class ChannelDownloadsSection(ConfigSection):
|
|||
invalid.add((key, '\0downloads'))
|
||||
continue
|
||||
for position, download in enumerate(downloads):
|
||||
if isinstance(download, dict) and download.get('url'):
|
||||
if isinstance(download, dict) and download.get('url') and isinstance(download['url'], str):
|
||||
rows[(key, download['url'])] = dict(frequency=download.get('frequency'))
|
||||
else:
|
||||
channel_import_logger.error(f'Skipping invalid Download of Channel {key!r}: {download!r}')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue