diff --git a/main.py b/main.py index 282966d1..d2b72d39 100755 --- a/main.py +++ b/main.py @@ -268,6 +268,10 @@ async def start_single_tasks(app: Sanic): from modules.archive.lib import get_archive_downloader_config get_archive_downloader_config().import_config() logger.debug('archive downloader config imported') + with log_and_suppress(Exception, message='Failed to import download cache config'): + from wrolpi.downloader import get_download_cache_config + get_download_cache_config().import_config() + logger.debug('download cache config imported') from modules.zim.lib import flag_outdated_zim_files try: diff --git a/wrolpi/common.py b/wrolpi/common.py index 049f4007..d2f3126f 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -450,7 +450,8 @@ def get_media_directory() -> Path: return MEDIA_DIRECTORY -DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'inventories.yaml'} +DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'inventories.yaml', + 'collections.yaml'} class ConfigFile: @@ -846,6 +847,14 @@ def get_all_configs() -> Dict[str, ConfigFile]: if archive_downloader_config := get_archive_downloader_config(): all_configs[archive_downloader_config.file_name] = archive_downloader_config + from wrolpi.downloader import get_download_cache_config + if download_cache_config := get_download_cache_config(): + all_configs[download_cache_config.file_name] = download_cache_config + + from wrolpi.collections.config import collections_config as _collections_config + if _collections_config: + all_configs[_collections_config.file_name] = _collections_config + return all_configs @@ -918,6 +927,16 @@ async def import_all_db_configs() -> dict[str, bool]: logger.warning(f'Failed to import inventories config: {e}') results['inventories'] = False + # Collections (uses tags for tag_name) + try: + from wrolpi.collections.config import collections_config as _collections_config + _collections_config.import_config() + results['collections'] = True + logger.debug('collections config imported') + except Exception as e: + logger.warning(f'Failed to import collections config: {e}') + results['collections'] = False + return results diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index 83f86046..060d8d24 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -31,6 +31,8 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.videos_downloader_config = manager.dict() app.shared_ctx.domains_config = manager.dict() app.shared_ctx.archive_downloader_config = manager.dict() + app.shared_ctx.download_cache_config = manager.dict() + app.shared_ctx.collections_config = manager.dict() # Shared dicts. app.shared_ctx.refresh = manager.dict() app.shared_ctx.uploaded_files = manager.dict() @@ -102,6 +104,8 @@ def reset_shared_contexts(app: Sanic): app.shared_ctx.videos_downloader_config.clear() app.shared_ctx.domains_config.clear() app.shared_ctx.archive_downloader_config.clear() + app.shared_ctx.download_cache_config.clear() + app.shared_ctx.collections_config.clear() # Shared dicts. app.shared_ctx.refresh.clear() app.shared_ctx.uploaded_files.clear() @@ -255,3 +259,15 @@ def initialize_configs_contexts(app: Sanic): ARCHIVE_DOWNLOADER_CONFIG.initialize(app.shared_ctx.archive_downloader_config) except Exception as e: logger.error(f'Failed to initialize in-memory archive downloader config: {e}') + + try: + from wrolpi.downloader import DOWNLOAD_CACHE_CONFIG + DOWNLOAD_CACHE_CONFIG.initialize(app.shared_ctx.download_cache_config) + except Exception as e: + logger.error(f'Failed to initialize in-memory download cache config: {e}') + + try: + from wrolpi.collections.config import collections_config as _collections_config + _collections_config.initialize(app.shared_ctx.collections_config) + except Exception as e: + logger.error(f'Failed to initialize in-memory collections config: {e}') diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 8ac806bf..e1b5d16f 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1798,6 +1798,14 @@ class DownloadCacheConfig(ConfigFile): ) validator = DownloadCacheConfigValidator + def import_config(self, file: pathlib.Path = None, send_events=False): + file = file or self.get_file() + if not file.is_file(): + self.successful_import = True + return + super().import_config(file, send_events) + self.successful_import = True + @property def video_durations(self) -> List[list]: return self._config['video_durations'] or list() diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 16db46ba..dbd4831b 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1190,3 +1190,33 @@ def test_get_paths_in_parent_directory(test_directory, make_files_structure): test_directory / 'foo/bar', test_directory / 'foo/bar/baz.txt', } + + +def test_get_all_configs_includes_all_config_subclasses(): + """Every ConfigFile subclass must be included in get_all_configs so it is imported on startup. + + If a ConfigFile subclass is not in get_all_configs, it will never have successful_import set to True and will fail + to save with: RuntimeError: Refusing to save config because it was never successfully imported! + """ + from wrolpi.common import ConfigFile, get_all_configs + + # Ensure all ConfigFile subclasses are imported. + import modules.archive.lib # noqa + import modules.inventory.common # noqa + import modules.videos.lib # noqa + import wrolpi.collections.config # noqa + import wrolpi.downloader # noqa + import wrolpi.tags # noqa + + # Temporarily disable PYTEST guard so get_channels_config() returns its global instance. + with mock.patch('modules.videos.lib.PYTEST', False): + all_configs = get_all_configs() + returned_types = {type(config) for config in all_configs.values()} + + all_subclasses = set(ConfigFile.__subclasses__()) + + missing = all_subclasses - returned_types + assert not missing, ( + f'ConfigFile subclasses not in get_all_configs(): {[c.__name__ for c in missing]}. ' + f'Add their getter to get_all_configs() so they are imported on startup.' + )