Importing new download cache config on startup.

main.py

@@ -268,6 +268,10 @@ async def start_single_tasks(app: Sanic):
                 from modules.archive.lib import get_archive_downloader_config
                 get_archive_downloader_config().import_config()
                 logger.debug('archive downloader config imported')
+            with log_and_suppress(Exception, message='Failed to import download cache config'):
+                from wrolpi.downloader import get_download_cache_config
+                get_download_cache_config().import_config()
+                logger.debug('download cache config imported')
 
     from modules.zim.lib import flag_outdated_zim_files
     try:

wrolpi/common.py

@@ -450,7 +450,8 @@ def get_media_directory() -> Path:
     return MEDIA_DIRECTORY
 
 
-DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'inventories.yaml'}
+DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml', 'inventories.yaml',
+                        'collections.yaml'}
 
 
 class ConfigFile:
@@ -846,6 +847,14 @@ def get_all_configs() -> Dict[str, ConfigFile]:
     if archive_downloader_config := get_archive_downloader_config():
         all_configs[archive_downloader_config.file_name] = archive_downloader_config
 
+    from wrolpi.downloader import get_download_cache_config
+    if download_cache_config := get_download_cache_config():
+        all_configs[download_cache_config.file_name] = download_cache_config
+
+    from wrolpi.collections.config import collections_config as _collections_config
+    if _collections_config:
+        all_configs[_collections_config.file_name] = _collections_config
+
     return all_configs
 
 
@@ -918,6 +927,16 @@ async def import_all_db_configs() -> dict[str, bool]:
         logger.warning(f'Failed to import inventories config: {e}')
         results['inventories'] = False
 
+    # Collections (uses tags for tag_name)
+    try:
+        from wrolpi.collections.config import collections_config as _collections_config
+        _collections_config.import_config()
+        results['collections'] = True
+        logger.debug('collections config imported')
+    except Exception as e:
+        logger.warning(f'Failed to import collections config: {e}')
+        results['collections'] = False
+
     return results
 
 

wrolpi/contexts.py

@@ -31,6 +31,8 @@ def attach_shared_contexts(app: Sanic):
     app.shared_ctx.videos_downloader_config = manager.dict()
     app.shared_ctx.domains_config = manager.dict()
     app.shared_ctx.archive_downloader_config = manager.dict()
+    app.shared_ctx.download_cache_config = manager.dict()
+    app.shared_ctx.collections_config = manager.dict()
     # Shared dicts.
     app.shared_ctx.refresh = manager.dict()
     app.shared_ctx.uploaded_files = manager.dict()
@@ -102,6 +104,8 @@ def reset_shared_contexts(app: Sanic):
     app.shared_ctx.videos_downloader_config.clear()
     app.shared_ctx.domains_config.clear()
     app.shared_ctx.archive_downloader_config.clear()
+    app.shared_ctx.download_cache_config.clear()
+    app.shared_ctx.collections_config.clear()
     # Shared dicts.
     app.shared_ctx.refresh.clear()
     app.shared_ctx.uploaded_files.clear()
@@ -255,3 +259,15 @@ def initialize_configs_contexts(app: Sanic):
         ARCHIVE_DOWNLOADER_CONFIG.initialize(app.shared_ctx.archive_downloader_config)
     except Exception as e:
         logger.error(f'Failed to initialize in-memory archive downloader config: {e}')
+
+    try:
+        from wrolpi.downloader import DOWNLOAD_CACHE_CONFIG
+        DOWNLOAD_CACHE_CONFIG.initialize(app.shared_ctx.download_cache_config)
+    except Exception as e:
+        logger.error(f'Failed to initialize in-memory download cache config: {e}')
+
+    try:
+        from wrolpi.collections.config import collections_config as _collections_config
+        _collections_config.initialize(app.shared_ctx.collections_config)
+    except Exception as e:
+        logger.error(f'Failed to initialize in-memory collections config: {e}')

wrolpi/downloader.py

@@ -1798,6 +1798,14 @@ class DownloadCacheConfig(ConfigFile):
     )
     validator = DownloadCacheConfigValidator
 
+    def import_config(self, file: pathlib.Path = None, send_events=False):
+        file = file or self.get_file()
+        if not file.is_file():
+            self.successful_import = True
+            return
+        super().import_config(file, send_events)
+        self.successful_import = True
+
     @property
     def video_durations(self) -> List[list]:
         return self._config['video_durations'] or list()

wrolpi/test/test_common.py

@@ -1190,3 +1190,33 @@ def test_get_paths_in_parent_directory(test_directory, make_files_structure):
         test_directory / 'foo/bar',
         test_directory / 'foo/bar/baz.txt',
     }
+
+
+def test_get_all_configs_includes_all_config_subclasses():
+    """Every ConfigFile subclass must be included in get_all_configs so it is imported on startup.
+
+    If a ConfigFile subclass is not in get_all_configs, it will never have successful_import set to True and will fail
+    to save with: RuntimeError: Refusing to save config because it was never successfully imported!
+    """
+    from wrolpi.common import ConfigFile, get_all_configs
+
+    # Ensure all ConfigFile subclasses are imported.
+    import modules.archive.lib  # noqa
+    import modules.inventory.common  # noqa
+    import modules.videos.lib  # noqa
+    import wrolpi.collections.config  # noqa
+    import wrolpi.downloader  # noqa
+    import wrolpi.tags  # noqa
+
+    # Temporarily disable PYTEST guard so get_channels_config() returns its global instance.
+    with mock.patch('modules.videos.lib.PYTEST', False):
+        all_configs = get_all_configs()
+    returned_types = {type(config) for config in all_configs.values()}
+
+    all_subclasses = set(ConfigFile.__subclasses__())
+
+    missing = all_subclasses - returned_types
+    assert not missing, (
+        f'ConfigFile subclasses not in get_all_configs(): {[c.__name__ for c in missing]}. '
+        f'Add their getter to get_all_configs() so they are imported on startup.'
+    )