Configs are initialized on import so Manager is started for all processes.

.gitignore

@@ -110,7 +110,7 @@ test/videos
 test/archive
 
 # testing configs
-test/config/*
+test/config
 
 app/package-lock.json
 modules/node_modules/

main.py

@@ -9,7 +9,7 @@ import pytz
 from sanic import Sanic
 
 from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent, admin
-from wrolpi.common import logger, get_config, import_modules, check_media_directory, set_wrol_mode
+from wrolpi.common import logger, get_config, import_modules, check_media_directory
 from wrolpi.dates import set_timezone
 from wrolpi.downloader import download_manager
 from wrolpi.vars import PROJECT_DIR, DOCKERIZED

modules/inventory/common.py

@@ -127,7 +127,7 @@ class InventoriesConfig(ConfigFile):
         self.update({'version': value})
 
 
-INVENTORIES_CONFIG: InventoriesConfig = None
+INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig(global_=True)
 TEST_INVENTORIES_CONFIG: InventoriesConfig = None
 
 
@@ -137,10 +137,6 @@ def get_inventories_config():
         return TEST_INVENTORIES_CONFIG
 
     global INVENTORIES_CONFIG
-    if isinstance(INVENTORIES_CONFIG, ConfigFile):
-        return INVENTORIES_CONFIG
-
-    INVENTORIES_CONFIG = InventoriesConfig()
     return INVENTORIES_CONFIG
 
 

modules/videos/configs/channels.yaml

@@ -1,8 +0,0 @@
-# This is an example config.  Do not edit this file.  Edit the config in your media directory!
-channels:
-  wrolpi:
-    name: WROLPi
-    url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos
-    directory: videos/wrolpi
-    download_frequency: 604800
-favorites: [ ]

modules/videos/configs/downloader.yaml

@@ -1,12 +0,0 @@
-# Most of these names were decided by YT-DLP project, we pass these to any calls to that project.
-continue_dl: true
-dateafter: 19900101
-file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s"
-format: 22/135/18/43
-nooverwrites: true
-quiet: false
-writeautomaticsub: true
-writeinfojson: true
-writesubtitles: true
-writethumbnail: true
-youtube_include_dash_manifest: false

modules/videos/conftest.py

@@ -8,7 +8,7 @@ import pytest
 from PIL import Image
 
 from modules.videos.downloader import VideoDownloader, ChannelDownloader
-from modules.videos.lib import set_test_channels_config
+from modules.videos.lib import set_test_channels_config, set_test_downloader_config
 from modules.videos.models import Channel, Video
 from wrolpi.common import sanitize_link
 from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
@@ -162,3 +162,12 @@ def test_channels_config(test_directory):
     set_test_channels_config(True)
     yield config_path
     set_test_channels_config(False)
+
+
+@pytest.fixture
+def test_downloader_config(test_directory):
+    (test_directory / 'config').mkdir(exist_ok=True)
+    config_path = test_directory / 'config/downloader.yaml'
+    set_test_downloader_config(True)
+    yield config_path
+    set_test_downloader_config(False)

modules/videos/lib.py

@@ -319,7 +319,7 @@ class ChannelsConfig(ConfigFile):
         self.update({'favorites': value})
 
 
-CHANNELS_CONFIG: ChannelsConfig = None
+CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig(global_=True)
 TEST_CHANNELS_CONFIG: ChannelsConfig = None
 
 
@@ -329,10 +329,6 @@ def get_channels_config():
         return TEST_CHANNELS_CONFIG
 
     global CHANNELS_CONFIG
-    if isinstance(CHANNELS_CONFIG, ConfigFile):
-        return CHANNELS_CONFIG
-
-    CHANNELS_CONFIG = ChannelsConfig()
     return CHANNELS_CONFIG
 
 
@@ -449,23 +445,27 @@ class DownloaderConfig(ConfigFile):
         self.update({'youtube_include_dash_manifest': value})
 
 
-DOWNLOADER_CONFIG: DownloaderConfig = None
+DOWNLOADER_CONFIG: DownloaderConfig = DownloaderConfig(global_=True)
 TEST_DOWNLOADER_CONFIG: DownloaderConfig = None
 
 
 def get_downloader_config():
     global TEST_DOWNLOADER_CONFIG
-    if isinstance(TEST_DOWNLOADER_CONFIG, WROLPiConfig):
+    if isinstance(TEST_DOWNLOADER_CONFIG, DownloaderConfig):
         return TEST_DOWNLOADER_CONFIG
 
     global DOWNLOADER_CONFIG
-    if isinstance(DOWNLOADER_CONFIG, WROLPiConfig):
-        return DOWNLOADER_CONFIG
-
-    DOWNLOADER_CONFIG = DownloaderConfig()
     return DOWNLOADER_CONFIG
 
 
+def set_test_downloader_config(enabled: bool):
+    global TEST_DOWNLOADER_CONFIG
+    if enabled:
+        TEST_DOWNLOADER_CONFIG = DownloaderConfig()
+    else:
+        TEST_DOWNLOADER_CONFIG = None
+
+
 def get_channels_config_from_db(session: Session) -> dict:
     """Create a dictionary that contains all the Channels from the DB."""
     channels = session.query(Channel).order_by(Channel.link).all()

modules/videos/test/test_downloader.py

@@ -266,24 +266,6 @@ class TestVideosDownloaders(TestAPI):
             # Download is skipped for test.
             self.assertEqual(download.status, 'new')
 
-    @wrap_test_db
-    def test_invalid_download_url(self):
-        """
-        An invalid url should not be attempted again.
-        """
-        _, session = get_db_context()
-        with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
-                mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \
-                mock.patch('modules.videos.downloader.YDL') as mock_ydl:
-            mock_get_downloader.return_value = (VideoDownloader, {'channel': 'foo', 'id': 'some id'})
-            mock_valid_url.return_value = (True, None)
-            mock_ydl.extract_info.side_effect = UnsupportedError('oops')
-            self.mgr.create_download('url', session)
-
-        with get_db_session() as session:
-            download = self.mgr.get_downloads(session)[0]
-            assert download.status == 'failed'
-
     @wrap_test_db
     def test_get_or_create_channel(self):
         """
@@ -416,3 +398,18 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
 
     with pytest.raises(InvalidDownload):
         download_channel(simple_channel.link)
+
+
+def test_invalid_download_url(test_session, test_download_manager, test_downloader_config):
+    """An invalid url should not be attempted again."""
+    _, session = get_db_context()
+    with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
+            mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \
+            mock.patch('modules.videos.downloader.YDL') as mock_ydl:
+        mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'})
+        mock_valid_url.return_value = (True, None)
+        mock_ydl.extract_info.side_effect = UnsupportedError('oops')
+        test_download_manager.create_download('url', session)
+
+    download = test_download_manager.get_downloads(test_session)[0]
+    assert download.status == 'failed'

wrolpi/common.py

@@ -176,7 +176,11 @@ class ConfigFile:
     file_name: str = None
     default_config: dict = None
 
-    def __init__(self):
+    def __init__(self, global_: bool = False):
+        if PYTEST and global_:
+            # Do not load a global config on import while testing.  A global instance will never be used for testing.
+            return
+
         config_file = self.get_file()
         self.file_lock = Lock()
         self._config = Manager().dict()
@@ -187,6 +191,9 @@ class ConfigFile:
             with config_file.open('rt') as fh:
                 self._config.update(yaml.load(fh, Loader=yaml.Loader))
 
+    def __repr__(self):
+        return f'<{self.__class__.__name__} file={self.get_file()}>'
+
     def save(self):
         """
         Write this config to its file.
@@ -242,6 +249,9 @@ class ConfigFile:
         """
         Get a deepcopy of this config.
         """
+        if not hasattr(self, '_config'):
+            raise NotImplementedError('You cannot use a global config while testing!')
+
         return deepcopy(self._config)
 
 
@@ -296,7 +306,7 @@ class WROLPiConfig(ConfigFile):
         self.update({'wrol_mode': value})
 
 
-WROLPI_CONFIG: WROLPiConfig = None
+WROLPI_CONFIG: WROLPiConfig = WROLPiConfig(global_=True)
 TEST_WROLPI_CONFIG: WROLPiConfig = None
 
 
@@ -307,11 +317,6 @@ def get_config() -> WROLPiConfig:
         return TEST_WROLPI_CONFIG
 
     global WROLPI_CONFIG
-    if isinstance(WROLPI_CONFIG, WROLPiConfig):
-        return WROLPI_CONFIG
-
-    # Create the config instance once.
-    WROLPI_CONFIG = WROLPiConfig()
     return WROLPI_CONFIG