From f2af5e048b3103e092f92be5408af0338cacd983 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 16 Feb 2022 23:05:43 -0700 Subject: [PATCH] Configs are initialized on import so Manager is started for all processes. --- .gitignore | 2 +- main.py | 2 +- modules/inventory/common.py | 6 +---- modules/videos/configs/channels.yaml | 8 ------- modules/videos/configs/downloader.yaml | 12 ---------- modules/videos/conftest.py | 11 ++++++++- modules/videos/lib.py | 22 ++++++++--------- modules/videos/test/test_downloader.py | 33 ++++++++++++-------------- wrolpi/common.py | 19 +++++++++------ 9 files changed, 51 insertions(+), 64 deletions(-) delete mode 100644 modules/videos/configs/channels.yaml delete mode 100644 modules/videos/configs/downloader.yaml diff --git a/.gitignore b/.gitignore index 2690176a..cd00e877 100644 --- a/.gitignore +++ b/.gitignore @@ -110,7 +110,7 @@ test/videos test/archive # testing configs -test/config/* +test/config app/package-lock.json modules/node_modules/ diff --git a/main.py b/main.py index 92238728..be55afe8 100755 --- a/main.py +++ b/main.py @@ -9,7 +9,7 @@ import pytz from sanic import Sanic from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent, admin -from wrolpi.common import logger, get_config, import_modules, check_media_directory, set_wrol_mode +from wrolpi.common import logger, get_config, import_modules, check_media_directory from wrolpi.dates import set_timezone from wrolpi.downloader import download_manager from wrolpi.vars import PROJECT_DIR, DOCKERIZED diff --git a/modules/inventory/common.py b/modules/inventory/common.py index 547166e1..c0fd9901 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -127,7 +127,7 @@ class InventoriesConfig(ConfigFile): self.update({'version': value}) -INVENTORIES_CONFIG: InventoriesConfig = None +INVENTORIES_CONFIG: InventoriesConfig = InventoriesConfig(global_=True) TEST_INVENTORIES_CONFIG: InventoriesConfig = None @@ -137,10 +137,6 @@ def get_inventories_config(): return TEST_INVENTORIES_CONFIG global INVENTORIES_CONFIG - if isinstance(INVENTORIES_CONFIG, ConfigFile): - return INVENTORIES_CONFIG - - INVENTORIES_CONFIG = InventoriesConfig() return INVENTORIES_CONFIG diff --git a/modules/videos/configs/channels.yaml b/modules/videos/configs/channels.yaml deleted file mode 100644 index c39ef457..00000000 --- a/modules/videos/configs/channels.yaml +++ /dev/null @@ -1,8 +0,0 @@ -# This is an example config. Do not edit this file. Edit the config in your media directory! -channels: - wrolpi: - name: WROLPi - url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos - directory: videos/wrolpi - download_frequency: 604800 -favorites: [ ] diff --git a/modules/videos/configs/downloader.yaml b/modules/videos/configs/downloader.yaml deleted file mode 100644 index 49377c92..00000000 --- a/modules/videos/configs/downloader.yaml +++ /dev/null @@ -1,12 +0,0 @@ -# Most of these names were decided by YT-DLP project, we pass these to any calls to that project. -continue_dl: true -dateafter: 19900101 -file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s" -format: 22/135/18/43 -nooverwrites: true -quiet: false -writeautomaticsub: true -writeinfojson: true -writesubtitles: true -writethumbnail: true -youtube_include_dash_manifest: false diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 03ecb4ae..a7386e86 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -8,7 +8,7 @@ import pytest from PIL import Image from modules.videos.downloader import VideoDownloader, ChannelDownloader -from modules.videos.lib import set_test_channels_config +from modules.videos.lib import set_test_channels_config, set_test_downloader_config from modules.videos.models import Channel, Video from wrolpi.common import sanitize_link from wrolpi.downloader import DownloadFrequency, DownloadManager, Download @@ -162,3 +162,12 @@ def test_channels_config(test_directory): set_test_channels_config(True) yield config_path set_test_channels_config(False) + + +@pytest.fixture +def test_downloader_config(test_directory): + (test_directory / 'config').mkdir(exist_ok=True) + config_path = test_directory / 'config/downloader.yaml' + set_test_downloader_config(True) + yield config_path + set_test_downloader_config(False) diff --git a/modules/videos/lib.py b/modules/videos/lib.py index c7c2e395..cf9b5b43 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -319,7 +319,7 @@ class ChannelsConfig(ConfigFile): self.update({'favorites': value}) -CHANNELS_CONFIG: ChannelsConfig = None +CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig(global_=True) TEST_CHANNELS_CONFIG: ChannelsConfig = None @@ -329,10 +329,6 @@ def get_channels_config(): return TEST_CHANNELS_CONFIG global CHANNELS_CONFIG - if isinstance(CHANNELS_CONFIG, ConfigFile): - return CHANNELS_CONFIG - - CHANNELS_CONFIG = ChannelsConfig() return CHANNELS_CONFIG @@ -449,23 +445,27 @@ class DownloaderConfig(ConfigFile): self.update({'youtube_include_dash_manifest': value}) -DOWNLOADER_CONFIG: DownloaderConfig = None +DOWNLOADER_CONFIG: DownloaderConfig = DownloaderConfig(global_=True) TEST_DOWNLOADER_CONFIG: DownloaderConfig = None def get_downloader_config(): global TEST_DOWNLOADER_CONFIG - if isinstance(TEST_DOWNLOADER_CONFIG, WROLPiConfig): + if isinstance(TEST_DOWNLOADER_CONFIG, DownloaderConfig): return TEST_DOWNLOADER_CONFIG global DOWNLOADER_CONFIG - if isinstance(DOWNLOADER_CONFIG, WROLPiConfig): - return DOWNLOADER_CONFIG - - DOWNLOADER_CONFIG = DownloaderConfig() return DOWNLOADER_CONFIG +def set_test_downloader_config(enabled: bool): + global TEST_DOWNLOADER_CONFIG + if enabled: + TEST_DOWNLOADER_CONFIG = DownloaderConfig() + else: + TEST_DOWNLOADER_CONFIG = None + + def get_channels_config_from_db(session: Session) -> dict: """Create a dictionary that contains all the Channels from the DB.""" channels = session.query(Channel).order_by(Channel.link).all() diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index 60684bde..d15b3033 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -266,24 +266,6 @@ class TestVideosDownloaders(TestAPI): # Download is skipped for test. self.assertEqual(download.status, 'new') - @wrap_test_db - def test_invalid_download_url(self): - """ - An invalid url should not be attempted again. - """ - _, session = get_db_context() - with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \ - mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \ - mock.patch('modules.videos.downloader.YDL') as mock_ydl: - mock_get_downloader.return_value = (VideoDownloader, {'channel': 'foo', 'id': 'some id'}) - mock_valid_url.return_value = (True, None) - mock_ydl.extract_info.side_effect = UnsupportedError('oops') - self.mgr.create_download('url', session) - - with get_db_session() as session: - download = self.mgr.get_downloads(session)[0] - assert download.status == 'failed' - @wrap_test_db def test_get_or_create_channel(self): """ @@ -416,3 +398,18 @@ def test_channel_download_no_download(test_session, video_download_manager, simp with pytest.raises(InvalidDownload): download_channel(simple_channel.link) + + +def test_invalid_download_url(test_session, test_download_manager, test_downloader_config): + """An invalid url should not be attempted again.""" + _, session = get_db_context() + with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \ + mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \ + mock.patch('modules.videos.downloader.YDL') as mock_ydl: + mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'}) + mock_valid_url.return_value = (True, None) + mock_ydl.extract_info.side_effect = UnsupportedError('oops') + test_download_manager.create_download('url', session) + + download = test_download_manager.get_downloads(test_session)[0] + assert download.status == 'failed' diff --git a/wrolpi/common.py b/wrolpi/common.py index a7e8cded..cf6c39f6 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -176,7 +176,11 @@ class ConfigFile: file_name: str = None default_config: dict = None - def __init__(self): + def __init__(self, global_: bool = False): + if PYTEST and global_: + # Do not load a global config on import while testing. A global instance will never be used for testing. + return + config_file = self.get_file() self.file_lock = Lock() self._config = Manager().dict() @@ -187,6 +191,9 @@ class ConfigFile: with config_file.open('rt') as fh: self._config.update(yaml.load(fh, Loader=yaml.Loader)) + def __repr__(self): + return f'<{self.__class__.__name__} file={self.get_file()}>' + def save(self): """ Write this config to its file. @@ -242,6 +249,9 @@ class ConfigFile: """ Get a deepcopy of this config. """ + if not hasattr(self, '_config'): + raise NotImplementedError('You cannot use a global config while testing!') + return deepcopy(self._config) @@ -296,7 +306,7 @@ class WROLPiConfig(ConfigFile): self.update({'wrol_mode': value}) -WROLPI_CONFIG: WROLPiConfig = None +WROLPI_CONFIG: WROLPiConfig = WROLPiConfig(global_=True) TEST_WROLPI_CONFIG: WROLPiConfig = None @@ -307,11 +317,6 @@ def get_config() -> WROLPiConfig: return TEST_WROLPI_CONFIG global WROLPI_CONFIG - if isinstance(WROLPI_CONFIG, WROLPiConfig): - return WROLPI_CONFIG - - # Create the config instance once. - WROLPI_CONFIG = WROLPiConfig() return WROLPI_CONFIG