From b7d163e7ff4a494d9483fb7709690e478d1d561a Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 28 Nov 2024 13:28:23 -0700 Subject: [PATCH] Config files are initialized on startup, if they are missing. --- main.py | 24 ++++++++----- modules/videos/conftest.py | 4 +-- modules/videos/downloader.py | 6 ++-- modules/videos/lib.py | 30 +++++++++------- modules/videos/test/test_lib.py | 8 ++--- modules/zim/test/test_lib.py | 2 +- wrolpi/common.py | 62 +++++++++++++++++++++++++++++---- wrolpi/contexts.py | 8 ++--- wrolpi/downloader.py | 2 -- wrolpi/test/test_common.py | 55 +++++++++++++++++++++++++++++ 10 files changed, 158 insertions(+), 43 deletions(-) diff --git a/main.py b/main.py index 042b263b..3a40dc75 100755 --- a/main.py +++ b/main.py @@ -11,13 +11,14 @@ from sanic.signals import Event from modules.inventory import init_inventory from modules.inventory.common import import_inventories_config -from modules.videos.lib import import_channels_config +from modules.videos.lib import import_channels_config, get_videos_downloader_config from wrolpi import flags, admin from wrolpi import root_api # noqa from wrolpi import tags from wrolpi.api_utils import api_app, perpetual_signal from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \ - cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled + cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled, \ + initialize_config_files from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts from wrolpi.dates import Seconds from wrolpi.downloader import import_downloads_config, download_manager @@ -170,6 +171,13 @@ async def main_process_startup(app: Sanic): attach_shared_contexts(app) logger.debug('main_process_startup done') + download_manager.disable() + + try: + initialize_config_files() + except Exception as e: + logger.error('Failed to initialize config files', exc_info=e) + @api_app.listener('after_server_start') # FileConfigs need to be initialized first. async def initialize_configs(app: Sanic): @@ -214,9 +222,6 @@ async def start_single_tasks(app: Sanic): logger.debug(f'start_single_tasks started') - # Do not start downloads until configs have been imported. - download_manager.disable() - # Import configs, ignore errors so the service will start. Configs will refuse to save if they failed to import. with suppress(Exception): get_wrolpi_config().import_config() @@ -224,6 +229,9 @@ async def start_single_tasks(app: Sanic): with suppress(Exception): tags.import_tags_config() logger.debug('tags config imported') + with suppress(Exception): + get_videos_downloader_config().import_config() + logger.debug('videos downloader config imported') with suppress(Exception): await import_downloads_config() logger.debug('downloads config imported') @@ -237,9 +245,6 @@ async def start_single_tasks(app: Sanic): with suppress(Exception): init_inventory() - if get_wrolpi_config().download_on_startup: - await download_manager.enable() - from modules.zim.lib import flag_outdated_zim_files try: flag_outdated_zim_files() @@ -254,6 +259,9 @@ async def start_single_tasks(app: Sanic): # Set all downloads to new. download_manager.retry_downloads() + if get_wrolpi_config().download_on_startup: + await download_manager.enable() + # Hotspot/throttle are not supported in Docker containers. if not DOCKERIZED: if get_wrolpi_config().hotspot_on_startup: diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index 5131e43a..c8ba1b86 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -179,9 +179,9 @@ def test_channels_config(test_directory): @pytest.fixture -def test_downloader_config(test_directory): +def test_videos_downloader_config(test_directory): (test_directory / 'config').mkdir(exist_ok=True) - config_path = test_directory / 'config/downloader.yaml' + config_path = test_directory / 'config/videos_downloader.yaml' set_test_downloader_config(True) yield config_path set_test_downloader_config(False) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index dbf2ac20..fa70507b 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -30,7 +30,7 @@ from .channel.lib import create_channel, get_channel from .common import get_no_channel_directory, update_view_counts_and_censored, \ ffmpeg_video_complete from .errors import UnknownChannel -from .lib import get_downloader_config, YDL, ydl_logger, format_videos_destination +from .lib import get_videos_downloader_config, YDL, ydl_logger, format_videos_destination from .models import Video, Channel from .normalize_video_url import normalize_video_url from .schema import ChannelPostRequest @@ -348,7 +348,7 @@ class VideoDownloader(Downloader, ABC): if tag_names and logger.isEnabledFor(logging.DEBUG): logger.debug(f'Downloading {url} with {tag_names=}') - config = get_downloader_config() + config = get_videos_downloader_config() logs = None # noqa try: @@ -573,7 +573,7 @@ class VideoDownloader(Downloader, ABC): raise ValueError(f'Output directory does not exist! {out_dir=}') # YoutubeDL expects specific options, add onto the default options - config = get_downloader_config() + config = get_videos_downloader_config() options = config.yt_dlp_options # yt-dlp expects the absolute path. options['outtmpl'] = f'{out_dir}/{config.file_name_format}' diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 6260d486..4ae7036a 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -358,7 +358,7 @@ class VideoDownloaderConfigYtDlpOptionsValidator: @dataclass class VideoDownloaderConfigValidator: - video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', '360p', 'maximum']) + video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', 'maximum']) version: int = None yt_dlp_options: VideoDownloaderConfigYtDlpOptionsValidator = field(default_factory=dict) @@ -366,7 +366,7 @@ class VideoDownloaderConfigValidator: class VideoDownloaderConfig(ConfigFile): file_name = 'videos_downloader.yaml' default_config = dict( - video_resolutions=['1080p', '720p', '480p', '360p', 'maximum'], + video_resolutions=['1080p', '720p', '480p', 'maximum'], version=0, yt_dlp_options=dict( continue_dl=True, @@ -439,23 +439,27 @@ class VideoDownloaderConfig(ConfigFile): def yt_dlp_options(self, value: dict): self.update({'yt_dlp_options': value}) - -VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig() -TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None + def import_config(self, file: pathlib.Path = None, send_events=False): + super().import_config(file, send_events) + self.successful_import = True -def get_downloader_config() -> VideoDownloaderConfig: - global TEST_VIDEO_DOWNLOADER_CONFIG - if isinstance(TEST_VIDEO_DOWNLOADER_CONFIG, VideoDownloaderConfig): - return TEST_VIDEO_DOWNLOADER_CONFIG +VIDEOS_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig() +TEST_VIDEOS_DOWNLOADER_CONFIG: VideoDownloaderConfig = None - global VIDEO_DOWNLOADER_CONFIG - return VIDEO_DOWNLOADER_CONFIG + +def get_videos_downloader_config() -> VideoDownloaderConfig: + global TEST_VIDEOS_DOWNLOADER_CONFIG + if isinstance(TEST_VIDEOS_DOWNLOADER_CONFIG, VideoDownloaderConfig): + return TEST_VIDEOS_DOWNLOADER_CONFIG + + global VIDEOS_DOWNLOADER_CONFIG + return VIDEOS_DOWNLOADER_CONFIG def set_test_downloader_config(enabled: bool): - global TEST_VIDEO_DOWNLOADER_CONFIG - TEST_VIDEO_DOWNLOADER_CONFIG = VideoDownloaderConfig() if enabled else None + global TEST_VIDEOS_DOWNLOADER_CONFIG + TEST_VIDEOS_DOWNLOADER_CONFIG = VideoDownloaderConfig() if enabled else None def get_channels_config_from_db(session: Session) -> dict: diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index faa1a957..c97de9e5 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -270,9 +270,9 @@ async def test_format_videos_destination(async_client, test_directory): @pytest.mark.asyncio -async def test_video_downloader_config_api(async_client, test_directory): - config = lib.get_downloader_config() - assert config.video_resolutions == ['1080p', '720p', '480p', '360p', 'maximum'] +async def test_videos_downloader_config_api(async_client, test_directory, test_videos_downloader_config): + config = lib.get_videos_downloader_config() + assert config.video_resolutions == ['1080p', '720p', '480p', 'maximum'] request, response = await async_client.get('/api/config/videos_downloader.yaml') assert response.status == HTTPStatus.OK @@ -283,7 +283,7 @@ async def test_video_downloader_config_api(async_client, test_directory): request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body) assert response.status == HTTPStatus.NO_CONTENT - config = lib.get_downloader_config() + config = lib.get_videos_downloader_config() assert config.video_resolutions == ['720p', 'maximum'] assert (test_directory / 'config/videos_downloader.yaml').is_file() assert config.is_valid() diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 2fd44553..4bb5912d 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -348,7 +348,7 @@ def test_zim_download_url_to_name(url, expected): @pytest.mark.asyncio -async def test_zim_subscription_download_import(test_session, test_downloader_config): +async def test_zim_subscription_download_import(async_client, test_session): # Subscription creates a ZimSubscription and Download await lib.subscribe('Wikisource', 'en', session=test_session) # Add a once-download. This should not be associated with a ZimSubscription. diff --git a/wrolpi/common.py b/wrolpi/common.py index 14fc1503..b60c1a1b 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -637,8 +637,8 @@ def get_all_configs() -> Dict[str, ConfigFile]: if inventories_config := get_inventories_config(): all_configs[inventories_config.file_name] = inventories_config - from modules.videos.lib import get_downloader_config - if videos_downloader_config := get_downloader_config(): + from modules.videos.lib import get_videos_downloader_config + if videos_downloader_config := get_videos_downloader_config(): all_configs[videos_downloader_config.file_name] = videos_downloader_config from wrolpi.downloader import get_download_manager_config @@ -705,10 +705,10 @@ class WROLPiConfig(ConfigFile): super().import_config(file) # Destinations cannot be empty. - self.archive_destination = self.archive_destination or self.default_config['archive_destination'] - self.map_destination = self.map_destination or self.default_config['map_destination'] - self.videos_destination = self.videos_destination or self.default_config['videos_destination'] - self.zims_destination = self.zims_destination or self.default_config['zims_destination'] + self._config['archive_destination'] = self.archive_destination or self.default_config['archive_destination'] + self._config['map_destination'] = self.map_destination or self.default_config['map_destination'] + self._config['videos_destination'] = self.videos_destination or self.default_config['videos_destination'] + self._config['zims_destination'] = self.zims_destination or self.default_config['zims_destination'] self.successful_import = True except Exception as e: @@ -2188,3 +2188,53 @@ def cached_multiprocessing_result(func: callable): return result return wrapped + + +def initialize_config_files() -> list[str]: + """Creates config files only if they do not exist and will not conflict with the DB.""" + from modules.inventory.common import get_inventories_config + from modules.videos.lib import get_channels_config, get_videos_downloader_config + from wrolpi.downloader import get_download_manager_config + from wrolpi.tags import get_tags_config + from wrolpi.db import get_db_session + + created = [] # The names of the configs that are created. + + if not get_wrolpi_config().get_file().is_file(): + get_wrolpi_config().save(overwrite=True) + created.append(get_wrolpi_config().get_file().name) + + with get_db_session() as session: + if not get_channels_config().get_file().is_file(): + from modules.videos.models import Channel + if session.query(Channel).count() == 0: + # No Channels will be deleted, create the empty Channels config file. + get_channels_config().save(overwrite=True) + created.append(get_channels_config().get_file().name) + + if not get_tags_config().get_file().is_file(): + from wrolpi.tags import Tag + if session.query(Tag).count() == 0: + # No Tags will be deleted, create the empty Tags config file. + get_tags_config().save(overwrite=True) + created.append(get_tags_config().get_file().name) + + if not get_download_manager_config().get_file().is_file(): + from wrolpi.downloader import Download + if session.query(Download).count() == 0: + # No Downloads will be deleted, create the empty Download Manager config file. + get_download_manager_config().save(overwrite=True) + created.append(get_download_manager_config().get_file().name) + + if not get_videos_downloader_config().get_file().is_file(): + get_videos_downloader_config().save(overwrite=True) + created.append(get_videos_downloader_config().get_file().name) + + if not get_inventories_config().get_file().is_file(): + from modules.inventory.models import Inventory + if session.query(Inventory).count() == 0: + # No Inventories will be deleted, create the empty Inventory config file. + get_inventories_config().save(overwrite=True) + created.append(get_inventories_config().get_file().name) + + return created diff --git a/wrolpi/contexts.py b/wrolpi/contexts.py index 451dc39a..5df6bd08 100644 --- a/wrolpi/contexts.py +++ b/wrolpi/contexts.py @@ -24,7 +24,7 @@ def attach_shared_contexts(app: Sanic): app.shared_ctx.inventories_config = manager.dict() app.shared_ctx.channels_config = manager.dict() app.shared_ctx.download_manager_config = manager.dict() - app.shared_ctx.video_downloader_config = manager.dict() + app.shared_ctx.videos_downloader_config = manager.dict() # Shared dicts. app.shared_ctx.refresh = manager.dict() app.shared_ctx.uploaded_files = manager.dict() @@ -74,7 +74,7 @@ def reset_shared_contexts(app: Sanic): app.shared_ctx.inventories_config.clear() app.shared_ctx.channels_config.clear() app.shared_ctx.download_manager_config.clear() - app.shared_ctx.video_downloader_config.clear() + app.shared_ctx.videos_downloader_config.clear() # Shared dicts. app.shared_ctx.refresh.clear() app.shared_ctx.uploaded_files.clear() @@ -119,13 +119,13 @@ def initialize_configs_contexts(app: Sanic): """Assign multiprocessing Dicts to their respective FileConfigs in this process.""" from modules.inventory.common import INVENTORIES_CONFIG from modules.videos.lib import CHANNELS_CONFIG - from modules.videos.lib import VIDEO_DOWNLOADER_CONFIG + from modules.videos.lib import VIDEOS_DOWNLOADER_CONFIG from wrolpi.common import WROLPI_CONFIG from wrolpi.tags import TAGS_CONFIG from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG INVENTORIES_CONFIG.initialize(app.shared_ctx.inventories_config) CHANNELS_CONFIG.initialize(app.shared_ctx.channels_config) - VIDEO_DOWNLOADER_CONFIG.initialize(app.shared_ctx.video_downloader_config) + VIDEOS_DOWNLOADER_CONFIG.initialize(app.shared_ctx.videos_downloader_config) WROLPI_CONFIG.initialize(app.shared_ctx.wrolpi_config) TAGS_CONFIG.initialize(app.shared_ctx.tags_config) DOWNLOAD_MANAGER_CONFIG.initialize(app.shared_ctx.download_manager_config) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index d4720703..77388864 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -709,8 +709,6 @@ class DownloadManager: stmt = "UPDATE download SET status='new' WHERE status='pending' OR status='deferred'" curs.execute(stmt) - save_downloads_config.activate_switch() - @optional_session def get_new_downloads(self, session: Session) -> Generator[Download, None, None]: """ diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index b37fe178..89096177 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -1056,3 +1056,58 @@ async def test_cached_multiprocessing_result(async_client): # Kwargs are supported assert await func(inc=1) == 4 assert await func(inc=1) == 4 + + +@pytest.mark.asyncio +async def test_initialize_config_files(async_client, test_session, test_directory, test_wrolpi_config, + test_channels_config, test_tags_config, test_videos_downloader_config, + test_download_manager_config, simple_channel, test_inventory, tag_factory, + test_download_manager, test_downloader): + """Config files should only be created if they are missing, and they will not overwrite existing entries in the DB.""" + from modules.videos.lib import get_videos_downloader_config, get_channels_config + from wrolpi.tags import get_tags_config + from wrolpi.downloader import get_download_manager_config + from modules.inventory.common import get_inventories_config + + # Create some DB entries so the configs will not be imported. + tag1, tag2 = await tag_factory(), await tag_factory() + download = test_download_manager.create_download('https://example.com', downloader_name=test_downloader.name) + test_session.commit() + # Write some config files, they should not be re-created. + get_wrolpi_config().get_file().write_text('wrolpi') + get_videos_downloader_config().get_file().write_text('video downloader') + + assert common.initialize_config_files() == [], 'No configs should have been created.' + # Config files were not overwritten with real configs. + assert get_wrolpi_config().get_file().read_text() == 'wrolpi' + assert get_videos_downloader_config().get_file().read_text() == 'video downloader' + + # WROLPi config can be created because it is now missing. + get_wrolpi_config().get_file().unlink() + assert common.initialize_config_files() == ['wrolpi.yaml'], 'WROLPi config should have been created.' + assert get_wrolpi_config().is_valid() + + # Videos Downloader config can be created because it is now missing. + get_videos_downloader_config().get_file().unlink() + assert common.initialize_config_files() == ['videos_downloader.yaml'], \ + 'Videos Downloader config should have been created.' + assert get_videos_downloader_config().is_valid() + + # Channels config file does not exist, no conflicting Channels so the file is created. + simple_channel.delete_with_videos() + assert common.initialize_config_files() == ['channels.yaml'], 'Channels config should have been created.' + assert get_channels_config().is_valid() + + tag1.delete() + tag2.delete() + assert common.initialize_config_files() == ['tags.yaml'], 'Tags config should have been created.' + assert get_tags_config().is_valid() + + download.delete() + assert common.initialize_config_files() == ['download_manager.yaml'], \ + 'Download Manager config should have been created.' + assert get_download_manager_config().is_valid() + + test_session.delete(test_inventory) + assert common.initialize_config_files() == ['inventories.yaml'], 'Inventory config should have been created.' + assert get_inventories_config().is_valid()