rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Config files are initialized on startup, if they are missing.

Browse source
This commit is contained in:
Roland 2024-11-28 13:28:23 -07:00
parent 8c17d04909
commit b7d163e7ff
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
10 changed files with 158 additions and 43 deletions
Download patch file Download diff file Expand all files Collapse all files

24
main.py
View file

@ -11,13 +11,14 @@ from sanic.signals import Event
from modules.inventory import init_inventory
from modules.inventory.common import import_inventories_config
from modules.videos.lib import import_channels_config
from modules.videos.lib import import_channels_config, get_videos_downloader_config
from wrolpi import flags, admin
from wrolpi import root_api # noqa
from wrolpi import tags
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \
cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled
cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled, \
initialize_config_files
from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts
from wrolpi.dates import Seconds
from wrolpi.downloader import import_downloads_config, download_manager
@ -170,6 +171,13 @@ async def main_process_startup(app: Sanic):
attach_shared_contexts(app)
logger.debug('main_process_startup done')
download_manager.disable()
try:
initialize_config_files()
except Exception as e:
logger.error('Failed to initialize config files', exc_info=e)
@api_app.listener('after_server_start') # FileConfigs need to be initialized first.
async def initialize_configs(app: Sanic):
@ -214,9 +222,6 @@ async def start_single_tasks(app: Sanic):
logger.debug(f'start_single_tasks started')
# Do not start downloads until configs have been imported.
download_manager.disable()
# Import configs, ignore errors so the service will start. Configs will refuse to save if they failed to import.
with suppress(Exception):
get_wrolpi_config().import_config()
@ -224,6 +229,9 @@ async def start_single_tasks(app: Sanic):
with suppress(Exception):
tags.import_tags_config()
logger.debug('tags config imported')
with suppress(Exception):
get_videos_downloader_config().import_config()
logger.debug('videos downloader config imported')
with suppress(Exception):
await import_downloads_config()
logger.debug('downloads config imported')
@ -237,9 +245,6 @@ async def start_single_tasks(app: Sanic):
with suppress(Exception):
init_inventory()
if get_wrolpi_config().download_on_startup:
await download_manager.enable()
from modules.zim.lib import flag_outdated_zim_files
try:
flag_outdated_zim_files()
@ -254,6 +259,9 @@ async def start_single_tasks(app: Sanic):
# Set all downloads to new.
download_manager.retry_downloads()
if get_wrolpi_config().download_on_startup:
await download_manager.enable()
# Hotspot/throttle are not supported in Docker containers.
if not DOCKERIZED:
if get_wrolpi_config().hotspot_on_startup:

4
modules/videos/conftest.py
View file

@ -179,9 +179,9 @@ def test_channels_config(test_directory):
@pytest.fixture
def test_downloader_config(test_directory):
def test_videos_downloader_config(test_directory):
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/downloader.yaml'
config_path = test_directory / 'config/videos_downloader.yaml'
set_test_downloader_config(True)
yield config_path
set_test_downloader_config(False)

6
modules/videos/downloader.py
View file

@ -30,7 +30,7 @@ from .channel.lib import create_channel, get_channel
from .common import get_no_channel_directory, update_view_counts_and_censored, \
ffmpeg_video_complete
from .errors import UnknownChannel
from .lib import get_downloader_config, YDL, ydl_logger, format_videos_destination
from .lib import get_videos_downloader_config, YDL, ydl_logger, format_videos_destination
from .models import Video, Channel
from .normalize_video_url import normalize_video_url
from .schema import ChannelPostRequest
@ -348,7 +348,7 @@ class VideoDownloader(Downloader, ABC):
if tag_names and logger.isEnabledFor(logging.DEBUG):
logger.debug(f'Downloading {url} with {tag_names=}')
config = get_downloader_config()
config = get_videos_downloader_config()
logs = None # noqa
try:
@ -573,7 +573,7 @@ class VideoDownloader(Downloader, ABC):
raise ValueError(f'Output directory does not exist! {out_dir=}')
# YoutubeDL expects specific options, add onto the default options
config = get_downloader_config()
config = get_videos_downloader_config()
options = config.yt_dlp_options
# yt-dlp expects the absolute path.
options['outtmpl'] = f'{out_dir}/{config.file_name_format}'

30
modules/videos/lib.py
View file

@ -358,7 +358,7 @@ class VideoDownloaderConfigYtDlpOptionsValidator:
@dataclass
class VideoDownloaderConfigValidator:
video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', '360p', 'maximum'])
video_resolutions: List[str] = field(default_factory=lambda: ['1080p', '720p', '480p', 'maximum'])
version: int = None
yt_dlp_options: VideoDownloaderConfigYtDlpOptionsValidator = field(default_factory=dict)
@ -366,7 +366,7 @@ class VideoDownloaderConfigValidator:
class VideoDownloaderConfig(ConfigFile):
file_name = 'videos_downloader.yaml'
default_config = dict(
video_resolutions=['1080p', '720p', '480p', '360p', 'maximum'],
video_resolutions=['1080p', '720p', '480p', 'maximum'],
version=0,
yt_dlp_options=dict(
continue_dl=True,
@ -439,23 +439,27 @@ class VideoDownloaderConfig(ConfigFile):
def yt_dlp_options(self, value: dict):
self.update({'yt_dlp_options': value})
VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig()
TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None
def import_config(self, file: pathlib.Path = None, send_events=False):
super().import_config(file, send_events)
self.successful_import = True
def get_downloader_config() -> VideoDownloaderConfig:
global TEST_VIDEO_DOWNLOADER_CONFIG
if isinstance(TEST_VIDEO_DOWNLOADER_CONFIG, VideoDownloaderConfig):
return TEST_VIDEO_DOWNLOADER_CONFIG
VIDEOS_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig()
TEST_VIDEOS_DOWNLOADER_CONFIG: VideoDownloaderConfig = None
global VIDEO_DOWNLOADER_CONFIG
return VIDEO_DOWNLOADER_CONFIG
def get_videos_downloader_config() -> VideoDownloaderConfig:
global TEST_VIDEOS_DOWNLOADER_CONFIG
if isinstance(TEST_VIDEOS_DOWNLOADER_CONFIG, VideoDownloaderConfig):
return TEST_VIDEOS_DOWNLOADER_CONFIG
global VIDEOS_DOWNLOADER_CONFIG
return VIDEOS_DOWNLOADER_CONFIG
def set_test_downloader_config(enabled: bool):
global TEST_VIDEO_DOWNLOADER_CONFIG
TEST_VIDEO_DOWNLOADER_CONFIG = VideoDownloaderConfig() if enabled else None
global TEST_VIDEOS_DOWNLOADER_CONFIG
TEST_VIDEOS_DOWNLOADER_CONFIG = VideoDownloaderConfig() if enabled else None
def get_channels_config_from_db(session: Session) -> dict:

8
modules/videos/test/test_lib.py
View file

@ -270,9 +270,9 @@ async def test_format_videos_destination(async_client, test_directory):
@pytest.mark.asyncio
async def test_video_downloader_config_api(async_client, test_directory):
config = lib.get_downloader_config()
assert config.video_resolutions == ['1080p', '720p', '480p', '360p', 'maximum']
async def test_videos_downloader_config_api(async_client, test_directory, test_videos_downloader_config):
config = lib.get_videos_downloader_config()
assert config.video_resolutions == ['1080p', '720p', '480p', 'maximum']
request, response = await async_client.get('/api/config/videos_downloader.yaml')
assert response.status == HTTPStatus.OK
@ -283,7 +283,7 @@ async def test_video_downloader_config_api(async_client, test_directory):
request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body)
assert response.status == HTTPStatus.NO_CONTENT
config = lib.get_downloader_config()
config = lib.get_videos_downloader_config()
assert config.video_resolutions == ['720p', 'maximum']
assert (test_directory / 'config/videos_downloader.yaml').is_file()
assert config.is_valid()

2
modules/zim/test/test_lib.py
View file

@ -348,7 +348,7 @@ def test_zim_download_url_to_name(url, expected):
@pytest.mark.asyncio
async def test_zim_subscription_download_import(test_session, test_downloader_config):
async def test_zim_subscription_download_import(async_client, test_session):
# Subscription creates a ZimSubscription and Download
await lib.subscribe('Wikisource', 'en', session=test_session)
# Add a once-download. This should not be associated with a ZimSubscription.

62
wrolpi/common.py
View file

@ -637,8 +637,8 @@ def get_all_configs() -> Dict[str, ConfigFile]:
if inventories_config := get_inventories_config():
all_configs[inventories_config.file_name] = inventories_config
from modules.videos.lib import get_downloader_config
if videos_downloader_config := get_downloader_config():
from modules.videos.lib import get_videos_downloader_config
if videos_downloader_config := get_videos_downloader_config():
all_configs[videos_downloader_config.file_name] = videos_downloader_config
from wrolpi.downloader import get_download_manager_config
@ -705,10 +705,10 @@ class WROLPiConfig(ConfigFile):
super().import_config(file)
# Destinations cannot be empty.
self.archive_destination = self.archive_destination or self.default_config['archive_destination']
self.map_destination = self.map_destination or self.default_config['map_destination']
self.videos_destination = self.videos_destination or self.default_config['videos_destination']
self.zims_destination = self.zims_destination or self.default_config['zims_destination']
self._config['archive_destination'] = self.archive_destination or self.default_config['archive_destination']
self._config['map_destination'] = self.map_destination or self.default_config['map_destination']
self._config['videos_destination'] = self.videos_destination or self.default_config['videos_destination']
self._config['zims_destination'] = self.zims_destination or self.default_config['zims_destination']
self.successful_import = True
except Exception as e:
@ -2188,3 +2188,53 @@ def cached_multiprocessing_result(func: callable):
return result
return wrapped
def initialize_config_files() -> list[str]:
"""Creates config files only if they do not exist and will not conflict with the DB."""
from modules.inventory.common import get_inventories_config
from modules.videos.lib import get_channels_config, get_videos_downloader_config
from wrolpi.downloader import get_download_manager_config
from wrolpi.tags import get_tags_config
from wrolpi.db import get_db_session
created = [] # The names of the configs that are created.
if not get_wrolpi_config().get_file().is_file():
get_wrolpi_config().save(overwrite=True)
created.append(get_wrolpi_config().get_file().name)
with get_db_session() as session:
if not get_channels_config().get_file().is_file():
from modules.videos.models import Channel
if session.query(Channel).count() == 0:
# No Channels will be deleted, create the empty Channels config file.
get_channels_config().save(overwrite=True)
created.append(get_channels_config().get_file().name)
if not get_tags_config().get_file().is_file():
from wrolpi.tags import Tag
if session.query(Tag).count() == 0:
# No Tags will be deleted, create the empty Tags config file.
get_tags_config().save(overwrite=True)
created.append(get_tags_config().get_file().name)
if not get_download_manager_config().get_file().is_file():
from wrolpi.downloader import Download
if session.query(Download).count() == 0:
# No Downloads will be deleted, create the empty Download Manager config file.
get_download_manager_config().save(overwrite=True)
created.append(get_download_manager_config().get_file().name)
if not get_videos_downloader_config().get_file().is_file():
get_videos_downloader_config().save(overwrite=True)
created.append(get_videos_downloader_config().get_file().name)
if not get_inventories_config().get_file().is_file():
from modules.inventory.models import Inventory
if session.query(Inventory).count() == 0:
# No Inventories will be deleted, create the empty Inventory config file.
get_inventories_config().save(overwrite=True)
created.append(get_inventories_config().get_file().name)
return created

8
wrolpi/contexts.py
View file

@ -24,7 +24,7 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.inventories_config = manager.dict()
app.shared_ctx.channels_config = manager.dict()
app.shared_ctx.download_manager_config = manager.dict()
app.shared_ctx.video_downloader_config = manager.dict()
app.shared_ctx.videos_downloader_config = manager.dict()
# Shared dicts.
app.shared_ctx.refresh = manager.dict()
app.shared_ctx.uploaded_files = manager.dict()
@ -74,7 +74,7 @@ def reset_shared_contexts(app: Sanic):
app.shared_ctx.inventories_config.clear()
app.shared_ctx.channels_config.clear()
app.shared_ctx.download_manager_config.clear()
app.shared_ctx.video_downloader_config.clear()
app.shared_ctx.videos_downloader_config.clear()
# Shared dicts.
app.shared_ctx.refresh.clear()
app.shared_ctx.uploaded_files.clear()
@ -119,13 +119,13 @@ def initialize_configs_contexts(app: Sanic):
"""Assign multiprocessing Dicts to their respective FileConfigs in this process."""
from modules.inventory.common import INVENTORIES_CONFIG
from modules.videos.lib import CHANNELS_CONFIG
from modules.videos.lib import VIDEO_DOWNLOADER_CONFIG
from modules.videos.lib import VIDEOS_DOWNLOADER_CONFIG
from wrolpi.common import WROLPI_CONFIG
from wrolpi.tags import TAGS_CONFIG
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
INVENTORIES_CONFIG.initialize(app.shared_ctx.inventories_config)
CHANNELS_CONFIG.initialize(app.shared_ctx.channels_config)
VIDEO_DOWNLOADER_CONFIG.initialize(app.shared_ctx.video_downloader_config)
VIDEOS_DOWNLOADER_CONFIG.initialize(app.shared_ctx.videos_downloader_config)
WROLPI_CONFIG.initialize(app.shared_ctx.wrolpi_config)
TAGS_CONFIG.initialize(app.shared_ctx.tags_config)
DOWNLOAD_MANAGER_CONFIG.initialize(app.shared_ctx.download_manager_config)

2
wrolpi/downloader.py
View file

@ -709,8 +709,6 @@ class DownloadManager:
stmt = "UPDATE download SET status='new' WHERE status='pending' OR status='deferred'"
curs.execute(stmt)
save_downloads_config.activate_switch()
@optional_session
def get_new_downloads(self, session: Session) -> Generator[Download, None, None]:
"""

55
wrolpi/test/test_common.py
View file

@ -1056,3 +1056,58 @@ async def test_cached_multiprocessing_result(async_client):
# Kwargs are supported
assert await func(inc=1) == 4
assert await func(inc=1) == 4
@pytest.mark.asyncio
async def test_initialize_config_files(async_client, test_session, test_directory, test_wrolpi_config,
test_channels_config, test_tags_config, test_videos_downloader_config,
test_download_manager_config, simple_channel, test_inventory, tag_factory,
test_download_manager, test_downloader):
"""Config files should only be created if they are missing, and they will not overwrite existing entries in the DB."""
from modules.videos.lib import get_videos_downloader_config, get_channels_config
from wrolpi.tags import get_tags_config
from wrolpi.downloader import get_download_manager_config
from modules.inventory.common import get_inventories_config
# Create some DB entries so the configs will not be imported.
tag1, tag2 = await tag_factory(), await tag_factory()
download = test_download_manager.create_download('https://example.com', downloader_name=test_downloader.name)
test_session.commit()
# Write some config files, they should not be re-created.
get_wrolpi_config().get_file().write_text('wrolpi')
get_videos_downloader_config().get_file().write_text('video downloader')
assert common.initialize_config_files() == [], 'No configs should have been created.'
# Config files were not overwritten with real configs.
assert get_wrolpi_config().get_file().read_text() == 'wrolpi'
assert get_videos_downloader_config().get_file().read_text() == 'video downloader'
# WROLPi config can be created because it is now missing.
get_wrolpi_config().get_file().unlink()
assert common.initialize_config_files() == ['wrolpi.yaml'], 'WROLPi config should have been created.'
assert get_wrolpi_config().is_valid()
# Videos Downloader config can be created because it is now missing.
get_videos_downloader_config().get_file().unlink()
assert common.initialize_config_files() == ['videos_downloader.yaml'], \
'Videos Downloader config should have been created.'
assert get_videos_downloader_config().is_valid()
# Channels config file does not exist, no conflicting Channels so the file is created.
simple_channel.delete_with_videos()
assert common.initialize_config_files() == ['channels.yaml'], 'Channels config should have been created.'
assert get_channels_config().is_valid()
tag1.delete()
tag2.delete()
assert common.initialize_config_files() == ['tags.yaml'], 'Tags config should have been created.'
assert get_tags_config().is_valid()
download.delete()
assert common.initialize_config_files() == ['download_manager.yaml'], \
'Download Manager config should have been created.'
assert get_download_manager_config().is_valid()
test_session.delete(test_inventory)
assert common.initialize_config_files() == ['inventories.yaml'], 'Inventory config should have been created.'
assert get_inventories_config().is_valid()