From acf3dee179450d146bae39776b267aaa24321713 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 28 Sep 2024 17:28:33 -0600 Subject: [PATCH] Configs are displayed in new ConfigsPage under Admin. - Configs can be imported and saved. - Configs are validated before imported. - API will refuse to save a config if it was not imported successfully. --- app/src/Events.js | 8 ++ app/src/api.js | 49 +++++++ app/src/components/Files.js | 1 - app/src/components/Placeholder.js | 6 + app/src/components/admin/AdminRoute.js | 3 + app/src/components/admin/Configs.js | 148 ++++++++++++++++++++ app/src/components/admin/Settings.js | 17 ++- app/src/hooks/customHooks.js | 44 +++++- modules/inventory/common.py | 70 ++++++---- modules/inventory/models.py | 6 +- modules/inventory/test/test_inventory.py | 13 ++ modules/videos/lib.py | 45 +++++- wrolpi/api_utils.py | 2 + wrolpi/common.py | 169 ++++++++++++++++++----- wrolpi/downloader.py | 81 ++++++++--- wrolpi/events.py | 8 ++ wrolpi/root_api.py | 36 ++++- wrolpi/schema.py | 6 + wrolpi/status.py | 2 +- wrolpi/tags.py | 124 ++++++++++------- wrolpi/test/test_common.py | 37 ++++- wrolpi/test/test_root_api.py | 49 +++++++ 22 files changed, 767 insertions(+), 157 deletions(-) create mode 100644 app/src/components/admin/Configs.js diff --git a/app/src/Events.js b/app/src/Events.js index bfb1ec7a..0aa66f35 100644 --- a/app/src/Events.js +++ b/app/src/Events.js @@ -83,6 +83,14 @@ function handleEvents(events) { eventToast('Move Failed', message, 'error'); } + if (event === 'config_import_failed') { + eventToast('Config Import Failed', message, 'error'); + } + + if (event === 'config_save_failed') { + eventToast('Config Save Failed', message, 'error'); + } + if (subject) { newestEvents[subject] = dt; } diff --git a/app/src/api.js b/app/src/api.js index c75fa449..a214f8d9 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -313,6 +313,55 @@ export async function getDownloads() { return response } +export async function getConfigs() { + const response = await apiGet(`${API_URI}/configs`); + if (response.ok) { + const data = await response.json(); + return { + configs: data.configs, + } + } else { + const message = getErrorMessage(response, 'Could not get configs.'); + toast({ + type: 'error', + title: 'Getting Configs Failed', + description: message, + time: 5000, + }); + } + return response +} + +export async function postImportConfig(fileName) { + const body = {file_name: fileName}; + const response = await apiPost(`${API_URI}/configs/import`, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not import config.'); + toast({ + type: 'error', + title: 'Importing config Failed', + description: message, + time: 5000, + }); + } + return response +} + +export async function postSaveConfig(fileName) { + const body = {file_name: fileName, overwrite: true}; + const response = await apiPost(`${API_URI}/configs/save`, body); + if (!response.ok) { + const message = getErrorMessage(response, 'Could not save config.'); + toast({ + type: 'error', + title: 'Saving config Failed', + description: message, + time: 5000, + }); + } + return response +} + export async function validateRegex(regex) { const response = await apiPost(`${API_URI}/valid_regex`, {regex}); try { diff --git a/app/src/components/Files.js b/app/src/components/Files.js index dcb9676c..d9a472aa 100644 --- a/app/src/components/Files.js +++ b/app/src/components/Files.js @@ -25,7 +25,6 @@ import { HandPointMessage, humanFileSize, isoDatetimeToAgoPopup, - isoDatetimeToString, mimetypeColor, PageContainer, Paginator, diff --git a/app/src/components/Placeholder.js b/app/src/components/Placeholder.js index debd9fa2..5da13fce 100644 --- a/app/src/components/Placeholder.js +++ b/app/src/components/Placeholder.js @@ -53,3 +53,9 @@ export function TagPlaceholder() { } + +export function TableRowPlaceholder() { + return + + +} \ No newline at end of file diff --git a/app/src/components/admin/AdminRoute.js b/app/src/components/admin/AdminRoute.js index f0e6f50d..b9e5b9b4 100644 --- a/app/src/components/admin/AdminRoute.js +++ b/app/src/components/admin/AdminRoute.js @@ -8,6 +8,7 @@ import {Header, Loader, Segment} from "../Theme"; import {DownloadsPage} from "./Downloads"; import {SettingsPage} from "./Settings"; import {StatusPage} from "./Status"; +import {ConfigsPage} from "./Configs"; class WROLMode extends React.Component { @@ -71,6 +72,7 @@ export default function AdminRoute() { {text: 'Downloads', to: '/admin', key: 'admin', end: true}, {text: 'Settings', to: '/admin/settings', key: 'settings'}, {text: 'Status', to: '/admin/status', key: 'status'}, + {text: 'Configs', to: '/admin/configs', key: 'configs'}, {text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'}, ]; @@ -80,6 +82,7 @@ export default function AdminRoute() { }/> }/> }/> + }/> }/> diff --git a/app/src/components/admin/Configs.js b/app/src/components/admin/Configs.js new file mode 100644 index 00000000..5bb47dc4 --- /dev/null +++ b/app/src/components/admin/Configs.js @@ -0,0 +1,148 @@ +import {useConfigs} from "../../hooks/customHooks"; +import {APIButton, PageContainer} from "../Common"; +import {Header, Icon, Popup, Table} from "../Theme"; +import React from "react"; +import { + Grid, + GridColumn, + GridRow, + TableBody, + TableCell, + TableHeader, + TableHeaderCell, + TableRow +} from "semantic-ui-react"; +import {TableRowPlaceholder} from "../Placeholder"; +import {useNavigate} from "react-router-dom"; + +function WarningIcon({ok}) { + return +} + +function ConfigTableRow({config, importConfig, saveConfig, fetchConfigs}) { + const {file_name, rel_path, valid, successful_import} = config; + const navigate = useNavigate(); + + const localImportConfig = async () => { + await importConfig(file_name); + await fetchConfigs(); + } + + const localSaveConfig = async () => { + await saveConfig(file_name); + await fetchConfigs(); + } + + const importButton = ; + + let saveButton; + if (successful_import) { + saveButton = ; + } else { + // Warn the user they may overwrite because import was not successful. + saveButton = ; + } + + return + navigate(`?preview=${rel_path}`)}>{file_name} + + + {importButton} + {saveButton} + +} + +function ConfigsTable({configs, loading, importConfig, saveConfig, fetchConfigs}) { + let body = + + + + if (loading) { + // Configs are being fetched again. + } else if (configs === undefined) { + body = + Failed to fetch configs + ; + } else if (configs && Object.keys(configs).length === 0) { + body = + No configs exist. Try tagging some files. + ; + } else if (configs) { + body = Object.entries(configs).map(([key, value]) => + + ); + } + + return + + + File Name + Imported + Valid + Import + Save + + + + + {body} + +
+} + +export function ConfigsPage() { + const {configs, loading, importConfig, saveConfig, fetchConfigs} = useConfigs(); + + React.useEffect(() => { + fetchConfigs(); + }, []); + + const refreshButton = ; + const refreshPopupButton = ; + + return + + + +
Configs
+ + {refreshPopupButton} + + + + + +} \ No newline at end of file diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js index 351de605..676eb6e4 100644 --- a/app/src/components/admin/Settings.js +++ b/app/src/components/admin/Settings.js @@ -297,8 +297,17 @@ export class SettingsPage extends React.Component { } render() { + const controlSegment = +
Control WROLPi
+ + + ; + if (this.state.ready === false) { - return + return <> + + {controlSegment} + } else if (this.state.ready === undefined) { return Unable to fetch settings } @@ -539,11 +548,7 @@ export class SettingsPage extends React.Component { - -
Control WROLPi
- - - + {controlSegment} ; } } diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 0f4c86ff..4172c855 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -7,6 +7,7 @@ import { getArchive, getChannel, getChannels, + getConfigs, getDirectories, getDownloads, getFiles, @@ -17,7 +18,10 @@ import { getStatus, getVideo, getVideosStatistics, - searchArchives, searchChannels, + postImportConfig, + postSaveConfig, + searchArchives, + searchChannels, searchDirectories, searchVideos, searchZim, @@ -679,6 +683,44 @@ export const useDownloads = () => { return {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads} } +export const useConfigs = () => { + const [configs, setConfigs] = useState(null); + const [loading, setLoading] = useState(false); + + const fetchConfigs = async () => { + setLoading(true); + console.log('fetching configs...'); + try { + const data = await getConfigs(); + setConfigs(data['configs']); + console.debug('fetching configs successful'); + } catch (e) { + console.error(e); + setConfigs(undefined); + } finally { + setLoading(false); + } + } + + const importConfig = async (fileName) => { + try { + await postImportConfig(fileName); + } catch (e) { + console.error(e); + } + } + + const saveConfig = async (fileName) => { + try { + await postSaveConfig(fileName); + } catch (e) { + console.error(e); + } + } + + return {configs, loading, fetchConfigs, importConfig, saveConfig} +} + export const useThrottle = () => { const [on, setOn] = useState(null); const {settings, fetchSettings} = React.useContext(SettingsContext); diff --git a/modules/inventory/common.py b/modules/inventory/common.py index f8749149..8601096d 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -1,4 +1,5 @@ import pathlib +from dataclasses import field, dataclass from decimal import Decimal from functools import partial from pathlib import Path @@ -8,6 +9,7 @@ from pint import Quantity from wrolpi.common import logger, Base, ConfigFile from wrolpi.db import get_db_session +from wrolpi.events import Events from .errors import NoInventories from .inventory import get_items, get_inventories, get_unit_registry from .models import Inventory, Item @@ -116,12 +118,19 @@ def cleanup_quantity(quantity: Quantity) -> Quantity: return Decimal(num) * unit +@dataclass +class InventoriesConfigValidator: + version: int = None + inventories: list[str] = field(default_factory=list) + + class InventoriesConfig(ConfigFile): file_name = 'inventories.yaml' default_config = dict( inventories=[], version=0, ) + validator = InventoriesConfigValidator @property def inventories(self) -> dict: @@ -131,7 +140,7 @@ class InventoriesConfig(ConfigFile): def inventories(self, value: dict): self.update({'inventories': value}) - def dump_config(self, file: pathlib.Path = None): + def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False): inventories = [] for inventory in get_inventories(): inventories.append(inventory.dict()) @@ -140,46 +149,51 @@ class InventoriesConfig(ConfigFile): raise NoInventories('No Inventories are in the database!') self._config['inventories'] = inventories - super().dump_config(file) + super().dump_config(file, send_events, overwrite) - def import_config(self, file: pathlib.Path = None): + def import_config(self, file: pathlib.Path = None, send_events=False): super().import_config(file) try: with get_db_session(commit=True) as session: - db_inventories = get_inventories(session) - config_inventory_names = {i['name'] for i in self.inventories} - keep_inventories_names = [] - for inventory in db_inventories: - if inventory.name in config_inventory_names: - keep_inventories_names.append(inventory.name) - # TODO delete missing items, add new items. - else: - # Inventory was deleted from the config file. - inventory.delete() - continue + db_inventories = session.query(Inventory).all() + db_inventories_by_name = {i.name: i for i in db_inventories} + config_inventories = [i['name'] for i in self.inventories] + for inventory_dict in self.inventories: + if inventory_dict['name'] not in db_inventories_by_name: + inventory = Inventory( + name=inventory_dict['name'], + created_at=inventory_dict['created_at'], + deleted_at=inventory_dict['deleted_at'], + ) + session.add(inventory) + # Get the Inventory from the DB so we can use its ID. + inventory.flush() + db_inventories_by_name[inventory.name] = inventory - new_inventories = [i for i in self.inventories if i['name'] not in keep_inventories_names] - for inventory_dict in new_inventories: - # New Inventory in config, copy it into the DB. - inventory = Inventory( - name=inventory_dict['name'], - created_at=inventory_dict['created_at'], - deleted_at=inventory_dict['deleted_at'], - ) - session.add(inventory) - # Get the Inventory from the DB so we can use its ID. - inventory.flush() + # Delete any inventories no longer in the config. + for name, inventory in db_inventories_by_name.items(): + if name not in config_inventories: + session.delete(inventory) + for inventory_dict in self.inventories: + # Items are complex and do not have useful primary keys. Delete all items, create new items. + session.query(Item).filter(Item.inventory_id == inventory_dict['id']).delete() + inventory = db_inventories_by_name[inventory_dict['name']] for item in inventory_dict['items']: del item['inventory_id'] - item = Item(inventory_id=inventory.id, **item) - item.inventory_id = inventory.id + del item['id'] + item = Item(inventory=inventory, inventory_id=inventory.id, **item) session.add(item) + item.flush() self.successful_import = True except Exception as e: - logger.error(f'Failed to import {self.file_name}', exc_info=e) + self.successful_import = False + message = f'Failed to import {self.file_name}' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_import_failed(message) raise diff --git a/modules/inventory/models.py b/modules/inventory/models.py index 64d4c365..c8289204 100644 --- a/modules/inventory/models.py +++ b/modules/inventory/models.py @@ -1,5 +1,5 @@ from sqlalchemy import Column, Integer, String, Date, ForeignKey, DECIMAL -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, Session from sqlalchemy.orm.collections import InstrumentedList from wrolpi.common import Base, ModelHelper @@ -56,6 +56,10 @@ class Inventory(Base, ModelHelper): """ self.deleted_at = now() + @staticmethod + def find_by_name(session: Session, name: str) -> 'Inventory': + return session.query(Inventory).filter_by(name=name).one() + class InventoriesVersion(Base, ModelHelper): __tablename__ = 'inventories_version' diff --git a/modules/inventory/test/test_inventory.py b/modules/inventory/test/test_inventory.py index fd45ba90..5e5a8bc3 100644 --- a/modules/inventory/test/test_inventory.py +++ b/modules/inventory/test/test_inventory.py @@ -303,3 +303,16 @@ async def test_inventories_config(async_client, test_session, test_directory, in test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]} db_items = {(i.name, i.brand, i.count) for i in non_deleted_items} assert test_items == db_items + + # An extra Inventory with items should be deleted from the DB. + inventory = Inventory(name='delete me') + test_session.add(inventory) + inventory.flush() + test_session.add(Item(inventory_id=inventory.id)) + test_session.commit() + assert test_session.query(Inventory).count() == 2 + assert {i for i, in test_session.query(Inventory.name)} == {'Food Storage', 'delete me'} + + import_inventories_config() + assert test_session.query(Inventory).count() == 1 + assert test_session.query(Inventory).one().name == 'Food Storage' diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 68720c48..95001b8c 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -3,6 +3,7 @@ import dataclasses import html import pathlib import re +from dataclasses import dataclass from datetime import datetime from typing import Generator, Type from typing import Tuple, Optional @@ -20,6 +21,7 @@ from wrolpi.dates import Seconds, from_timestamp from wrolpi.db import get_db_curs, get_db_session from wrolpi.downloader import Download from wrolpi.errors import UnknownDirectory +from wrolpi.events import Events from wrolpi.files.lib import split_path_stem_and_suffix from wrolpi.switches import register_switch_handler, ActivateSwitchMethod from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST @@ -207,6 +209,20 @@ def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Op return None, None +@dataclass +class ChannelDictValidator: + name: str + directory: str + download_frequency: int + url: str = None + + +@dataclass +class ChannelsConfigValidator: + version: int = None + channels: list[ChannelDictValidator] = dataclasses.field(default_factory=list) + + class ChannelsConfig(ConfigFile): file_name = 'channels.yaml' default_config = dict( @@ -218,6 +234,7 @@ class ChannelsConfig(ConfigFile): )], version=0, ) + validator = ChannelsConfigValidator @property def channels(self) -> dict: @@ -227,7 +244,7 @@ class ChannelsConfig(ConfigFile): def channels(self, value: dict): self.update({'channels': value}) - def import_config(self, file: pathlib.Path = None): + def import_config(self, file: pathlib.Path = None, send_events=False): from modules.videos.channel.lib import get_channel super().import_config() try: @@ -296,10 +313,12 @@ class ChannelsConfig(ConfigFile): self.successful_import = True channel_import_logger.info('Importing channels config complete') except Exception as e: - channel_import_logger.warning('Failed to load channels config!', exc_info=e) - if PYTEST: - # Do not interrupt startup, only raise during testing. - raise + self.successful_import = False + message = f'Failed to import {self.get_relative_file()} config!' + if send_events: + channel_import_logger.warning(message, exc_info=e) + Events.send_config_import_failed(message) + raise CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig() @@ -328,6 +347,21 @@ def set_test_channels_config(): TEST_CHANNELS_CONFIG = None +@dataclass +class VideoDownloaderConfigValidator: + continue_dl: bool + dateafter: str + file_name_format: str + nooverwrites: bool + quiet: bool + writeautomaticsub: bool + writeinfojson: bool + writesubtitles: bool + writethumbnail: bool + youtube_include_dash_manifest: bool + version: int = None + + class VideoDownloaderConfig(ConfigFile): file_name = 'videos_downloader.yaml' default_config = dict( @@ -343,6 +377,7 @@ class VideoDownloaderConfig(ConfigFile): writethumbnail=True, youtube_include_dash_manifest=False, ) + validator = VideoDownloaderConfigValidator @property def continue_dl(self) -> bool: diff --git a/wrolpi/api_utils.py b/wrolpi/api_utils.py index a316e76c..023b0c97 100644 --- a/wrolpi/api_utils.py +++ b/wrolpi/api_utils.py @@ -111,6 +111,8 @@ def json_error_handler(request: Request, exception: Exception): return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR) +api_app.error_handler.add(Exception, json_error_handler) + PERPETUAL_WORKERS = list() diff --git a/wrolpi/common.py b/wrolpi/common.py index e7eb8a3c..572f9361 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -14,10 +14,9 @@ import socket import string import sys import tempfile -import traceback from asyncio import Task from copy import deepcopy -from dataclasses import dataclass +from dataclasses import dataclass, asdict, field, fields from datetime import datetime, date from decimal import Decimal from functools import wraps @@ -26,7 +25,7 @@ from itertools import islice, filterfalse, tee from multiprocessing.managers import DictProxy from pathlib import Path from types import GeneratorType -from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set, Coroutine +from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set from urllib.parse import urlparse, urlunsplit import aiohttp @@ -41,7 +40,7 @@ from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import Session from wrolpi.dates import now, from_timestamp, seconds_to_timestamp -from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError +from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS, LOG_LEVEL LOGGING_CONFIG = { @@ -192,6 +191,7 @@ __all__ = [ 'format_html_string', 'format_json_file', 'get_absolute_media_path', + 'get_all_configs', 'get_download_info', 'get_files_and_directories', 'get_global_statistics', @@ -346,10 +346,12 @@ class ConfigFile: Configs cannot be saved until they are imported without error. """ - file_name: str = None - default_config: dict = None - width: int = None + background_dump: callable = None background_save: callable = None + default_config: dict = None + file_name: str = None + validator: dataclass = None + width: int = None def __init__(self): self.width = self.width or 90 @@ -377,6 +379,15 @@ class ConfigFile: def __repr__(self): return f'<{self.__class__.__name__} file={self.get_file()}>' + def __json__(self) -> dict: + d = dict( + file_name=self.file_name, + rel_path=get_relative_to_media_directory(self.get_file()), + successful_import=self.successful_import, + valid=self.is_valid(), + ) + return d + def initialize(self, multiprocessing_dict: Optional[DictProxy] = None): """Initializes this config dict using the default config and the config file.""" # Use the provided multiprocessing.Manager().dict(), or dict() for testing. @@ -386,8 +397,10 @@ class ConfigFile: file = self.get_file() if file.is_file(): - config_data = {k: v for k, v in self.read_config_file(file).items() if k in self.default_config} - self._config.update(config_data) + if not self.is_valid(file): + raise InvalidConfig(f'Config file is invalid: {str(self.get_relative_file())}') + config_data = self.read_config_file(file) + self._config.update(asdict(self.validator(**config_data))) return self @property @@ -405,7 +418,7 @@ class ConfigFile: with file.open('rt') as fh: config_data = yaml.load(fh, Loader=yaml.Loader) if not isinstance(config_data, dict): - raise RuntimeError(f'Config file is invalid: {file}') + raise InvalidConfig(f'Config file is invalid: {file}') return config_data def _get_backup_filename(self): @@ -417,9 +430,13 @@ class ConfigFile: path = path.with_name(name) return path - def save(self, file: pathlib.Path = None): + def save(self, file: pathlib.Path = None, send_events: bool = False, overwrite: bool = False): """ Write this config to its file. + + @param file: The destination of the config file (defaults to `self.get_file()`). + @param send_events: Send failure Events to UI. + @param overwrite: Will overwrite the config file even if it was not imported successfully. """ from wrolpi.api_utils import api_app @@ -429,13 +446,12 @@ class ConfigFile: rel_path = get_relative_to_media_directory(file) - if file.exists(): + if file.exists() and overwrite is False: if not self.successful_import: raise RuntimeError(f'Refusing to save config because it was never successfully imported! {rel_path}') version = self.read_config_file(file).get('version') if version and version > self.version: - raise RuntimeError(f'Refusing to overwrite newer config ({rel_path}): ' - f'{version} > {self.version}') + raise RuntimeError(f'Refusing to overwrite newer config ({rel_path}): {version} > {self.version}') # Don't overwrite a real config while testing. if PYTEST and not str(file).startswith('/tmp'): @@ -469,7 +485,11 @@ class ConfigFile: logger_.info(f'Saved config: {rel_path}') except Exception as e: # Configs are vital, raise a big error when this fails. - logger_.critical(f'Failed to save config: {rel_path}', exc_info=e) + message = f'Failed to save config: {rel_path}' + logger_.critical(message, exc_info=e) + if send_events: + from wrolpi.events import Events + Events.send_config_save_failed(message) raise e def write_config_data(self, config: dict, config_file: pathlib.Path): @@ -488,11 +508,19 @@ class ConfigFile: return CONFIG_DIR / self.file_name - def import_config(self, file: pathlib.Path = None): + def get_relative_file(self): + return get_relative_to_media_directory(self.get_file()) + + def import_config(self, file: pathlib.Path = None, send_events=False): """Read config file data, apply it to the in-memory config and database.""" file = file or self.get_file() file_str = str(get_relative_to_media_directory(file)) + # Caller will set to successful if it works. + self.successful_import = False if file.is_file(): + if not self.is_valid(file): + raise InvalidConfig(f'Config is invalid: {file_str}') + data = self.read_config_file(file) new_data, extra_data = partition(lambda i: i[0] in self.default_config, data.items()) new_data, extra_data = dict(new_data), dict(extra_data) @@ -504,10 +532,10 @@ class ConfigFile: else: logger_.error(f'Failed to import {file_str} because it does not exist.') - def dump_config(self, file: pathlib.Path = None): + def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False): """Dump database data, copy it to this the in-memory config, then write it to the config file.""" # Copy the data as-is by default. Other classes will overwrite this and dump the database before saving. - self.save(file) + self.save(file, send_events, overwrite) def update(self, config: dict): """Update any values of this config. Save the config to its file.""" @@ -524,6 +552,28 @@ class ConfigFile: return deepcopy(self._config) + def is_valid(self, file: pathlib.Path = None) -> bool: + if not self.validator: + raise NotImplementedError(f'Cannot validate {self.file_name} without validator!') + + file = file or self.get_file() + if not file.is_file(): + return False + + allowed_fields = {i.name for i in fields(self.validator)} + try: + # Remove keys no longer in the config. + config_items = self.read_config_file(file).items() + extra_items = {k: v for k, v in config_items if k not in allowed_fields} + if extra_items: + logger.error(f'Invalid config items ({get_relative_to_media_directory(file)}): {extra_items}') + config = {k: v for k, v in config_items if k in allowed_fields} + self.validator(**config) + return True + except Exception as e: + logger.debug(f'Failed to validate config: {file}', exc_info=e) + return False + # `version` is used to prevent overwriting of newer configs. Cannot not be modified directly. @property @@ -531,6 +581,63 @@ class ConfigFile: return self._config['version'] +@dataclass +class WROLPiConfigValidator: + archive_destination: str = None + download_on_startup: bool = None + download_timeout: int = None + hotspot_device: str = None + hotspot_on_startup: bool = None + hotspot_password: str = None + hotspot_ssid: str = None + ignore_outdated_zims: bool = None + map_destination: str = None + nav_color: str = None + throttle_on_startup: bool = None + version: int = None + videos_destination: str = None + wrol_mode: bool = None + zims_destination: str = None + ignored_directories: list[str] = field(default_factory=list) + + +def get_all_configs() -> Dict[str, ConfigFile]: + all_configs = dict() + + if wrolpi_config := get_wrolpi_config(): + all_configs[wrolpi_config.file_name] = wrolpi_config + + from wrolpi.tags import get_tags_config + if tags_config := get_tags_config(): + all_configs[tags_config.file_name] = tags_config + + from modules.videos.lib import get_channels_config + if channels_config := get_channels_config(): + all_configs[channels_config.file_name] = channels_config + + from modules.inventory.common import get_inventories_config + if inventories_config := get_inventories_config(): + all_configs[inventories_config.file_name] = inventories_config + + from modules.videos.lib import get_downloader_config + if videos_downloader_config := get_downloader_config(): + all_configs[videos_downloader_config.file_name] = videos_downloader_config + + from wrolpi.downloader import get_download_manager_config + if download_manager_config := get_download_manager_config(): + all_configs[download_manager_config.file_name] = download_manager_config + + return all_configs + + +def get_config_by_file_name(file_name: str) -> ConfigFile: + configs = get_all_configs() + if file_name not in configs: + raise InvalidConfig(f'No config with {file_name} exists') + + return configs[file_name] + + class WROLPiConfig(ConfigFile): file_name = 'wrolpi.yaml' default_config = dict( @@ -551,6 +658,7 @@ class WROLPiConfig(ConfigFile): wrol_mode=False, zims_destination='zims', ) + validator = WROLPiConfigValidator def get_file(self) -> Path: """WROLPiConfig must be discovered so that other config files can be found.""" @@ -573,13 +681,18 @@ class WROLPiConfig(ConfigFile): # Not testing, and can't find file deep in media directory. Use the default. return default_config_path - def import_config(self, file: pathlib.Path = None): + def import_config(self, file: pathlib.Path = None, send_events=False): try: super().import_config(file) # WROLPiConfig does not sync to database. self.successful_import = True except Exception as e: - logger.error(f'Failed to import {self.file_name}', exc_info=e) + self.successful_import = False + message = f'Failed to import {self.file_name}' + logger.error(message, exc_info=e) + if send_events: + from wrolpi.events import Events + Events.send_config_import_failed(message) raise @property @@ -820,7 +933,6 @@ def run_after(after: callable, *args, **kwargs) -> callable: async def after(*a, **kw): if logger.isEnabledFor(logging.DEBUG): - traceback.print_stack() logger.debug(f'run_after synchronous_after called for {after}') return synchronous_after(*a, **kw) @@ -833,7 +945,6 @@ def run_after(after: callable, *args, **kwargs) -> callable: async def wrapped(*a, **kw): results = await func(*a, **kw) if logger.isEnabledFor(logging.DEBUG): - traceback.print_stack() logger.debug(f'run_after async called for {func}') coro = after(*args, **kwargs) asyncio.ensure_future(coro) @@ -843,7 +954,6 @@ def run_after(after: callable, *args, **kwargs) -> callable: def wrapped(*a, **kw): results = func(*a, **kw) if logger.isEnabledFor(logging.DEBUG): - traceback.print_stack() logger.debug(f'run_after sync called for {func}') coro = after(*args, **kwargs) asyncio.ensure_future(coro) @@ -1483,7 +1593,7 @@ def chunks_by_stem(it: List[Union[pathlib.Path, str, int]], size: int) -> Genera @contextlib.contextmanager -def timer(name, level: str = 'debug', logger__: logging.Logger = logger_): +def timer(name, level: str = 'debug', logger__: logging.Logger = None): """Prints out the time elapsed during the call of some block. Example: @@ -1491,6 +1601,7 @@ def timer(name, level: str = 'debug', logger__: logging.Logger = logger_): time.sleep(10) """ + logger__ = logger__ or logger_ before = datetime.now() log_method = getattr(logger__, level) try: @@ -1500,16 +1611,6 @@ def timer(name, level: str = 'debug', logger__: logging.Logger = logger_): log_method(f'{name} elapsed {elapsed} seconds') -def async_timer(coro: Coroutine, name: str = 'async timer', level: str = 'debug') -> callable: - """Returns a new coroutine which prints out time elapsed when calling the coroutine.""" - - async def _(): - with timer(name, level): - return await coro - - return _() - - @contextlib.contextmanager def slow_logger(max_seconds: int, message: str, logger__=logger_, level=logging.WARNING): """Only logs when the context duration exceeds the `max_seconds`.""" diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 82d2608b..49f0cad0 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -31,6 +31,7 @@ from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, C from wrolpi.dates import TZDateTime, now, Seconds from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload +from wrolpi.events import Events from wrolpi.switches import register_switch_handler, ActivateSwitchMethod, await_switches from wrolpi.vars import PYTEST, SIMULTANEOUS_DOWNLOAD_DOMAINS @@ -1143,6 +1144,25 @@ async def signal_download_download(download_id: int, download_url: str): download_manager._delete_processing_domain(download_domain) +@dataclass +class DownloadDictConfig: + downloader: str + last_successful_download: str + next_download: str + url: str + frequency: int = None + settings: dict = None + status: str = None + sub_downloader: str = None + + +@dataclass +class DownloadManagerConfigValidator: + version: int = None + downloads: list[DownloadDictConfig] = field(default_factory=list) + skip_urls: list[str] = field(default_factory=list) + + class DownloadManagerConfig(ConfigFile): file_name = 'download_manager.yaml' default_config = dict( @@ -1150,6 +1170,7 @@ class DownloadManagerConfig(ConfigFile): skip_urls=[], version=0, ) + validator = DownloadManagerConfigValidator @property def skip_urls(self) -> List[str]: @@ -1167,27 +1188,33 @@ class DownloadManagerConfig(ConfigFile): def downloads(self, value: List[dict]): self.update({'downloads': value}) - def dump_config(self, file: pathlib.Path = None): - with get_db_session() as session: - downloads: Iterable[Download] = session.query(Download).order_by(Download.url) - new_downloads = [] - for download in downloads: - if download.last_successful_download and not download.frequency: - # This once-download has completed, do not save it. - continue - new_downloads.append(dict( - downloader=download.downloader, - frequency=download.frequency, - last_successful_download=download.last_successful_download, - next_download=download.next_download, - settings=download.settings, - status=download.status, - sub_downloader=download.sub_downloader, - url=download.url, - )) - get_download_manager_config().downloads = new_downloads + def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False): + try: + with get_db_session() as session: + downloads: Iterable[Download] = session.query(Download).order_by(Download.url) + new_downloads = [] + for download in downloads: + if download.last_successful_download and not download.frequency: + # This once-download has completed, do not save it. + continue + new_downloads.append(dict( + downloader=download.downloader, + frequency=download.frequency, + last_successful_download=download.last_successful_download, + next_download=download.next_download, + settings=download.settings, + status=download.status, + sub_downloader=download.sub_downloader, + url=download.url, + )) + get_download_manager_config().downloads = new_downloads + except Exception as e: + message = f'Failed to save {self.get_relative_file()} config' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_save_failed(message) - def import_config(self, file: pathlib.Path = None): + def import_config(self, file: pathlib.Path = None, send_events=False): super().import_config(file) with get_db_session(commit=True) as session: from modules.zim.lib import zim_download_url_to_name @@ -1231,8 +1258,13 @@ class DownloadManagerConfig(ConfigFile): logger.info(f'Adding new download {url}') session.commit() + self.successful_import = True except Exception as e: - logger.error('Failed to import downloads', exc_info=e) + self.successful_import = False + message = f'Failed to import {self.file_name}' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_import_failed(message) raise try: @@ -1254,8 +1286,13 @@ class DownloadManagerConfig(ConfigFile): if need_commit: session.commit() + self.successful_import = True except Exception as e: - logger.error('Failed to restore ZimSubscriptions', exc_info=e) + self.successful_import = False + message = 'Failed to restore ZimSubscriptions' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_import_failed(message) raise self.successful_import = True diff --git a/wrolpi/events.py b/wrolpi/events.py index e4603271..ce61378a 100644 --- a/wrolpi/events.py +++ b/wrolpi/events.py @@ -84,6 +84,14 @@ class Events: def send_file_move_failed(message: str = None): send_event('file_move_failed', message, subject='refresh') + @staticmethod + def send_config_import_failed(message: str = None): + send_event('config_import_failed', message, subject='configs') + + @staticmethod + def send_config_save_failed(message: str = None): + send_event('config_save_failed', message, subject='configs') + def log_event(event: str, message: str = None, action: str = None, subject: str = None): log = f'{event=}' diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index b383a2a8..afae85e6 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -20,10 +20,11 @@ from modules.zim.api import zim_bp from wrolpi import admin, flags, schema, dates from wrolpi import tags from wrolpi.admin import HotspotStatus -from wrolpi.api_utils import json_response, json_error_handler, api_app +from wrolpi.api_utils import json_response, api_app from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \ - set_global_log_level, get_relative_to_media_directory, search_other_estimates + set_global_log_level, get_relative_to_media_directory, search_other_estimates, get_all_configs, \ + get_config_by_file_name from wrolpi.dates import now from wrolpi.downloader import download_manager from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \ @@ -605,4 +606,33 @@ async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstima return json_response(ret) -api_app.error_handler.add(Exception, json_error_handler) +@api_bp.get('/configs') +def get_configs(_: Request): + configs = get_all_configs() + return json_response(dict(configs=configs)) + + +@api_bp.post('/configs/import') +@validate(json=schema.ConfigsImportRequest) +def post_configs_import(_: Request, body: schema.ConfigsImportRequest): + config = get_config_by_file_name(body.file_name) + try: + config.import_config(send_events=True) + except Exception as e: + logger.error(f'Failed to import config: {body.file_name}', exc_info=e) + raise InvalidConfig(f'Failed to import config {body.file_name}') + + return response.empty() + + +@api_bp.post('/configs/save') +@validate(json=schema.ConfigsImportRequest) +def post_configs_save(_: Request, body: schema.ConfigsImportRequest): + config = get_config_by_file_name(body.file_name) + try: + config.dump_config(send_events=True, overwrite=body.overwrite) + except Exception as e: + logger.error(f'Failed to dump config: {body.file_name}', exc_info=e) + raise InvalidConfig(f'Failed to dump config {body.file_name}') + + return response.empty() diff --git a/wrolpi/schema.py b/wrolpi/schema.py index bde4862d..4acb898e 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -213,3 +213,9 @@ class SearchFileEstimateRequest: @dataclass class SearchOtherEstimateRequest: tag_names: List[str] = field(default_factory=lambda: list()) + + +@dataclass +class ConfigsImportRequest: + file_name: str + overwrite: bool = False diff --git a/wrolpi/status.py b/wrolpi/status.py index f50b5008..99e8e12d 100755 --- a/wrolpi/status.py +++ b/wrolpi/status.py @@ -527,7 +527,7 @@ async def status_worker(count: int = None, sleep_time: int = 5): # Always sleep so this does not become a busy loop with errors. if load_stats and load_stats.minute_1 and load_stats.minute_1 > multiprocessing.cpu_count(): # System is stressed, slow down status updates. - await asyncio.sleep((sleep_time * load_stats.minute_1) / multiprocessing.cpu_count()) + await asyncio.sleep((sleep_time * float(load_stats.minute_1)) / multiprocessing.cpu_count()) if count: # Running in testing, or __main__. diff --git a/wrolpi/tags.py b/wrolpi/tags.py index e83f9cfe..0fe944fb 100644 --- a/wrolpi/tags.py +++ b/wrolpi/tags.py @@ -1,5 +1,6 @@ import contextlib import pathlib +from dataclasses import dataclass, field from datetime import datetime from typing import List, Dict, Tuple, Optional @@ -13,6 +14,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc from wrolpi.dates import TZDateTime from wrolpi.db import optional_session, get_db_curs, get_db_session from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileGroupAlreadyTagged, RefreshConflict, NoPrimaryFile +from wrolpi.events import Events from wrolpi.switches import register_switch_handler, ActivateSwitchMethod from wrolpi.vars import PYTEST @@ -218,10 +220,18 @@ class Tag(ModelHelper, Base): sync_tags_directory.activate_switch() +@dataclass +class TagsConfigValidator: + tag_files: list[list] = field(default_factory=list) + tag_zims: list[list] = field(default_factory=list) + tags: dict = field(default_factory=dict) + version: int = None + + class TagsConfig(ConfigFile): file_name = 'tags.yaml' width = 500 - + validator = TagsConfigValidator default_config = dict( tag_files=list(), tag_zims=list(), @@ -255,58 +265,64 @@ class TagsConfig(ConfigFile): def tags(self, value: dict): self.update({'tags': value}) - def dump_config(self, file: pathlib.Path = None): - media_directory = get_media_directory() + def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False): + try: + media_directory = get_media_directory() - with get_db_session() as session: - tags = dict() - tag_rows = session.query(Tag) - for tag in tag_rows: - tags[tag.name] = dict(color=tag.color) + with get_db_session() as session: + tags = dict() + tag_rows = session.query(Tag) + for tag in tag_rows: + tags[tag.name] = dict(color=tag.color) - from wrolpi.files.models import FileGroup - results = session.query(Tag, TagFile, FileGroup) \ - .filter(TagFile.tag_id == Tag.id, TagFile.file_group_id == FileGroup.id) \ - .order_by(FileGroup.primary_path) + from wrolpi.files.models import FileGroup + results = session.query(Tag, TagFile, FileGroup) \ + .filter(TagFile.tag_id == Tag.id, TagFile.file_group_id == FileGroup.id) \ + .order_by(FileGroup.primary_path) - tag_files = [] - for tag, tag_file, file_group in results: - value = [ - tag.name, - str(file_group.primary_path.relative_to(media_directory)), - # Fallback to current time if not set. - tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(), - ] - tag_files.append(value) - logger.debug(f'Got {len(tag_files)} tag files for config.') + tag_files = [] + for tag, tag_file, file_group in results: + value = [ + tag.name, + str(file_group.primary_path.relative_to(media_directory)), + # Fallback to current time if not set. + tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(), + ] + tag_files.append(value) + logger.debug(f'Got {len(tag_files)} tag files for config.') - from modules.zim.models import Zim, TagZimEntry - results = session.query(Tag, Zim, TagZimEntry) \ - .filter(Tag.id == TagZimEntry.tag_id, Zim.id == TagZimEntry.zim_id) \ - .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) + from modules.zim.models import Zim, TagZimEntry + results = session.query(Tag, Zim, TagZimEntry) \ + .filter(Tag.id == TagZimEntry.tag_id, Zim.id == TagZimEntry.zim_id) \ + .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry) - tag_zims = [] - for tag, zim, tag_zim_entry in results: - zim: Zim - tag_zim_entry: TagZimEntry - value = [ - tag.name, - str(get_relative_to_media_directory(zim.path)), - tag_zim_entry.zim_entry, - # Fallback to current time if not set. - tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(), - ] - tag_zims.append(value) - logger.debug(f'Got {len(tag_zims)} tag zims for config.') + tag_zims = [] + for tag, zim, tag_zim_entry in results: + zim: Zim + tag_zim_entry: TagZimEntry + value = [ + tag.name, + str(get_relative_to_media_directory(zim.path)), + tag_zim_entry.zim_entry, + # Fallback to current time if not set. + tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(), + ] + tag_zims.append(value) + logger.debug(f'Got {len(tag_zims)} tag zims for config.') - # Write to the config. - self.update({ - 'tag_files': tag_files, - 'tag_zims': tag_zims, - 'tags': tags, - }) + # Write to the config. + self.update({ + 'tag_files': tag_files, + 'tag_zims': tag_zims, + 'tags': tags, + }) + except Exception as e: + message = f'Failed to save {self.get_relative_file()} config' + logger.error(message, exc_info=e) + if send_events: + Events.send_config_save_failed(message) - def import_config(self, file: pathlib.Path = None): + def import_config(self, file: pathlib.Path = None, send_events=False): from modules.zim import lib as zim_lib from modules.zim.models import Zim, TagZimEntry from wrolpi.files.lib import glob_shared_stem @@ -317,14 +333,12 @@ class TagsConfig(ConfigFile): return config = get_tags_config() - if not (path := config.get_file()).is_file(): - logger.warning(f'Refusing to import tags config because it does not exist: {path}') - return - logger.info('Importing tags config') super().import_config(file) try: + need_commit = False + with get_db_session() as session: if config.tags: # Tags have been saved to config, import them @@ -341,11 +355,10 @@ class TagsConfig(ConfigFile): if new_tags: session.add_all(new_tags) + need_commit = True media_directory = get_media_directory() - need_commit = False - # Get all Tags again because new ones may exist. tags_by_name: Dict[str, Tag] = {i.name: i for i in session.query(Tag)} @@ -451,7 +464,12 @@ class TagsConfig(ConfigFile): self.successful_import = True logger.info('Importing tags config complete') except Exception as e: - logger.error(f'Failed to import tags config', exc_info=e) + self.successful_import = False + message = f'Failed to import {self.file_name}' + logger.error(message, exc_info=e) + if send_events: + from wrolpi.events import Events + Events.send_config_import_failed(message) raise diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 074f9206..ca56bb56 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -6,6 +6,7 @@ import os import pathlib import re import tempfile +import time from datetime import date, datetime from decimal import Decimal from http import HTTPStatus @@ -19,6 +20,7 @@ import pytz import wrolpi.vars from wrolpi import common from wrolpi.common import cum_timer, TIMERS, print_timer, limit_concurrent, run_after, get_wrolpi_config +from wrolpi.errors import InvalidConfig from wrolpi.switches import await_switches from wrolpi.test.common import build_test_directories, skip_circleci @@ -462,6 +464,12 @@ async def test_cum_timer(): print_timer() +def test_timer(): + """`cum_timer` can be used to profile code.""" + with common.timer(name='test_timer'): + time.sleep(0.1) + + @pytest.mark.asyncio async def test_limit_concurrent_async(): """`limit_concurrent` can throw an error when the limit is reached.""" @@ -926,12 +934,14 @@ async def test_config_lifecycle(async_client, test_wrolpi_config): # Config is not imported, default values are used. assert config.successful_import is False assert config.wrol_mode is False + assert config.is_valid() is False assert config.version == 0 # Bad config cannot be imported. test_wrolpi_config.parent.mkdir(exist_ok=True) test_wrolpi_config.write_text('bad config\ndata') - with pytest.raises(RuntimeError) as e: + assert config.is_valid() is False + with pytest.raises(InvalidConfig) as e: config.import_config() assert config.successful_import is False assert config.version == 0 @@ -945,29 +955,35 @@ async def test_config_lifecycle(async_client, test_wrolpi_config): # Incomplete config can still be imported. test_wrolpi_config.write_text('wrol_mode: true') + assert config.is_valid() is True config.import_config() assert config.successful_import is True assert config.wrol_mode is True + assert config.is_valid() is True assert config.version == 0 # Items from the default config are written to the file. config.dump_config() assert 'version: ' in test_wrolpi_config.read_text() assert config.wrol_mode is True + assert config.is_valid() is True assert config.version == 1 # Other data is ignored. test_wrolpi_config.write_text('wrol_mode: false\nother_data: false') + assert config.is_valid() is True config.import_config() assert 'other_data' not in config._config, 'Extra data in config should be ignored' + assert config.is_valid() is True assert config.wrol_mode is False # Changing value changes both file and config data. config.wrol_mode = True assert config._config['wrol_mode'] is True - await await_switches() + await await_switches() # `wrol_mode = True` calls background switch assert 'wrol_mode: true' in test_wrolpi_config.read_text() assert 'other_data' not in test_wrolpi_config.read_text(), 'Other data should be removed on save.' + assert config.is_valid() is True assert config.version == 2 # Cannot overwrite newer config. @@ -981,3 +997,20 @@ async def test_config_lifecycle(async_client, test_wrolpi_config): # Can import, then config can be saved. config.import_config() config.dump_config() + assert config.is_valid() is True + + +@pytest.mark.asyncio +async def test_config_valid(async_client, test_wrolpi_config): + # Config file does not yet exist. + config = get_wrolpi_config() + assert config.is_valid() is False + + config.dump_config() + assert config.is_valid() is True + + test_wrolpi_config.write_text('bad config') + assert config.is_valid() is False + + test_wrolpi_config.write_text('wrol_mode: true') + assert config.is_valid() is True diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index c93fe31a..37097b73 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -12,6 +12,7 @@ from wrolpi.common import get_wrolpi_config from wrolpi.downloader import Download, get_download_manager_config from wrolpi.errors import ValidationError, SearchEmpty from wrolpi.switches import await_switches +from wrolpi.tags import Tag from wrolpi.test.common import skip_circleci, assert_dict_contains @@ -759,3 +760,51 @@ async def test_search_other_estimates(async_client, test_session, channel_factor request, response = await async_client.post('/api/search_other_estimates', json=body) assert response.status_code == HTTPStatus.OK assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.' + + +@pytest.mark.asyncio +async def test_import_configs(async_client, test_session, test_tags_config, tag_factory, test_wrolpi_config, + test_channels_config): + """Tags can be imported or saved on demand.""" + # Create a tag, save it to config, delete it from the DB. + one = await tag_factory() + test_session.commit() + await await_switches() + test_session.delete(one) + test_session.commit() + assert 'one' in test_tags_config.read_text() + assert test_session.query(Tag).count() == 0, 'Tag should have been deleted' + + # Can get list of configs, and if they are valid. + request, response = await async_client.get('/api/configs') + assert response.status_code == HTTPStatus.OK + # WROLPiConfig was not saved. + assert 'wrolpi.yaml' in response.json['configs'] + assert response.json['configs']['wrolpi.yaml']['valid'] is False + assert response.json['configs']['wrolpi.yaml']['successful_import'] is False + # Tags was saved, and is valid. + assert 'tags.yaml' in response.json['configs'] + assert response.json['configs']['tags.yaml']['valid'] is True + assert response.json['configs']['wrolpi.yaml']['successful_import'] is False + + # Can import the config. + body = dict(file_name='tags.yaml') + request, response = await async_client.post('/api/configs/import', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + # Deleted Tag was re-created. + assert [i for i, in test_session.query(Tag.name)] == ['one', ] + + # Can trigger a save of a config + assert not test_wrolpi_config.exists() + body = dict(file_name='wrolpi.yaml') + request, response = await async_client.post('/api/configs/save', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + assert test_wrolpi_config.is_file() + assert 'wrol_mode:' in test_wrolpi_config.read_text() + + # Config is now valid because it exists. + request, response = await async_client.get('/api/configs') + assert response.status_code == HTTPStatus.OK + assert 'wrolpi.yaml' in response.json['configs'] + assert response.json['configs']['wrolpi.yaml']['valid'] is True + assert response.json['configs']['tags.yaml']['successful_import'] is True