diff --git a/app/src/Events.js b/app/src/Events.js
index bfb1ec7a..0aa66f35 100644
--- a/app/src/Events.js
+++ b/app/src/Events.js
@@ -83,6 +83,14 @@ function handleEvents(events) {
eventToast('Move Failed', message, 'error');
}
+ if (event === 'config_import_failed') {
+ eventToast('Config Import Failed', message, 'error');
+ }
+
+ if (event === 'config_save_failed') {
+ eventToast('Config Save Failed', message, 'error');
+ }
+
if (subject) {
newestEvents[subject] = dt;
}
diff --git a/app/src/api.js b/app/src/api.js
index c75fa449..a214f8d9 100644
--- a/app/src/api.js
+++ b/app/src/api.js
@@ -313,6 +313,55 @@ export async function getDownloads() {
return response
}
+export async function getConfigs() {
+ const response = await apiGet(`${API_URI}/configs`);
+ if (response.ok) {
+ const data = await response.json();
+ return {
+ configs: data.configs,
+ }
+ } else {
+ const message = getErrorMessage(response, 'Could not get configs.');
+ toast({
+ type: 'error',
+ title: 'Getting Configs Failed',
+ description: message,
+ time: 5000,
+ });
+ }
+ return response
+}
+
+export async function postImportConfig(fileName) {
+ const body = {file_name: fileName};
+ const response = await apiPost(`${API_URI}/configs/import`, body);
+ if (!response.ok) {
+ const message = getErrorMessage(response, 'Could not import config.');
+ toast({
+ type: 'error',
+ title: 'Importing config Failed',
+ description: message,
+ time: 5000,
+ });
+ }
+ return response
+}
+
+export async function postSaveConfig(fileName) {
+ const body = {file_name: fileName, overwrite: true};
+ const response = await apiPost(`${API_URI}/configs/save`, body);
+ if (!response.ok) {
+ const message = getErrorMessage(response, 'Could not save config.');
+ toast({
+ type: 'error',
+ title: 'Saving config Failed',
+ description: message,
+ time: 5000,
+ });
+ }
+ return response
+}
+
export async function validateRegex(regex) {
const response = await apiPost(`${API_URI}/valid_regex`, {regex});
try {
diff --git a/app/src/components/Files.js b/app/src/components/Files.js
index dcb9676c..d9a472aa 100644
--- a/app/src/components/Files.js
+++ b/app/src/components/Files.js
@@ -25,7 +25,6 @@ import {
HandPointMessage,
humanFileSize,
isoDatetimeToAgoPopup,
- isoDatetimeToString,
mimetypeColor,
PageContainer,
Paginator,
diff --git a/app/src/components/Placeholder.js b/app/src/components/Placeholder.js
index debd9fa2..5da13fce 100644
--- a/app/src/components/Placeholder.js
+++ b/app/src/components/Placeholder.js
@@ -53,3 +53,9 @@ export function TagPlaceholder() {
}
+
+export function TableRowPlaceholder() {
+ return
+
+
+}
\ No newline at end of file
diff --git a/app/src/components/admin/AdminRoute.js b/app/src/components/admin/AdminRoute.js
index f0e6f50d..b9e5b9b4 100644
--- a/app/src/components/admin/AdminRoute.js
+++ b/app/src/components/admin/AdminRoute.js
@@ -8,6 +8,7 @@ import {Header, Loader, Segment} from "../Theme";
import {DownloadsPage} from "./Downloads";
import {SettingsPage} from "./Settings";
import {StatusPage} from "./Status";
+import {ConfigsPage} from "./Configs";
class WROLMode extends React.Component {
@@ -71,6 +72,7 @@ export default function AdminRoute() {
{text: 'Downloads', to: '/admin', key: 'admin', end: true},
{text: 'Settings', to: '/admin/settings', key: 'settings'},
{text: 'Status', to: '/admin/status', key: 'status'},
+ {text: 'Configs', to: '/admin/configs', key: 'configs'},
{text: 'WROL Mode', to: '/admin/wrol', key: 'wrol'},
];
@@ -80,6 +82,7 @@ export default function AdminRoute() {
}/>
}/>
}/>
+ }/>
}/>
diff --git a/app/src/components/admin/Configs.js b/app/src/components/admin/Configs.js
new file mode 100644
index 00000000..5bb47dc4
--- /dev/null
+++ b/app/src/components/admin/Configs.js
@@ -0,0 +1,148 @@
+import {useConfigs} from "../../hooks/customHooks";
+import {APIButton, PageContainer} from "../Common";
+import {Header, Icon, Popup, Table} from "../Theme";
+import React from "react";
+import {
+ Grid,
+ GridColumn,
+ GridRow,
+ TableBody,
+ TableCell,
+ TableHeader,
+ TableHeaderCell,
+ TableRow
+} from "semantic-ui-react";
+import {TableRowPlaceholder} from "../Placeholder";
+import {useNavigate} from "react-router-dom";
+
+function WarningIcon({ok}) {
+ return
+}
+
+function ConfigTableRow({config, importConfig, saveConfig, fetchConfigs}) {
+ const {file_name, rel_path, valid, successful_import} = config;
+ const navigate = useNavigate();
+
+ const localImportConfig = async () => {
+ await importConfig(file_name);
+ await fetchConfigs();
+ }
+
+ const localSaveConfig = async () => {
+ await saveConfig(file_name);
+ await fetchConfigs();
+ }
+
+ const importButton = ;
+
+ let saveButton;
+ if (successful_import) {
+ saveButton = ;
+ } else {
+ // Warn the user they may overwrite because import was not successful.
+ saveButton = ;
+ }
+
+ return
+ navigate(`?preview=${rel_path}`)}>{file_name}
+
+
+ {importButton}
+ {saveButton}
+
+}
+
+function ConfigsTable({configs, loading, importConfig, saveConfig, fetchConfigs}) {
+ let body =
+
+
+
+ if (loading) {
+ // Configs are being fetched again.
+ } else if (configs === undefined) {
+ body =
+ Failed to fetch configs
+ ;
+ } else if (configs && Object.keys(configs).length === 0) {
+ body =
+ No configs exist. Try tagging some files.
+ ;
+ } else if (configs) {
+ body = Object.entries(configs).map(([key, value]) =>
+
+ );
+ }
+
+ return
+
+
+ File Name
+ Imported
+ Valid
+ Import
+ Save
+
+
+
+
+ {body}
+
+
+}
+
+export function ConfigsPage() {
+ const {configs, loading, importConfig, saveConfig, fetchConfigs} = useConfigs();
+
+ React.useEffect(() => {
+ fetchConfigs();
+ }, []);
+
+ const refreshButton = ;
+ const refreshPopupButton = ;
+
+ return
+
+
+
+
+
+ {refreshPopupButton}
+
+
+
+
+
+}
\ No newline at end of file
diff --git a/app/src/components/admin/Settings.js b/app/src/components/admin/Settings.js
index 351de605..676eb6e4 100644
--- a/app/src/components/admin/Settings.js
+++ b/app/src/components/admin/Settings.js
@@ -297,8 +297,17 @@ export class SettingsPage extends React.Component {
}
render() {
+ const controlSegment =
+
+
+
+ ;
+
if (this.state.ready === false) {
- return
+ return <>
+
+ {controlSegment}
+
} else if (this.state.ready === undefined) {
return Unable to fetch settings
}
@@ -539,11 +548,7 @@ export class SettingsPage extends React.Component {
-
-
-
-
-
+ {controlSegment}
;
}
}
diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js
index 0f4c86ff..4172c855 100644
--- a/app/src/hooks/customHooks.js
+++ b/app/src/hooks/customHooks.js
@@ -7,6 +7,7 @@ import {
getArchive,
getChannel,
getChannels,
+ getConfigs,
getDirectories,
getDownloads,
getFiles,
@@ -17,7 +18,10 @@ import {
getStatus,
getVideo,
getVideosStatistics,
- searchArchives, searchChannels,
+ postImportConfig,
+ postSaveConfig,
+ searchArchives,
+ searchChannels,
searchDirectories,
searchVideos,
searchZim,
@@ -679,6 +683,44 @@ export const useDownloads = () => {
return {onceDownloads, recurringDownloads, pendingOnceDownloads, fetchDownloads}
}
+export const useConfigs = () => {
+ const [configs, setConfigs] = useState(null);
+ const [loading, setLoading] = useState(false);
+
+ const fetchConfigs = async () => {
+ setLoading(true);
+ console.log('fetching configs...');
+ try {
+ const data = await getConfigs();
+ setConfigs(data['configs']);
+ console.debug('fetching configs successful');
+ } catch (e) {
+ console.error(e);
+ setConfigs(undefined);
+ } finally {
+ setLoading(false);
+ }
+ }
+
+ const importConfig = async (fileName) => {
+ try {
+ await postImportConfig(fileName);
+ } catch (e) {
+ console.error(e);
+ }
+ }
+
+ const saveConfig = async (fileName) => {
+ try {
+ await postSaveConfig(fileName);
+ } catch (e) {
+ console.error(e);
+ }
+ }
+
+ return {configs, loading, fetchConfigs, importConfig, saveConfig}
+}
+
export const useThrottle = () => {
const [on, setOn] = useState(null);
const {settings, fetchSettings} = React.useContext(SettingsContext);
diff --git a/modules/inventory/common.py b/modules/inventory/common.py
index f8749149..8601096d 100644
--- a/modules/inventory/common.py
+++ b/modules/inventory/common.py
@@ -1,4 +1,5 @@
import pathlib
+from dataclasses import field, dataclass
from decimal import Decimal
from functools import partial
from pathlib import Path
@@ -8,6 +9,7 @@ from pint import Quantity
from wrolpi.common import logger, Base, ConfigFile
from wrolpi.db import get_db_session
+from wrolpi.events import Events
from .errors import NoInventories
from .inventory import get_items, get_inventories, get_unit_registry
from .models import Inventory, Item
@@ -116,12 +118,19 @@ def cleanup_quantity(quantity: Quantity) -> Quantity:
return Decimal(num) * unit
+@dataclass
+class InventoriesConfigValidator:
+ version: int = None
+ inventories: list[str] = field(default_factory=list)
+
+
class InventoriesConfig(ConfigFile):
file_name = 'inventories.yaml'
default_config = dict(
inventories=[],
version=0,
)
+ validator = InventoriesConfigValidator
@property
def inventories(self) -> dict:
@@ -131,7 +140,7 @@ class InventoriesConfig(ConfigFile):
def inventories(self, value: dict):
self.update({'inventories': value})
- def dump_config(self, file: pathlib.Path = None):
+ def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
inventories = []
for inventory in get_inventories():
inventories.append(inventory.dict())
@@ -140,46 +149,51 @@ class InventoriesConfig(ConfigFile):
raise NoInventories('No Inventories are in the database!')
self._config['inventories'] = inventories
- super().dump_config(file)
+ super().dump_config(file, send_events, overwrite)
- def import_config(self, file: pathlib.Path = None):
+ def import_config(self, file: pathlib.Path = None, send_events=False):
super().import_config(file)
try:
with get_db_session(commit=True) as session:
- db_inventories = get_inventories(session)
- config_inventory_names = {i['name'] for i in self.inventories}
- keep_inventories_names = []
- for inventory in db_inventories:
- if inventory.name in config_inventory_names:
- keep_inventories_names.append(inventory.name)
- # TODO delete missing items, add new items.
- else:
- # Inventory was deleted from the config file.
- inventory.delete()
- continue
+ db_inventories = session.query(Inventory).all()
+ db_inventories_by_name = {i.name: i for i in db_inventories}
+ config_inventories = [i['name'] for i in self.inventories]
+ for inventory_dict in self.inventories:
+ if inventory_dict['name'] not in db_inventories_by_name:
+ inventory = Inventory(
+ name=inventory_dict['name'],
+ created_at=inventory_dict['created_at'],
+ deleted_at=inventory_dict['deleted_at'],
+ )
+ session.add(inventory)
+ # Get the Inventory from the DB so we can use its ID.
+ inventory.flush()
+ db_inventories_by_name[inventory.name] = inventory
- new_inventories = [i for i in self.inventories if i['name'] not in keep_inventories_names]
- for inventory_dict in new_inventories:
- # New Inventory in config, copy it into the DB.
- inventory = Inventory(
- name=inventory_dict['name'],
- created_at=inventory_dict['created_at'],
- deleted_at=inventory_dict['deleted_at'],
- )
- session.add(inventory)
- # Get the Inventory from the DB so we can use its ID.
- inventory.flush()
+ # Delete any inventories no longer in the config.
+ for name, inventory in db_inventories_by_name.items():
+ if name not in config_inventories:
+ session.delete(inventory)
+ for inventory_dict in self.inventories:
+ # Items are complex and do not have useful primary keys. Delete all items, create new items.
+ session.query(Item).filter(Item.inventory_id == inventory_dict['id']).delete()
+ inventory = db_inventories_by_name[inventory_dict['name']]
for item in inventory_dict['items']:
del item['inventory_id']
- item = Item(inventory_id=inventory.id, **item)
- item.inventory_id = inventory.id
+ del item['id']
+ item = Item(inventory=inventory, inventory_id=inventory.id, **item)
session.add(item)
+ item.flush()
self.successful_import = True
except Exception as e:
- logger.error(f'Failed to import {self.file_name}', exc_info=e)
+ self.successful_import = False
+ message = f'Failed to import {self.file_name}'
+ logger.error(message, exc_info=e)
+ if send_events:
+ Events.send_config_import_failed(message)
raise
diff --git a/modules/inventory/models.py b/modules/inventory/models.py
index 64d4c365..c8289204 100644
--- a/modules/inventory/models.py
+++ b/modules/inventory/models.py
@@ -1,5 +1,5 @@
from sqlalchemy import Column, Integer, String, Date, ForeignKey, DECIMAL
-from sqlalchemy.orm import relationship
+from sqlalchemy.orm import relationship, Session
from sqlalchemy.orm.collections import InstrumentedList
from wrolpi.common import Base, ModelHelper
@@ -56,6 +56,10 @@ class Inventory(Base, ModelHelper):
"""
self.deleted_at = now()
+ @staticmethod
+ def find_by_name(session: Session, name: str) -> 'Inventory':
+ return session.query(Inventory).filter_by(name=name).one()
+
class InventoriesVersion(Base, ModelHelper):
__tablename__ = 'inventories_version'
diff --git a/modules/inventory/test/test_inventory.py b/modules/inventory/test/test_inventory.py
index fd45ba90..5e5a8bc3 100644
--- a/modules/inventory/test/test_inventory.py
+++ b/modules/inventory/test/test_inventory.py
@@ -303,3 +303,16 @@ async def test_inventories_config(async_client, test_session, test_directory, in
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
assert test_items == db_items
+
+ # An extra Inventory with items should be deleted from the DB.
+ inventory = Inventory(name='delete me')
+ test_session.add(inventory)
+ inventory.flush()
+ test_session.add(Item(inventory_id=inventory.id))
+ test_session.commit()
+ assert test_session.query(Inventory).count() == 2
+ assert {i for i, in test_session.query(Inventory.name)} == {'Food Storage', 'delete me'}
+
+ import_inventories_config()
+ assert test_session.query(Inventory).count() == 1
+ assert test_session.query(Inventory).one().name == 'Food Storage'
diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 68720c48..95001b8c 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -3,6 +3,7 @@ import dataclasses
import html
import pathlib
import re
+from dataclasses import dataclass
from datetime import datetime
from typing import Generator, Type
from typing import Tuple, Optional
@@ -20,6 +21,7 @@ from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import Download
from wrolpi.errors import UnknownDirectory
+from wrolpi.events import Events
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
@@ -207,6 +209,20 @@ def convert_or_generate_poster(video: Video) -> Tuple[Optional[pathlib.Path], Op
return None, None
+@dataclass
+class ChannelDictValidator:
+ name: str
+ directory: str
+ download_frequency: int
+ url: str = None
+
+
+@dataclass
+class ChannelsConfigValidator:
+ version: int = None
+ channels: list[ChannelDictValidator] = dataclasses.field(default_factory=list)
+
+
class ChannelsConfig(ConfigFile):
file_name = 'channels.yaml'
default_config = dict(
@@ -218,6 +234,7 @@ class ChannelsConfig(ConfigFile):
)],
version=0,
)
+ validator = ChannelsConfigValidator
@property
def channels(self) -> dict:
@@ -227,7 +244,7 @@ class ChannelsConfig(ConfigFile):
def channels(self, value: dict):
self.update({'channels': value})
- def import_config(self, file: pathlib.Path = None):
+ def import_config(self, file: pathlib.Path = None, send_events=False):
from modules.videos.channel.lib import get_channel
super().import_config()
try:
@@ -296,10 +313,12 @@ class ChannelsConfig(ConfigFile):
self.successful_import = True
channel_import_logger.info('Importing channels config complete')
except Exception as e:
- channel_import_logger.warning('Failed to load channels config!', exc_info=e)
- if PYTEST:
- # Do not interrupt startup, only raise during testing.
- raise
+ self.successful_import = False
+ message = f'Failed to import {self.get_relative_file()} config!'
+ if send_events:
+ channel_import_logger.warning(message, exc_info=e)
+ Events.send_config_import_failed(message)
+ raise
CHANNELS_CONFIG: ChannelsConfig = ChannelsConfig()
@@ -328,6 +347,21 @@ def set_test_channels_config():
TEST_CHANNELS_CONFIG = None
+@dataclass
+class VideoDownloaderConfigValidator:
+ continue_dl: bool
+ dateafter: str
+ file_name_format: str
+ nooverwrites: bool
+ quiet: bool
+ writeautomaticsub: bool
+ writeinfojson: bool
+ writesubtitles: bool
+ writethumbnail: bool
+ youtube_include_dash_manifest: bool
+ version: int = None
+
+
class VideoDownloaderConfig(ConfigFile):
file_name = 'videos_downloader.yaml'
default_config = dict(
@@ -343,6 +377,7 @@ class VideoDownloaderConfig(ConfigFile):
writethumbnail=True,
youtube_include_dash_manifest=False,
)
+ validator = VideoDownloaderConfigValidator
@property
def continue_dl(self) -> bool:
diff --git a/wrolpi/api_utils.py b/wrolpi/api_utils.py
index a316e76c..023b0c97 100644
--- a/wrolpi/api_utils.py
+++ b/wrolpi/api_utils.py
@@ -111,6 +111,8 @@ def json_error_handler(request: Request, exception: Exception):
return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR)
+api_app.error_handler.add(Exception, json_error_handler)
+
PERPETUAL_WORKERS = list()
diff --git a/wrolpi/common.py b/wrolpi/common.py
index e7eb8a3c..572f9361 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -14,10 +14,9 @@ import socket
import string
import sys
import tempfile
-import traceback
from asyncio import Task
from copy import deepcopy
-from dataclasses import dataclass
+from dataclasses import dataclass, asdict, field, fields
from datetime import datetime, date
from decimal import Decimal
from functools import wraps
@@ -26,7 +25,7 @@ from itertools import islice, filterfalse, tee
from multiprocessing.managers import DictProxy
from pathlib import Path
from types import GeneratorType
-from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set, Coroutine
+from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any, Set
from urllib.parse import urlparse, urlunsplit
import aiohttp
@@ -41,7 +40,7 @@ from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from wrolpi.dates import now, from_timestamp, seconds_to_timestamp
-from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError
+from wrolpi.errors import WROLModeEnabled, NativeOnly, UnrecoverableDownloadError, LogLevelError, InvalidConfig
from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS, LOG_LEVEL
LOGGING_CONFIG = {
@@ -192,6 +191,7 @@ __all__ = [
'format_html_string',
'format_json_file',
'get_absolute_media_path',
+ 'get_all_configs',
'get_download_info',
'get_files_and_directories',
'get_global_statistics',
@@ -346,10 +346,12 @@ class ConfigFile:
Configs cannot be saved until they are imported without error.
"""
- file_name: str = None
- default_config: dict = None
- width: int = None
+ background_dump: callable = None
background_save: callable = None
+ default_config: dict = None
+ file_name: str = None
+ validator: dataclass = None
+ width: int = None
def __init__(self):
self.width = self.width or 90
@@ -377,6 +379,15 @@ class ConfigFile:
def __repr__(self):
return f'<{self.__class__.__name__} file={self.get_file()}>'
+ def __json__(self) -> dict:
+ d = dict(
+ file_name=self.file_name,
+ rel_path=get_relative_to_media_directory(self.get_file()),
+ successful_import=self.successful_import,
+ valid=self.is_valid(),
+ )
+ return d
+
def initialize(self, multiprocessing_dict: Optional[DictProxy] = None):
"""Initializes this config dict using the default config and the config file."""
# Use the provided multiprocessing.Manager().dict(), or dict() for testing.
@@ -386,8 +397,10 @@ class ConfigFile:
file = self.get_file()
if file.is_file():
- config_data = {k: v for k, v in self.read_config_file(file).items() if k in self.default_config}
- self._config.update(config_data)
+ if not self.is_valid(file):
+ raise InvalidConfig(f'Config file is invalid: {str(self.get_relative_file())}')
+ config_data = self.read_config_file(file)
+ self._config.update(asdict(self.validator(**config_data)))
return self
@property
@@ -405,7 +418,7 @@ class ConfigFile:
with file.open('rt') as fh:
config_data = yaml.load(fh, Loader=yaml.Loader)
if not isinstance(config_data, dict):
- raise RuntimeError(f'Config file is invalid: {file}')
+ raise InvalidConfig(f'Config file is invalid: {file}')
return config_data
def _get_backup_filename(self):
@@ -417,9 +430,13 @@ class ConfigFile:
path = path.with_name(name)
return path
- def save(self, file: pathlib.Path = None):
+ def save(self, file: pathlib.Path = None, send_events: bool = False, overwrite: bool = False):
"""
Write this config to its file.
+
+ @param file: The destination of the config file (defaults to `self.get_file()`).
+ @param send_events: Send failure Events to UI.
+ @param overwrite: Will overwrite the config file even if it was not imported successfully.
"""
from wrolpi.api_utils import api_app
@@ -429,13 +446,12 @@ class ConfigFile:
rel_path = get_relative_to_media_directory(file)
- if file.exists():
+ if file.exists() and overwrite is False:
if not self.successful_import:
raise RuntimeError(f'Refusing to save config because it was never successfully imported! {rel_path}')
version = self.read_config_file(file).get('version')
if version and version > self.version:
- raise RuntimeError(f'Refusing to overwrite newer config ({rel_path}): '
- f'{version} > {self.version}')
+ raise RuntimeError(f'Refusing to overwrite newer config ({rel_path}): {version} > {self.version}')
# Don't overwrite a real config while testing.
if PYTEST and not str(file).startswith('/tmp'):
@@ -469,7 +485,11 @@ class ConfigFile:
logger_.info(f'Saved config: {rel_path}')
except Exception as e:
# Configs are vital, raise a big error when this fails.
- logger_.critical(f'Failed to save config: {rel_path}', exc_info=e)
+ message = f'Failed to save config: {rel_path}'
+ logger_.critical(message, exc_info=e)
+ if send_events:
+ from wrolpi.events import Events
+ Events.send_config_save_failed(message)
raise e
def write_config_data(self, config: dict, config_file: pathlib.Path):
@@ -488,11 +508,19 @@ class ConfigFile:
return CONFIG_DIR / self.file_name
- def import_config(self, file: pathlib.Path = None):
+ def get_relative_file(self):
+ return get_relative_to_media_directory(self.get_file())
+
+ def import_config(self, file: pathlib.Path = None, send_events=False):
"""Read config file data, apply it to the in-memory config and database."""
file = file or self.get_file()
file_str = str(get_relative_to_media_directory(file))
+ # Caller will set to successful if it works.
+ self.successful_import = False
if file.is_file():
+ if not self.is_valid(file):
+ raise InvalidConfig(f'Config is invalid: {file_str}')
+
data = self.read_config_file(file)
new_data, extra_data = partition(lambda i: i[0] in self.default_config, data.items())
new_data, extra_data = dict(new_data), dict(extra_data)
@@ -504,10 +532,10 @@ class ConfigFile:
else:
logger_.error(f'Failed to import {file_str} because it does not exist.')
- def dump_config(self, file: pathlib.Path = None):
+ def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
"""Dump database data, copy it to this the in-memory config, then write it to the config file."""
# Copy the data as-is by default. Other classes will overwrite this and dump the database before saving.
- self.save(file)
+ self.save(file, send_events, overwrite)
def update(self, config: dict):
"""Update any values of this config. Save the config to its file."""
@@ -524,6 +552,28 @@ class ConfigFile:
return deepcopy(self._config)
+ def is_valid(self, file: pathlib.Path = None) -> bool:
+ if not self.validator:
+ raise NotImplementedError(f'Cannot validate {self.file_name} without validator!')
+
+ file = file or self.get_file()
+ if not file.is_file():
+ return False
+
+ allowed_fields = {i.name for i in fields(self.validator)}
+ try:
+ # Remove keys no longer in the config.
+ config_items = self.read_config_file(file).items()
+ extra_items = {k: v for k, v in config_items if k not in allowed_fields}
+ if extra_items:
+ logger.error(f'Invalid config items ({get_relative_to_media_directory(file)}): {extra_items}')
+ config = {k: v for k, v in config_items if k in allowed_fields}
+ self.validator(**config)
+ return True
+ except Exception as e:
+ logger.debug(f'Failed to validate config: {file}', exc_info=e)
+ return False
+
# `version` is used to prevent overwriting of newer configs. Cannot not be modified directly.
@property
@@ -531,6 +581,63 @@ class ConfigFile:
return self._config['version']
+@dataclass
+class WROLPiConfigValidator:
+ archive_destination: str = None
+ download_on_startup: bool = None
+ download_timeout: int = None
+ hotspot_device: str = None
+ hotspot_on_startup: bool = None
+ hotspot_password: str = None
+ hotspot_ssid: str = None
+ ignore_outdated_zims: bool = None
+ map_destination: str = None
+ nav_color: str = None
+ throttle_on_startup: bool = None
+ version: int = None
+ videos_destination: str = None
+ wrol_mode: bool = None
+ zims_destination: str = None
+ ignored_directories: list[str] = field(default_factory=list)
+
+
+def get_all_configs() -> Dict[str, ConfigFile]:
+ all_configs = dict()
+
+ if wrolpi_config := get_wrolpi_config():
+ all_configs[wrolpi_config.file_name] = wrolpi_config
+
+ from wrolpi.tags import get_tags_config
+ if tags_config := get_tags_config():
+ all_configs[tags_config.file_name] = tags_config
+
+ from modules.videos.lib import get_channels_config
+ if channels_config := get_channels_config():
+ all_configs[channels_config.file_name] = channels_config
+
+ from modules.inventory.common import get_inventories_config
+ if inventories_config := get_inventories_config():
+ all_configs[inventories_config.file_name] = inventories_config
+
+ from modules.videos.lib import get_downloader_config
+ if videos_downloader_config := get_downloader_config():
+ all_configs[videos_downloader_config.file_name] = videos_downloader_config
+
+ from wrolpi.downloader import get_download_manager_config
+ if download_manager_config := get_download_manager_config():
+ all_configs[download_manager_config.file_name] = download_manager_config
+
+ return all_configs
+
+
+def get_config_by_file_name(file_name: str) -> ConfigFile:
+ configs = get_all_configs()
+ if file_name not in configs:
+ raise InvalidConfig(f'No config with {file_name} exists')
+
+ return configs[file_name]
+
+
class WROLPiConfig(ConfigFile):
file_name = 'wrolpi.yaml'
default_config = dict(
@@ -551,6 +658,7 @@ class WROLPiConfig(ConfigFile):
wrol_mode=False,
zims_destination='zims',
)
+ validator = WROLPiConfigValidator
def get_file(self) -> Path:
"""WROLPiConfig must be discovered so that other config files can be found."""
@@ -573,13 +681,18 @@ class WROLPiConfig(ConfigFile):
# Not testing, and can't find file deep in media directory. Use the default.
return default_config_path
- def import_config(self, file: pathlib.Path = None):
+ def import_config(self, file: pathlib.Path = None, send_events=False):
try:
super().import_config(file)
# WROLPiConfig does not sync to database.
self.successful_import = True
except Exception as e:
- logger.error(f'Failed to import {self.file_name}', exc_info=e)
+ self.successful_import = False
+ message = f'Failed to import {self.file_name}'
+ logger.error(message, exc_info=e)
+ if send_events:
+ from wrolpi.events import Events
+ Events.send_config_import_failed(message)
raise
@property
@@ -820,7 +933,6 @@ def run_after(after: callable, *args, **kwargs) -> callable:
async def after(*a, **kw):
if logger.isEnabledFor(logging.DEBUG):
- traceback.print_stack()
logger.debug(f'run_after synchronous_after called for {after}')
return synchronous_after(*a, **kw)
@@ -833,7 +945,6 @@ def run_after(after: callable, *args, **kwargs) -> callable:
async def wrapped(*a, **kw):
results = await func(*a, **kw)
if logger.isEnabledFor(logging.DEBUG):
- traceback.print_stack()
logger.debug(f'run_after async called for {func}')
coro = after(*args, **kwargs)
asyncio.ensure_future(coro)
@@ -843,7 +954,6 @@ def run_after(after: callable, *args, **kwargs) -> callable:
def wrapped(*a, **kw):
results = func(*a, **kw)
if logger.isEnabledFor(logging.DEBUG):
- traceback.print_stack()
logger.debug(f'run_after sync called for {func}')
coro = after(*args, **kwargs)
asyncio.ensure_future(coro)
@@ -1483,7 +1593,7 @@ def chunks_by_stem(it: List[Union[pathlib.Path, str, int]], size: int) -> Genera
@contextlib.contextmanager
-def timer(name, level: str = 'debug', logger__: logging.Logger = logger_):
+def timer(name, level: str = 'debug', logger__: logging.Logger = None):
"""Prints out the time elapsed during the call of some block.
Example:
@@ -1491,6 +1601,7 @@ def timer(name, level: str = 'debug', logger__: logging.Logger = logger_):
time.sleep(10)
"""
+ logger__ = logger__ or logger_
before = datetime.now()
log_method = getattr(logger__, level)
try:
@@ -1500,16 +1611,6 @@ def timer(name, level: str = 'debug', logger__: logging.Logger = logger_):
log_method(f'{name} elapsed {elapsed} seconds')
-def async_timer(coro: Coroutine, name: str = 'async timer', level: str = 'debug') -> callable:
- """Returns a new coroutine which prints out time elapsed when calling the coroutine."""
-
- async def _():
- with timer(name, level):
- return await coro
-
- return _()
-
-
@contextlib.contextmanager
def slow_logger(max_seconds: int, message: str, logger__=logger_, level=logging.WARNING):
"""Only logs when the context duration exceeds the `max_seconds`."""
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index 82d2608b..49f0cad0 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -31,6 +31,7 @@ from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, C
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
+from wrolpi.events import Events
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod, await_switches
from wrolpi.vars import PYTEST, SIMULTANEOUS_DOWNLOAD_DOMAINS
@@ -1143,6 +1144,25 @@ async def signal_download_download(download_id: int, download_url: str):
download_manager._delete_processing_domain(download_domain)
+@dataclass
+class DownloadDictConfig:
+ downloader: str
+ last_successful_download: str
+ next_download: str
+ url: str
+ frequency: int = None
+ settings: dict = None
+ status: str = None
+ sub_downloader: str = None
+
+
+@dataclass
+class DownloadManagerConfigValidator:
+ version: int = None
+ downloads: list[DownloadDictConfig] = field(default_factory=list)
+ skip_urls: list[str] = field(default_factory=list)
+
+
class DownloadManagerConfig(ConfigFile):
file_name = 'download_manager.yaml'
default_config = dict(
@@ -1150,6 +1170,7 @@ class DownloadManagerConfig(ConfigFile):
skip_urls=[],
version=0,
)
+ validator = DownloadManagerConfigValidator
@property
def skip_urls(self) -> List[str]:
@@ -1167,27 +1188,33 @@ class DownloadManagerConfig(ConfigFile):
def downloads(self, value: List[dict]):
self.update({'downloads': value})
- def dump_config(self, file: pathlib.Path = None):
- with get_db_session() as session:
- downloads: Iterable[Download] = session.query(Download).order_by(Download.url)
- new_downloads = []
- for download in downloads:
- if download.last_successful_download and not download.frequency:
- # This once-download has completed, do not save it.
- continue
- new_downloads.append(dict(
- downloader=download.downloader,
- frequency=download.frequency,
- last_successful_download=download.last_successful_download,
- next_download=download.next_download,
- settings=download.settings,
- status=download.status,
- sub_downloader=download.sub_downloader,
- url=download.url,
- ))
- get_download_manager_config().downloads = new_downloads
+ def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
+ try:
+ with get_db_session() as session:
+ downloads: Iterable[Download] = session.query(Download).order_by(Download.url)
+ new_downloads = []
+ for download in downloads:
+ if download.last_successful_download and not download.frequency:
+ # This once-download has completed, do not save it.
+ continue
+ new_downloads.append(dict(
+ downloader=download.downloader,
+ frequency=download.frequency,
+ last_successful_download=download.last_successful_download,
+ next_download=download.next_download,
+ settings=download.settings,
+ status=download.status,
+ sub_downloader=download.sub_downloader,
+ url=download.url,
+ ))
+ get_download_manager_config().downloads = new_downloads
+ except Exception as e:
+ message = f'Failed to save {self.get_relative_file()} config'
+ logger.error(message, exc_info=e)
+ if send_events:
+ Events.send_config_save_failed(message)
- def import_config(self, file: pathlib.Path = None):
+ def import_config(self, file: pathlib.Path = None, send_events=False):
super().import_config(file)
with get_db_session(commit=True) as session:
from modules.zim.lib import zim_download_url_to_name
@@ -1231,8 +1258,13 @@ class DownloadManagerConfig(ConfigFile):
logger.info(f'Adding new download {url}')
session.commit()
+ self.successful_import = True
except Exception as e:
- logger.error('Failed to import downloads', exc_info=e)
+ self.successful_import = False
+ message = f'Failed to import {self.file_name}'
+ logger.error(message, exc_info=e)
+ if send_events:
+ Events.send_config_import_failed(message)
raise
try:
@@ -1254,8 +1286,13 @@ class DownloadManagerConfig(ConfigFile):
if need_commit:
session.commit()
+ self.successful_import = True
except Exception as e:
- logger.error('Failed to restore ZimSubscriptions', exc_info=e)
+ self.successful_import = False
+ message = 'Failed to restore ZimSubscriptions'
+ logger.error(message, exc_info=e)
+ if send_events:
+ Events.send_config_import_failed(message)
raise
self.successful_import = True
diff --git a/wrolpi/events.py b/wrolpi/events.py
index e4603271..ce61378a 100644
--- a/wrolpi/events.py
+++ b/wrolpi/events.py
@@ -84,6 +84,14 @@ class Events:
def send_file_move_failed(message: str = None):
send_event('file_move_failed', message, subject='refresh')
+ @staticmethod
+ def send_config_import_failed(message: str = None):
+ send_event('config_import_failed', message, subject='configs')
+
+ @staticmethod
+ def send_config_save_failed(message: str = None):
+ send_event('config_save_failed', message, subject='configs')
+
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'
diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py
index b383a2a8..afae85e6 100644
--- a/wrolpi/root_api.py
+++ b/wrolpi/root_api.py
@@ -20,10 +20,11 @@ from modules.zim.api import zim_bp
from wrolpi import admin, flags, schema, dates
from wrolpi import tags
from wrolpi.admin import HotspotStatus
-from wrolpi.api_utils import json_response, json_error_handler, api_app
+from wrolpi.api_utils import json_response, api_app
from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \
wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \
- set_global_log_level, get_relative_to_media_directory, search_other_estimates
+ set_global_log_level, get_relative_to_media_directory, search_other_estimates, get_all_configs, \
+ get_config_by_file_name
from wrolpi.dates import now
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, HotspotError, InvalidDownload, \
@@ -605,4 +606,33 @@ async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstima
return json_response(ret)
-api_app.error_handler.add(Exception, json_error_handler)
+@api_bp.get('/configs')
+def get_configs(_: Request):
+ configs = get_all_configs()
+ return json_response(dict(configs=configs))
+
+
+@api_bp.post('/configs/import')
+@validate(json=schema.ConfigsImportRequest)
+def post_configs_import(_: Request, body: schema.ConfigsImportRequest):
+ config = get_config_by_file_name(body.file_name)
+ try:
+ config.import_config(send_events=True)
+ except Exception as e:
+ logger.error(f'Failed to import config: {body.file_name}', exc_info=e)
+ raise InvalidConfig(f'Failed to import config {body.file_name}')
+
+ return response.empty()
+
+
+@api_bp.post('/configs/save')
+@validate(json=schema.ConfigsImportRequest)
+def post_configs_save(_: Request, body: schema.ConfigsImportRequest):
+ config = get_config_by_file_name(body.file_name)
+ try:
+ config.dump_config(send_events=True, overwrite=body.overwrite)
+ except Exception as e:
+ logger.error(f'Failed to dump config: {body.file_name}', exc_info=e)
+ raise InvalidConfig(f'Failed to dump config {body.file_name}')
+
+ return response.empty()
diff --git a/wrolpi/schema.py b/wrolpi/schema.py
index bde4862d..4acb898e 100644
--- a/wrolpi/schema.py
+++ b/wrolpi/schema.py
@@ -213,3 +213,9 @@ class SearchFileEstimateRequest:
@dataclass
class SearchOtherEstimateRequest:
tag_names: List[str] = field(default_factory=lambda: list())
+
+
+@dataclass
+class ConfigsImportRequest:
+ file_name: str
+ overwrite: bool = False
diff --git a/wrolpi/status.py b/wrolpi/status.py
index f50b5008..99e8e12d 100755
--- a/wrolpi/status.py
+++ b/wrolpi/status.py
@@ -527,7 +527,7 @@ async def status_worker(count: int = None, sleep_time: int = 5):
# Always sleep so this does not become a busy loop with errors.
if load_stats and load_stats.minute_1 and load_stats.minute_1 > multiprocessing.cpu_count():
# System is stressed, slow down status updates.
- await asyncio.sleep((sleep_time * load_stats.minute_1) / multiprocessing.cpu_count())
+ await asyncio.sleep((sleep_time * float(load_stats.minute_1)) / multiprocessing.cpu_count())
if count:
# Running in testing, or __main__.
diff --git a/wrolpi/tags.py b/wrolpi/tags.py
index e83f9cfe..0fe944fb 100644
--- a/wrolpi/tags.py
+++ b/wrolpi/tags.py
@@ -1,5 +1,6 @@
import contextlib
import pathlib
+from dataclasses import dataclass, field
from datetime import datetime
from typing import List, Dict, Tuple, Optional
@@ -13,6 +14,7 @@ from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_direc
from wrolpi.dates import TZDateTime
from wrolpi.db import optional_session, get_db_curs, get_db_session
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileGroupAlreadyTagged, RefreshConflict, NoPrimaryFile
+from wrolpi.events import Events
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import PYTEST
@@ -218,10 +220,18 @@ class Tag(ModelHelper, Base):
sync_tags_directory.activate_switch()
+@dataclass
+class TagsConfigValidator:
+ tag_files: list[list] = field(default_factory=list)
+ tag_zims: list[list] = field(default_factory=list)
+ tags: dict = field(default_factory=dict)
+ version: int = None
+
+
class TagsConfig(ConfigFile):
file_name = 'tags.yaml'
width = 500
-
+ validator = TagsConfigValidator
default_config = dict(
tag_files=list(),
tag_zims=list(),
@@ -255,58 +265,64 @@ class TagsConfig(ConfigFile):
def tags(self, value: dict):
self.update({'tags': value})
- def dump_config(self, file: pathlib.Path = None):
- media_directory = get_media_directory()
+ def dump_config(self, file: pathlib.Path = None, send_events=False, overwrite=False):
+ try:
+ media_directory = get_media_directory()
- with get_db_session() as session:
- tags = dict()
- tag_rows = session.query(Tag)
- for tag in tag_rows:
- tags[tag.name] = dict(color=tag.color)
+ with get_db_session() as session:
+ tags = dict()
+ tag_rows = session.query(Tag)
+ for tag in tag_rows:
+ tags[tag.name] = dict(color=tag.color)
- from wrolpi.files.models import FileGroup
- results = session.query(Tag, TagFile, FileGroup) \
- .filter(TagFile.tag_id == Tag.id, TagFile.file_group_id == FileGroup.id) \
- .order_by(FileGroup.primary_path)
+ from wrolpi.files.models import FileGroup
+ results = session.query(Tag, TagFile, FileGroup) \
+ .filter(TagFile.tag_id == Tag.id, TagFile.file_group_id == FileGroup.id) \
+ .order_by(FileGroup.primary_path)
- tag_files = []
- for tag, tag_file, file_group in results:
- value = [
- tag.name,
- str(file_group.primary_path.relative_to(media_directory)),
- # Fallback to current time if not set.
- tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(),
- ]
- tag_files.append(value)
- logger.debug(f'Got {len(tag_files)} tag files for config.')
+ tag_files = []
+ for tag, tag_file, file_group in results:
+ value = [
+ tag.name,
+ str(file_group.primary_path.relative_to(media_directory)),
+ # Fallback to current time if not set.
+ tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(),
+ ]
+ tag_files.append(value)
+ logger.debug(f'Got {len(tag_files)} tag files for config.')
- from modules.zim.models import Zim, TagZimEntry
- results = session.query(Tag, Zim, TagZimEntry) \
- .filter(Tag.id == TagZimEntry.tag_id, Zim.id == TagZimEntry.zim_id) \
- .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
+ from modules.zim.models import Zim, TagZimEntry
+ results = session.query(Tag, Zim, TagZimEntry) \
+ .filter(Tag.id == TagZimEntry.tag_id, Zim.id == TagZimEntry.zim_id) \
+ .order_by(TagZimEntry.zim_id, TagZimEntry.zim_entry)
- tag_zims = []
- for tag, zim, tag_zim_entry in results:
- zim: Zim
- tag_zim_entry: TagZimEntry
- value = [
- tag.name,
- str(get_relative_to_media_directory(zim.path)),
- tag_zim_entry.zim_entry,
- # Fallback to current time if not set.
- tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(),
- ]
- tag_zims.append(value)
- logger.debug(f'Got {len(tag_zims)} tag zims for config.')
+ tag_zims = []
+ for tag, zim, tag_zim_entry in results:
+ zim: Zim
+ tag_zim_entry: TagZimEntry
+ value = [
+ tag.name,
+ str(get_relative_to_media_directory(zim.path)),
+ tag_zim_entry.zim_entry,
+ # Fallback to current time if not set.
+ tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(),
+ ]
+ tag_zims.append(value)
+ logger.debug(f'Got {len(tag_zims)} tag zims for config.')
- # Write to the config.
- self.update({
- 'tag_files': tag_files,
- 'tag_zims': tag_zims,
- 'tags': tags,
- })
+ # Write to the config.
+ self.update({
+ 'tag_files': tag_files,
+ 'tag_zims': tag_zims,
+ 'tags': tags,
+ })
+ except Exception as e:
+ message = f'Failed to save {self.get_relative_file()} config'
+ logger.error(message, exc_info=e)
+ if send_events:
+ Events.send_config_save_failed(message)
- def import_config(self, file: pathlib.Path = None):
+ def import_config(self, file: pathlib.Path = None, send_events=False):
from modules.zim import lib as zim_lib
from modules.zim.models import Zim, TagZimEntry
from wrolpi.files.lib import glob_shared_stem
@@ -317,14 +333,12 @@ class TagsConfig(ConfigFile):
return
config = get_tags_config()
- if not (path := config.get_file()).is_file():
- logger.warning(f'Refusing to import tags config because it does not exist: {path}')
- return
-
logger.info('Importing tags config')
super().import_config(file)
try:
+ need_commit = False
+
with get_db_session() as session:
if config.tags:
# Tags have been saved to config, import them
@@ -341,11 +355,10 @@ class TagsConfig(ConfigFile):
if new_tags:
session.add_all(new_tags)
+ need_commit = True
media_directory = get_media_directory()
- need_commit = False
-
# Get all Tags again because new ones may exist.
tags_by_name: Dict[str, Tag] = {i.name: i for i in session.query(Tag)}
@@ -451,7 +464,12 @@ class TagsConfig(ConfigFile):
self.successful_import = True
logger.info('Importing tags config complete')
except Exception as e:
- logger.error(f'Failed to import tags config', exc_info=e)
+ self.successful_import = False
+ message = f'Failed to import {self.file_name}'
+ logger.error(message, exc_info=e)
+ if send_events:
+ from wrolpi.events import Events
+ Events.send_config_import_failed(message)
raise
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index 074f9206..ca56bb56 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -6,6 +6,7 @@ import os
import pathlib
import re
import tempfile
+import time
from datetime import date, datetime
from decimal import Decimal
from http import HTTPStatus
@@ -19,6 +20,7 @@ import pytz
import wrolpi.vars
from wrolpi import common
from wrolpi.common import cum_timer, TIMERS, print_timer, limit_concurrent, run_after, get_wrolpi_config
+from wrolpi.errors import InvalidConfig
from wrolpi.switches import await_switches
from wrolpi.test.common import build_test_directories, skip_circleci
@@ -462,6 +464,12 @@ async def test_cum_timer():
print_timer()
+def test_timer():
+ """`cum_timer` can be used to profile code."""
+ with common.timer(name='test_timer'):
+ time.sleep(0.1)
+
+
@pytest.mark.asyncio
async def test_limit_concurrent_async():
"""`limit_concurrent` can throw an error when the limit is reached."""
@@ -926,12 +934,14 @@ async def test_config_lifecycle(async_client, test_wrolpi_config):
# Config is not imported, default values are used.
assert config.successful_import is False
assert config.wrol_mode is False
+ assert config.is_valid() is False
assert config.version == 0
# Bad config cannot be imported.
test_wrolpi_config.parent.mkdir(exist_ok=True)
test_wrolpi_config.write_text('bad config\ndata')
- with pytest.raises(RuntimeError) as e:
+ assert config.is_valid() is False
+ with pytest.raises(InvalidConfig) as e:
config.import_config()
assert config.successful_import is False
assert config.version == 0
@@ -945,29 +955,35 @@ async def test_config_lifecycle(async_client, test_wrolpi_config):
# Incomplete config can still be imported.
test_wrolpi_config.write_text('wrol_mode: true')
+ assert config.is_valid() is True
config.import_config()
assert config.successful_import is True
assert config.wrol_mode is True
+ assert config.is_valid() is True
assert config.version == 0
# Items from the default config are written to the file.
config.dump_config()
assert 'version: ' in test_wrolpi_config.read_text()
assert config.wrol_mode is True
+ assert config.is_valid() is True
assert config.version == 1
# Other data is ignored.
test_wrolpi_config.write_text('wrol_mode: false\nother_data: false')
+ assert config.is_valid() is True
config.import_config()
assert 'other_data' not in config._config, 'Extra data in config should be ignored'
+ assert config.is_valid() is True
assert config.wrol_mode is False
# Changing value changes both file and config data.
config.wrol_mode = True
assert config._config['wrol_mode'] is True
- await await_switches()
+ await await_switches() # `wrol_mode = True` calls background switch
assert 'wrol_mode: true' in test_wrolpi_config.read_text()
assert 'other_data' not in test_wrolpi_config.read_text(), 'Other data should be removed on save.'
+ assert config.is_valid() is True
assert config.version == 2
# Cannot overwrite newer config.
@@ -981,3 +997,20 @@ async def test_config_lifecycle(async_client, test_wrolpi_config):
# Can import, then config can be saved.
config.import_config()
config.dump_config()
+ assert config.is_valid() is True
+
+
+@pytest.mark.asyncio
+async def test_config_valid(async_client, test_wrolpi_config):
+ # Config file does not yet exist.
+ config = get_wrolpi_config()
+ assert config.is_valid() is False
+
+ config.dump_config()
+ assert config.is_valid() is True
+
+ test_wrolpi_config.write_text('bad config')
+ assert config.is_valid() is False
+
+ test_wrolpi_config.write_text('wrol_mode: true')
+ assert config.is_valid() is True
diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py
index c93fe31a..37097b73 100644
--- a/wrolpi/test/test_root_api.py
+++ b/wrolpi/test/test_root_api.py
@@ -12,6 +12,7 @@ from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, get_download_manager_config
from wrolpi.errors import ValidationError, SearchEmpty
from wrolpi.switches import await_switches
+from wrolpi.tags import Tag
from wrolpi.test.common import skip_circleci, assert_dict_contains
@@ -759,3 +760,51 @@ async def test_search_other_estimates(async_client, test_session, channel_factor
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'
+
+
+@pytest.mark.asyncio
+async def test_import_configs(async_client, test_session, test_tags_config, tag_factory, test_wrolpi_config,
+ test_channels_config):
+ """Tags can be imported or saved on demand."""
+ # Create a tag, save it to config, delete it from the DB.
+ one = await tag_factory()
+ test_session.commit()
+ await await_switches()
+ test_session.delete(one)
+ test_session.commit()
+ assert 'one' in test_tags_config.read_text()
+ assert test_session.query(Tag).count() == 0, 'Tag should have been deleted'
+
+ # Can get list of configs, and if they are valid.
+ request, response = await async_client.get('/api/configs')
+ assert response.status_code == HTTPStatus.OK
+ # WROLPiConfig was not saved.
+ assert 'wrolpi.yaml' in response.json['configs']
+ assert response.json['configs']['wrolpi.yaml']['valid'] is False
+ assert response.json['configs']['wrolpi.yaml']['successful_import'] is False
+ # Tags was saved, and is valid.
+ assert 'tags.yaml' in response.json['configs']
+ assert response.json['configs']['tags.yaml']['valid'] is True
+ assert response.json['configs']['wrolpi.yaml']['successful_import'] is False
+
+ # Can import the config.
+ body = dict(file_name='tags.yaml')
+ request, response = await async_client.post('/api/configs/import', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ # Deleted Tag was re-created.
+ assert [i for i, in test_session.query(Tag.name)] == ['one', ]
+
+ # Can trigger a save of a config
+ assert not test_wrolpi_config.exists()
+ body = dict(file_name='wrolpi.yaml')
+ request, response = await async_client.post('/api/configs/save', json=body)
+ assert response.status_code == HTTPStatus.NO_CONTENT
+ assert test_wrolpi_config.is_file()
+ assert 'wrol_mode:' in test_wrolpi_config.read_text()
+
+ # Config is now valid because it exists.
+ request, response = await async_client.get('/api/configs')
+ assert response.status_code == HTTPStatus.OK
+ assert 'wrolpi.yaml' in response.json['configs']
+ assert response.json['configs']['wrolpi.yaml']['valid'] is True
+ assert response.json['configs']['tags.yaml']['successful_import'] is True