From 376850ceeb1591ddcc101ddfc9611dafd54d8609 Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 3 Dec 2024 22:30:12 -0700 Subject: [PATCH] Moving config api endpoints to their own file.. --- app/src/api.js | 28 ++++++++----- app/src/components/Videos.js | 4 +- app/src/hooks/customHooks.js | 4 +- modules/videos/test/test_lib.py | 6 +-- wrolpi/config_api.py | 72 +++++++++++++++++++++++++++++++++ wrolpi/config_schema.py | 22 ++++++++++ wrolpi/root_api.py | 66 ++---------------------------- wrolpi/schema.py | 10 ----- wrolpi/test/test_config_api.py | 68 +++++++++++++++++++++++++++++++ wrolpi/test/test_root_api.py | 49 ---------------------- 10 files changed, 190 insertions(+), 139 deletions(-) create mode 100644 wrolpi/config_api.py create mode 100644 wrolpi/config_schema.py create mode 100644 wrolpi/test/test_config_api.py diff --git a/app/src/api.js b/app/src/api.js index 6f7c5fcc..26a1e833 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -327,7 +327,7 @@ export async function getDownloads() { } export async function getConfigs() { - const response = await apiGet(`${API_URI}/configs`); + const response = await apiGet(`${API_URI}/config`); if (response.ok) { const data = await response.json(); return { @@ -347,7 +347,7 @@ export async function getConfigs() { export async function postImportConfig(fileName) { const body = {file_name: fileName}; - const response = await apiPost(`${API_URI}/configs/import`, body); + const response = await apiPost(`${API_URI}/config/import`, body); if (!response.ok) { const message = await getErrorMessage(response, 'Could not import config.'); toast({ @@ -360,14 +360,14 @@ export async function postImportConfig(fileName) { return response } -export async function postSaveConfig(fileName) { +export async function postDumpConfig(fileName) { const body = {file_name: fileName, overwrite: true}; - const response = await apiPost(`${API_URI}/configs/save`, body); + const response = await apiPost(`${API_URI}/config/dump`, body); if (!response.ok) { - const message = await getErrorMessage(response, 'Could not save config.'); + const message = await getErrorMessage(response, 'Could not dump config.'); toast({ type: 'error', - title: 'Saving config Failed', + title: 'Dumping config failed', description: message, time: 5000, }); @@ -375,19 +375,27 @@ export async function postSaveConfig(fileName) { return response } -export async function fetchVideoDownloaderConfig() { - const response = await apiGet(`${API_URI}/config/videos_downloader.yaml`); +export async function getConfig(fileName) { + const response = await apiGet(`${API_URI}/config?file_name=${fileName}`); return (await response.json())['config']; } -export async function saveVideoDownloaderConfig(config) { +export async function fetchVideoDownloaderConfig() { + return await getConfig('videos_downloader.yaml'); +} + +export async function updateConfig(fileName, config) { const body = {config}; - const response = await apiPost(`${API_URI}/config/videos_downloader.yaml`, body); + const response = await apiPost(`${API_URI}/config?file_name=${fileName}`, body); if (!response.ok) { throw Error('Failed to save config'); } } +export async function updateVideoDownloaderConfig(config) { + return await updateConfig('videos_downloader.yaml', config); +} + export async function validateRegex(regex) { const response = await apiPost(`${API_URI}/valid_regex`, {regex}); try { diff --git a/app/src/components/Videos.js b/app/src/components/Videos.js index 47b033c3..4ca82941 100644 --- a/app/src/components/Videos.js +++ b/app/src/components/Videos.js @@ -43,7 +43,7 @@ import {FileRowTagIcon, FilesView} from "./Files"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Icon from "semantic-ui-react/dist/commonjs/elements/Icon"; import {Button, Card, Header, Loader, Placeholder, Popup, Segment, Statistic, StatisticGroup} from "./Theme"; -import {deleteVideos, fetchVideoDownloaderConfig, postVideoFileFormat, saveVideoDownloaderConfig} from "../api"; +import {deleteVideos, fetchVideoDownloaderConfig, postVideoFileFormat, updateVideoDownloaderConfig} from "../api"; import {Media, QueryContext, ThemeContext} from "../contexts/contexts"; import _ from "lodash"; import {defaultFileOrder, defaultSearchOrder} from "./Vars"; @@ -264,7 +264,7 @@ function VideosSettings() { } const submitter = async () => { - return await saveVideoDownloaderConfig(form.formData); + return await updateVideoDownloaderConfig(form.formData); } const form = useForm({ diff --git a/app/src/hooks/customHooks.js b/app/src/hooks/customHooks.js index 04da57a2..72216976 100644 --- a/app/src/hooks/customHooks.js +++ b/app/src/hooks/customHooks.js @@ -19,7 +19,7 @@ import { getVideo, getVideosStatistics, postImportConfig, - postSaveConfig, + postDumpConfig, saveSettings, searchArchives, searchChannels, @@ -722,7 +722,7 @@ export const useConfigs = () => { const saveConfig = async (fileName) => { try { - await postSaveConfig(fileName); + await postDumpConfig(fileName); } catch (e) { console.error(e); } diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py index 12e4fb4f..cf2dfa93 100644 --- a/modules/videos/test/test_lib.py +++ b/modules/videos/test/test_lib.py @@ -275,14 +275,14 @@ async def test_videos_downloader_config_api(async_client, test_directory, test_v config = lib.get_videos_downloader_config() assert config.video_resolutions == ['1080p', '720p', '480p', 'maximum'] - request, response = await async_client.get('/api/config/videos_downloader.yaml') + request, response = await async_client.get('/api/config?file_name=videos_downloader.yaml') assert response.status == HTTPStatus.OK config = response.json['config'] # Change the `video_resolutions` config['video_resolutions'] = ['720p', 'maximum'] body = dict(config=config) - request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body) + request, response = await async_client.post('/api/config?file_name=videos_downloader.yaml', json=body) assert response.status == HTTPStatus.NO_CONTENT # Config file was actually changed. config = lib.get_videos_downloader_config() @@ -297,6 +297,6 @@ async def test_videos_downloader_config_api(async_client, test_directory, test_v config = lib.get_videos_downloader_config().dict() config['yt_dlp_options']['file_name_format'] = 'invalid format' body = dict(config=config) - request, response = await async_client.post('/api/config/videos_downloader.yaml', json=body) + request, response = await async_client.post('/api/config?file_name=videos_downloader.yaml', json=body) assert response.status == HTTPStatus.BAD_REQUEST assert 'Invalid config' in response.json.get('error') diff --git a/wrolpi/config_api.py b/wrolpi/config_api.py new file mode 100644 index 00000000..0a71c92a --- /dev/null +++ b/wrolpi/config_api.py @@ -0,0 +1,72 @@ +from sanic import response, Blueprint +from sanic.request import Request +from sanic_ext.extensions.openapi import openapi + +from wrolpi import config_schema +from wrolpi.api_utils import json_response +from wrolpi.common import get_all_configs, get_config_by_file_name +from wrolpi.common import logger +from wrolpi.errors import InvalidConfig + +config_bp = Blueprint('config', '/api/config') + + +@config_bp.get('/') +@openapi.description('Get the status (imported, valid, etc.) of all configs.') +@openapi.parameter('file_name', str, location='query') +def get_config(request: Request): + if file_name := request.args.get('file_name'): + config = get_config_by_file_name(file_name) + return json_response(dict(config=config)) + else: + configs = get_all_configs() + configs = {k: v.config_status() for k, v in configs.items()} + return json_response(dict(configs=configs)) + + +@config_bp.post('/') +@openapi.parameter('file_name', str, location='query') +@openapi.definition( + description='Update the contents of a specific config.', + body=config_schema.ConfigUpdateRequest, + validate=True, +) +def post_config_update(request: Request, body: config_schema.ConfigUpdateRequest): + file_name, = request.args['file_name'] + config = get_config_by_file_name(file_name) + config.update(body.config) + return response.empty() + + +@config_bp.post('/import') +@openapi.definition( + description='Import the contents of a specific config.', + body=config_schema.ConfigsRequest, + validate=True, +) +def post_config_import(_: Request, body: config_schema.ConfigsRequest): + config = get_config_by_file_name(body.file_name) + try: + config.import_config(send_events=True) + except Exception as e: + logger.error(f'Failed to import config: {body.file_name}', exc_info=e) + raise InvalidConfig(f'Failed to import config {body.file_name}') + + return response.empty() + + +@config_bp.post('/dump') +@openapi.definition( + description="Dump the contents of a specific config to it's file.", + body=config_schema.ConfigsRequest, + validate=True, +) +def post_config_dump(_: Request, body: config_schema.ConfigsRequest): + config = get_config_by_file_name(body.file_name) + try: + config.dump_config(send_events=True, overwrite=body.overwrite) + except Exception as e: + logger.error(f'Failed to dump config: {body.file_name}', exc_info=e) + raise InvalidConfig(f'Failed to dump config {body.file_name}') + + return response.empty() diff --git a/wrolpi/config_schema.py b/wrolpi/config_schema.py new file mode 100644 index 00000000..f713c8df --- /dev/null +++ b/wrolpi/config_schema.py @@ -0,0 +1,22 @@ +from dataclasses import dataclass + + +@dataclass +class ConfigGetRequest: + file_name: str = None + + +@dataclass +class ConfigPostRequest: + file_name: str + + +@dataclass +class ConfigsRequest: + file_name: str + overwrite: bool = False + + +@dataclass +class ConfigUpdateRequest: + config: dict diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 04cb6235..22361a18 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -23,8 +23,8 @@ from wrolpi.admin import HotspotStatus from wrolpi.api_utils import json_response, api_app from wrolpi.common import logger, get_wrolpi_config, wrol_mode_enabled, get_media_directory, \ wrol_mode_check, native_only, disable_wrol_mode, enable_wrol_mode, get_global_statistics, url_strip_host, \ - set_global_log_level, get_relative_to_media_directory, search_other_estimates, get_all_configs, \ - get_config_by_file_name + set_global_log_level, get_relative_to_media_directory, search_other_estimates +from wrolpi.config_api import config_bp from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.downloader import download_manager @@ -47,6 +47,7 @@ api_bp = Blueprint('RootAPI', url_prefix='/api') # Blueprints order here defines what order they are displayed in OpenAPI Docs. api_app.blueprint(api_bp) api_app.blueprint(archive_bp) +api_app.blueprint(config_bp) api_app.blueprint(files_bp) api_app.blueprint(inventory_bp) api_app.blueprint(map_bp) @@ -670,64 +671,3 @@ async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstima others = await search_other_estimates(body.tag_names) ret = dict(others=others) return json_response(ret) - - -@api_bp.get('/configs') -@openapi.description('Get the status (imported, valid, etc.) of all configs.') -def get_configs(_: Request): - configs = get_all_configs() - configs = {k: v.config_status() for k, v in configs.items()} - return json_response(dict(configs=configs)) - - -@api_bp.get('/config/') -@openapi.description('Get the contents of a specific config.') -def get_config(_: Request, name: str): - config = get_all_configs()[name] - return json_response(dict(config=config)) - - -@api_bp.post('/config/') -@openapi.definition( - description='Save the contents of a specific config.', - body=schema.ConfigSaveRequest, - validate=True, -) -def save_config(_: Request, name: str, body: schema.ConfigSaveRequest): - config = get_all_configs()[name] - config.update(body.config) - return response.empty() - - -@api_bp.post('/configs/import') -@openapi.definition( - description='Import the contents of a specific config.', - body=schema.ConfigsImportRequest, - validate=True, -) -def post_configs_import(_: Request, body: schema.ConfigsImportRequest): - config = get_config_by_file_name(body.file_name) - try: - config.import_config(send_events=True) - except Exception as e: - logger.error(f'Failed to import config: {body.file_name}', exc_info=e) - raise InvalidConfig(f'Failed to import config {body.file_name}') - - return response.empty() - - -@api_bp.post('/configs/save') -@openapi.definition( - description='Save the contents of a specific config.', - body=schema.ConfigsImportRequest, - validate=True, -) -def post_configs_save(_: Request, body: schema.ConfigsImportRequest): - config = get_config_by_file_name(body.file_name) - try: - config.dump_config(send_events=True, overwrite=body.overwrite) - except Exception as e: - logger.error(f'Failed to dump config: {body.file_name}', exc_info=e) - raise InvalidConfig(f'Failed to dump config {body.file_name}') - - return response.empty() diff --git a/wrolpi/schema.py b/wrolpi/schema.py index f56282e8..3b97fff1 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -371,13 +371,3 @@ class SearchFileEstimateRequest: class SearchOtherEstimateRequest: tag_names: List[str] = field(default_factory=list) - -@dataclass -class ConfigsImportRequest: - file_name: str - overwrite: bool = False - - -@dataclass -class ConfigSaveRequest: - config: dict diff --git a/wrolpi/test/test_config_api.py b/wrolpi/test/test_config_api.py new file mode 100644 index 00000000..940d99cb --- /dev/null +++ b/wrolpi/test/test_config_api.py @@ -0,0 +1,68 @@ +from http import HTTPStatus + +import pytest + +from wrolpi.common import get_wrolpi_config +from wrolpi.switches import await_switches +from wrolpi.tags import Tag + + +@pytest.mark.asyncio +async def test_configs_crud(async_client, test_session, test_tags_config, tag_factory, test_wrolpi_config, + test_channels_config): + """Configs can be imported or saved on demand.""" + # Create a tag, save it to config, delete it from the DB. + one = await tag_factory() + test_session.commit() + await await_switches() + test_session.delete(one) + test_session.commit() + assert 'one' in test_tags_config.read_text() + assert test_session.query(Tag).count() == 0, 'Tag should have been deleted' + + # Can get list of configs, and if they are valid. + request, response = await async_client.get('/api/config') + assert response.status_code == HTTPStatus.OK + # WROLPiConfig was not saved. + assert 'wrolpi.yaml' in response.json['configs'] + assert response.json['configs']['wrolpi.yaml']['valid'] is False + assert response.json['configs']['wrolpi.yaml']['successful_import'] is False + # Tags was saved, and is valid. + assert 'tags.yaml' in response.json['configs'] + assert response.json['configs']['tags.yaml']['valid'] is True + assert response.json['configs']['wrolpi.yaml']['successful_import'] is False + + # Can import the config. + body = dict(file_name='tags.yaml') + request, response = await async_client.post('/api/config/import', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + # Deleted Tag was re-created. + assert [i for i, in test_session.query(Tag.name)] == ['one', ] + + # Can trigger a save of a config + assert not test_wrolpi_config.exists() + body = dict(file_name='wrolpi.yaml') + request, response = await async_client.post('/api/config/dump', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + assert test_wrolpi_config.is_file() + assert 'wrol_mode:' in test_wrolpi_config.read_text() + + # Config is now valid because it exists. + request, response = await async_client.get('/api/config') + assert response.status_code == HTTPStatus.OK + assert 'wrolpi.yaml' in response.json['configs'] + assert response.json['configs']['wrolpi.yaml']['valid'] is True + assert response.json['configs']['tags.yaml']['successful_import'] is True + + # Get and change a config. + request, response = await async_client.get('/api/config?file_name=wrolpi.yaml') + assert response.status_code == HTTPStatus.OK + wrolpi_config = response.json['config'] + assert wrolpi_config['archive_destination'] == 'archive/%(domain)s' + # Change config value. + wrolpi_config['archive_destination'] = 'some new value' + body = dict(config=wrolpi_config) + # Update the config file. + request, response = await async_client.post('/api/config?file_name=wrolpi.yaml', json=body) + assert response.status_code == HTTPStatus.NO_CONTENT + assert 'some new value' in get_wrolpi_config().get_file().read_text() diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index fb1ef349..bf8b0540 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -12,7 +12,6 @@ from wrolpi.common import get_wrolpi_config from wrolpi.downloader import Download, get_download_manager_config from wrolpi.errors import ValidationError, SearchEmpty from wrolpi.switches import await_switches -from wrolpi.tags import Tag from wrolpi.test.common import skip_circleci, assert_dict_contains @@ -793,51 +792,3 @@ async def test_search_other_estimates(async_client, test_session, channel_factor request, response = await async_client.post('/api/search_other_estimates', json=body) assert response.status_code == HTTPStatus.OK assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.' - - -@pytest.mark.asyncio -async def test_import_configs(async_client, test_session, test_tags_config, tag_factory, test_wrolpi_config, - test_channels_config): - """Tags can be imported or saved on demand.""" - # Create a tag, save it to config, delete it from the DB. - one = await tag_factory() - test_session.commit() - await await_switches() - test_session.delete(one) - test_session.commit() - assert 'one' in test_tags_config.read_text() - assert test_session.query(Tag).count() == 0, 'Tag should have been deleted' - - # Can get list of configs, and if they are valid. - request, response = await async_client.get('/api/configs') - assert response.status_code == HTTPStatus.OK - # WROLPiConfig was not saved. - assert 'wrolpi.yaml' in response.json['configs'] - assert response.json['configs']['wrolpi.yaml']['valid'] is False - assert response.json['configs']['wrolpi.yaml']['successful_import'] is False - # Tags was saved, and is valid. - assert 'tags.yaml' in response.json['configs'] - assert response.json['configs']['tags.yaml']['valid'] is True - assert response.json['configs']['wrolpi.yaml']['successful_import'] is False - - # Can import the config. - body = dict(file_name='tags.yaml') - request, response = await async_client.post('/api/configs/import', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT - # Deleted Tag was re-created. - assert [i for i, in test_session.query(Tag.name)] == ['one', ] - - # Can trigger a save of a config - assert not test_wrolpi_config.exists() - body = dict(file_name='wrolpi.yaml') - request, response = await async_client.post('/api/configs/save', json=body) - assert response.status_code == HTTPStatus.NO_CONTENT - assert test_wrolpi_config.is_file() - assert 'wrol_mode:' in test_wrolpi_config.read_text() - - # Config is now valid because it exists. - request, response = await async_client.get('/api/configs') - assert response.status_code == HTTPStatus.OK - assert 'wrolpi.yaml' in response.json['configs'] - assert response.json['configs']['wrolpi.yaml']['valid'] is True - assert response.json['configs']['tags.yaml']['successful_import'] is True