diff --git a/.gitignore b/.gitignore index fb9b5fb5..2690176a 100644 --- a/.gitignore +++ b/.gitignore @@ -109,10 +109,8 @@ venv.bak/ test/videos test/archive -# local configs -config/local.yaml -config/local.yaml.bak -config/inventories.yaml +# testing configs +test/config/* app/package-lock.json modules/node_modules/ diff --git a/app/src/api.js b/app/src/api.js index b03346e4..c9f6182a 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -1,4 +1,12 @@ -import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, secondsToDateTime, VIDEOS_API} from "./components/Common"; +import { + API_URI, + ARCHIVES_API, + DEFAULT_LIMIT, + emptyToNull, + OTP_API, + secondsToDateTime, + VIDEOS_API +} from "./components/Common"; import {toast} from 'react-semantic-toasts'; function timeoutPromise(ms, promise) { @@ -87,12 +95,13 @@ let apiPatch = async (url, body) => { return await apiCall(url, 'PATCH', body) }; - export async function updateChannel(link, channel) { + channel = emptyToNull(channel); return await apiPut(`${VIDEOS_API}/channels/${link}`, channel); } export async function createChannel(channel) { + channel = emptyToNull(channel); return await apiPost(`${VIDEOS_API}/channels`, channel); } @@ -160,14 +169,13 @@ export async function getDirectories(search_str) { return []; } -export async function getConfig() { +export async function getSettings() { let response = await apiGet(`${API_URI}/settings`); - let data = await response.json(); - return data['config']; + return await response.json(); } -export async function saveConfig(config) { - return await apiPatch(`${API_URI}/settings`, config); +export async function saveSettings(settings) { + return await apiPatch(`${API_URI}/settings`, settings); } export async function getDownloads() { @@ -291,10 +299,12 @@ export async function getItems(inventoryId) { } export async function saveItem(inventoryId, item) { + item = emptyToNull(item); return await apiPost(`${API_URI}/inventory/${inventoryId}/item`, item); } export async function updateItem(itemId, item) { + item = emptyToNull(item); return await apiPut(`${API_URI}/inventory/item/${itemId}`, item); } @@ -438,25 +448,8 @@ export async function getVersion() { } export async function getHotspotStatus() { - let response = await apiGet(`${API_URI}/hotspot`); - if (response.status === 200) { - let {status} = await response.json(); - return status; - } else { - let code = (await response.json())['code']; - if (response.status === 400 && code === 34) { - // Must not be running on a RPi. Don't warn the user. - return null; - } else { - toast({ - type: 'error', - title: "Can't get Hotspot status", - description: 'Failed to get Hotspot status. Check client and server logs.', - time: 5000, - }); - } - } - return null; + let response = await getSettings(); + return response['hotspot_status']; } export async function setHotspot(on) { @@ -489,25 +482,8 @@ export async function setHotspot(on) { } export async function getThrottleStatus() { - let response = await apiGet(`${API_URI}/throttle`); - if (response.status === 200) { - let {status} = await response.json(); - return status; - } else { - let code = (await response.json())['code']; - if (response.status === 400 && code === 34) { - // Must not be running on a RPi. Don't warn the user. - return null; - } else { - toast({ - type: 'error', - title: "Can't get throttle status", - description: 'Failed to get CPU Throttle status. Check client and server logs.', - time: 5000, - }); - } - } - return null; + let response = await getSettings(); + return response['throttle_status']; } export async function setThrottle(on) { diff --git a/app/src/components/Admin.js b/app/src/components/Admin.js index 8efbcc9a..a400b701 100644 --- a/app/src/components/Admin.js +++ b/app/src/components/Admin.js @@ -11,7 +11,7 @@ import { Placeholder, Table } from "semantic-ui-react"; -import {getConfig, getDownloads, killDownload, postDownload, saveConfig} from "../api"; +import {getDownloads, getSettings, killDownload, postDownload, saveSettings} from "../api"; import TimezoneSelect from 'react-timezone-select'; import { DisableDownloadsToggle, @@ -33,42 +33,51 @@ class Settings extends React.Component { this.state = { disabled: false, ready: false, - validTimezone: true, - timezone: '', + + download_on_startup: null, + hotspot_on_startup: null, + hotspot_status: null, + throttle_on_startup: null, + throttle_status: null, + timezone: {timezone: null}, } - this.mediaDirectory = React.createRef(); this.handleSubmit = this.handleSubmit.bind(this); } async componentDidMount() { - let config = await getConfig(); + const settings = await getSettings(); this.setState({ ready: true, - disabled: config.wrol_mode, - timezone: config.timezone || '', + disabled: settings.wrol_mode, + download_on_startup: settings.download_on_startup, + hotspot_on_startup: settings.hotspot_on_startup, + hotspot_status: settings.hotspot_status, + throttle_on_startup: settings.throttle_on_startup, + throttle_status: settings.throttle_status, + timezone: {value: settings.timezone, label: settings.timezone}, }); - this.mediaDirectory.current.value = config.media_directory; } async handleSubmit(e) { e.preventDefault(); - let config = { - media_directory: this.mediaDirectory.current.value, + let settings = { + download_on_startup: this.state.download_on_startup, + hotspot_on_startup: this.state.hotspot_on_startup, + throttle_on_startup: this.state.throttle_on_startup, timezone: this.state.timezone.value, - }; - this.setState({validTimezone: true}); - let response = await saveConfig(config); - if (response.status !== 200) { - let json = await response.json(); - if (json.api_error === 'Invalid timezone') { - this.setState({validTimezone: false}); - } } + console.log(settings); + await saveSettings(settings); } handleTimezoneChange = async (timezone) => { - this.setState({timezone}); + this.setState({timezone: timezone}); + } + + handleInputChange = async (e, name, checked) => { + e.preventDefault(); + this.setState({[name]: checked}); } render() { @@ -76,6 +85,8 @@ class Settings extends React.Component { return } + let {disabled, download_on_startup, hotspot_on_startup, throttle_on_startup, timezone} = this.state; + return ( @@ -83,46 +94,52 @@ class Settings extends React.Component {
- -

- The global settings for your server. -

+ +
+ The settings for your WROLPi. +

- When saved, these will be written to your local.yaml file. Storing these settings in your - local configuration file (rather than the database) allows your WROLPi to be rebuilt and save your - settings. + Any changes will be written to config/wrolpi.yaml.

- +
+ this.handleInputChange(e, 'download_on_startup', d.checked)} + /> - - - - +
+ this.handleInputChange(e, 'hotspot_on_startup', d.checked)} + /> -

- This directory must already exist. - The directory in which your media will be stored. Typically, this will be some external - drive like /media/wrolpi. -

+
- + this.handleInputChange(e, 'throttle_on_startup', d.checked)} + /> - @@ -142,8 +159,8 @@ class WROLMode extends React.Component { } async componentDidMount() { - let config = await getConfig(); - this.setState({ready: true, WROLMode: config.wrol_mode}); + let settings = await getSettings(); + this.setState({ready: true, WROLMode: settings.wrol_mode}); } toggleWROLMode = async () => { @@ -151,7 +168,7 @@ class WROLMode extends React.Component { let config = { wrol_mode: !this.state.WROLMode, } - await saveConfig(config); + await saveSettings(config); this.setState({disabled: !this.state.WROLMode, WROLMode: !this.state.WROLMode}); } diff --git a/app/src/components/Channels.js b/app/src/components/Channels.js index 4e740e26..4b690f80 100644 --- a/app/src/components/Channels.js +++ b/app/src/components/Channels.js @@ -18,8 +18,7 @@ import { downloadChannel, getChannel, getChannels, - getConfig, - getDirectories, + getDirectories, getSettings, refreshChannel, updateChannel, validateRegex @@ -57,11 +56,11 @@ class DirectoryInput extends React.Component { } async componentDidMount() { - let global_config = await getConfig(); + let settings = await getSettings(); let directories = await getDirectories(this.state.directory); this.setState({ directories, - mediaDirectory: global_config.media_directory, + mediaDirectory: settings.media_directory, directory: this.props.value, }); } diff --git a/app/src/components/Common.js b/app/src/components/Common.js index 54cd25bc..84d59b86 100644 --- a/app/src/components/Common.js +++ b/app/src/components/Common.js @@ -4,7 +4,7 @@ import {Link, NavLink} from "react-router-dom"; import Button from "semantic-ui-react/dist/commonjs/elements/Button"; import Grid from "semantic-ui-react/dist/commonjs/collections/Grid"; import Message from "semantic-ui-react/dist/commonjs/collections/Message"; -import {getConfig} from "../api"; +import {getConfig, getSettings} from "../api"; import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks"; export const API_URI = `http://${window.location.host}/api`; @@ -560,8 +560,8 @@ export const useWROLMode = () => { const [enabled, setEnabled] = useState(false); const fetchStatus = async () => { - let config = await getConfig(); - setEnabled(config['wrol_mode']); + let settings = await getSettings(); + setEnabled(settings.wrol_mode); } useEffect(() => { @@ -674,3 +674,14 @@ export function DisableDownloadsToggle() { onChange={(e, data) => setDownloads(data.checked)} />; } + +export function emptyToNull(obj) { + const keys = Object.keys(obj); + for (let i = 0; i < keys.length; i++) { + const key = keys[i]; + if (obj[key] === '') { + obj[key] = null; + } + } + return obj; +} diff --git a/config/example.yaml b/config/example.yaml deleted file mode 100644 index 6047f7d5..00000000 --- a/config/example.yaml +++ /dev/null @@ -1,27 +0,0 @@ -# This is an example config for your WROLPi. Do NOT modify this file, rather, copy this file to local.yaml and -# modify that file. -media_directory: /media/wrolpi # this should be an absolute directory, for example, /media/wrolpi - -downloader: - # Most of these names were decided by YT-DLP project, we pass these to any calls to that project. - continue_dl: true - dateafter: 19900101 - file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s" - format: 22/135/18/43 - nooverwrites: true - quiet: false - writeautomaticsub: true - writeinfojson: true - writesubtitles: true - writethumbnail: true - youtube_include_dash_manifest: false - -channels: - # These are channels or playlists that will be downloaded. You can configure them in the UI, or in your local.yaml. - wrolpi: - name: WROLPi # the pretty name for the channel - url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos - directory: videos/wrolpi # relative to `media_directory` - download_frequency: 604800 - -timezone: America/Boise diff --git a/main.py b/main.py index d19abc2a..92238728 100755 --- a/main.py +++ b/main.py @@ -8,8 +8,8 @@ import sys import pytz from sanic import Sanic -from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent -from wrolpi.common import logger, get_config, wrol_mode_enabled, import_modules, check_media_directory +from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent, admin +from wrolpi.common import logger, get_config, import_modules, check_media_directory, set_wrol_mode from wrolpi.dates import set_timezone from wrolpi.downloader import download_manager from wrolpi.vars import PROJECT_DIR, DOCKERIZED @@ -89,10 +89,15 @@ async def main(loop): # Set the Timezone config = get_config() - if 'timezone' in config: - tz = pytz.timezone(config['timezone']) + if config.timezone: + tz = pytz.timezone(config.timezone) set_timezone(tz) + if config.hotspot_on_startup: + admin.enable_hotspot() + if config.throttle_on_startup: + admin.throttle_cpu_on() + check_media_directory() # Import the API in every module. Each API should attach itself to `root_api`. @@ -131,12 +136,15 @@ async def set_log_level(args): @after_startup @limit_concurrent(1) def periodic_downloads(app: Sanic, loop): - """ - A simple function that perpetually calls downloader_manager.do_downloads() after sleeping. - """ - if wrol_mode_enabled(): + """A simple function that perpetually calls downloader_manager.do_downloads() after sleeping.""" + config = get_config() + if config.wrol_mode: logger.warning(f'Not starting download worker because WROL Mode is enabled.') return + if config.download_on_startup is False: + logger.warning(f'Not starting download worker because Downloads are disabled on startup.') + return + logger.info('Starting download manager.') # Set all downloads to new. diff --git a/media_directory b/media_directory new file mode 100644 index 00000000..e6a9b45f --- /dev/null +++ b/media_directory @@ -0,0 +1 @@ +/media/wrolpi diff --git a/modules/files/lib.py b/modules/files/lib.py index ac5d3c80..4a215094 100644 --- a/modules/files/lib.py +++ b/modules/files/lib.py @@ -5,7 +5,6 @@ from pathlib import Path from typing import List from uuid import uuid4 -import magic import psycopg2 from sqlalchemy.orm import Session @@ -16,6 +15,13 @@ from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidFile from wrolpi.vars import PYTEST +try: + import magic + have_magic = True +except ImportError: + logger.error('Could not import magic. Is libmagic installed?') + have_magic = False + logger = logger.getChild(__name__) __all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search'] @@ -96,7 +102,7 @@ def upsert_file(path: Path, session: Session) -> File: if not file: file = File(path=path) session.add(file) - if not file.mimetype: + if not file.mimetype and have_magic: file.mimetype = magic.from_file(str(path), mime=True) if not file.title: file.title = split_file_name(path) diff --git a/modules/inventory/api.py b/modules/inventory/api.py index a4b61750..56be56a3 100644 --- a/modules/inventory/api.py +++ b/modules/inventory/api.py @@ -5,7 +5,7 @@ from sanic.request import Request from sanic_ext import validate from sanic_ext.extensions.openapi import openapi -from wrolpi.common import run_after, remove_dict_value_whitespace +from wrolpi.common import run_after, recursive_map from wrolpi.errors import ValidationError from wrolpi.root_api import get_blueprint, json_response from . import common, inventory, schema @@ -48,9 +48,8 @@ def inventory_get(_: Request, inventory_id: int): ) @validate(schema.InventoryPostRequest) @run_after(common.save_inventories_file) -def post_inventory(_: Request, data: dict): - # Cleanup the whitespace. - data = remove_dict_value_whitespace(data) +def post_inventory(_: Request, body: schema.InventoryPostRequest): + data = remove_whitespace(body.__dict__) inventory.save_inventory(data) return response.empty(HTTPStatus.CREATED) @@ -63,9 +62,8 @@ def post_inventory(_: Request, data: dict): ) @validate(schema.InventoryPutRequest) @run_after(common.save_inventories_file) -def put_inventory(_: Request, inventory_id: int, data: dict): - # Cleanup the whitespace. - data = remove_dict_value_whitespace(data) +def put_inventory(_: Request, inventory_id: int, body: schema.InventoryPutRequest): + data = remove_whitespace(body.__dict__) inventory.update_inventory(inventory_id, data) return response.empty() @@ -93,9 +91,8 @@ def items_get(_: Request, inventory_id: int): ) @validate(schema.ItemPostRequest) @run_after(common.save_inventories_file) -def post_item(_: Request, inventory_id: int, data: dict): - # Cleanup the whitespace. - data = remove_dict_value_whitespace(data) +def post_item(_: Request, inventory_id: int, body: schema.ItemPostRequest): + data = remove_whitespace(body.__dict__) inventory.save_item(inventory_id, data) return response.empty() @@ -108,9 +105,8 @@ def post_item(_: Request, inventory_id: int, data: dict): ) @validate(schema.ItemPutRequest) @run_after(common.save_inventories_file) -def put_item(_: Request, item_id: int, data: dict): - # Cleanup the whitespace. - data = remove_dict_value_whitespace(data) +def put_item(_: Request, item_id: int, body: schema.ItemPutRequest): + data = remove_whitespace(body.__dict__) inventory.update_item(item_id, data) return response.empty() @@ -131,3 +127,7 @@ def item_delete(_: Request, item_ids: str): inventory.delete_items(item_ids) return response.empty() + + +def remove_whitespace(obj): + return recursive_map(obj, lambda i: i.strip() if hasattr(i, 'strip') else i) diff --git a/modules/inventory/common.py b/modules/inventory/common.py index 7b4e1b05..547166e1 100644 --- a/modules/inventory/common.py +++ b/modules/inventory/common.py @@ -3,14 +3,12 @@ from functools import partial from pathlib import Path from typing import List, Tuple -import yaml from pint import Quantity from wrolpi import before_startup -from wrolpi.common import logger, Base +from wrolpi.common import logger, Base, ConfigFile from wrolpi.db import get_db_session from wrolpi.errors import NoInventories, InventoriesVersionMismatch -from wrolpi.vars import CONFIG_DIR from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \ get_inventories_version from .models import Inventory, Item @@ -98,23 +96,65 @@ def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quanti def cleanup_quantity(quantity: Quantity) -> Quantity: - """ - Remove trailing zeros from a Quantity. - """ + """Remove trailing zeros from a Quantity.""" num, unit = quantity_to_tuple(quantity) num = round(num, UNIT_PRECISION) num = str(num).rstrip('0').rstrip('.') return Decimal(num) * unit -DEFAULT_SAVE_PATH = CONFIG_DIR / 'inventories.yaml' +class InventoriesConfig(ConfigFile): + file_name = 'inventories.yaml' + default_config = dict( + inventories=[], + version=1, + ) + + @property + def inventories(self) -> dict: + return self._config['inventories'] + + @inventories.setter + def inventories(self, value: dict): + self.update({'inventories': value}) + + @property + def version(self) -> int: + return self._config['version'] + + @version.setter + def version(self, value: int): + self.update({'version': value}) -def save_inventories_file(path: str = None): - """ - Write all inventories and their respective items to a YAML file. - """ - path: Path = Path(path) if path else DEFAULT_SAVE_PATH +INVENTORIES_CONFIG: InventoriesConfig = None +TEST_INVENTORIES_CONFIG: InventoriesConfig = None + + +def get_inventories_config(): + global TEST_INVENTORIES_CONFIG + if isinstance(TEST_INVENTORIES_CONFIG, ConfigFile): + return TEST_INVENTORIES_CONFIG + + global INVENTORIES_CONFIG + if isinstance(INVENTORIES_CONFIG, ConfigFile): + return INVENTORIES_CONFIG + + INVENTORIES_CONFIG = InventoriesConfig() + return INVENTORIES_CONFIG + + +def set_test_inventories_config(enabled: bool): + global TEST_INVENTORIES_CONFIG + if enabled: + TEST_INVENTORIES_CONFIG = InventoriesConfig() + else: + TEST_INVENTORIES_CONFIG = None + + +def save_inventories_file(): + """Write all inventories and their respective items to a YAML file.""" + config = get_inventories_config() inventories = [] for inventory in get_inventories(): @@ -124,44 +164,22 @@ def save_inventories_file(path: str = None): raise NoInventories('No Inventories are in the database!') with increment_inventories_version() as version: - if path.is_file(): - # Check that we aren't overwriting our inventories with empty inventories. - with open(path, 'rt') as fh: - old = yaml.load(fh, Loader=yaml.Loader) - if old and not inventories: - raise FileExistsError(f'Refusing to overwrite non-empty inventories.yaml with empty inventories.' - f' {path}') + if config.version and config.version > version: + raise InventoriesVersionMismatch( + f'Inventories config version is {config.version} but DB version is {get_inventories_version()}') - if old and old.get('version') > version: - raise InventoriesVersionMismatch( - f'Inventories config version is {old["version"]} but DB version is {get_inventories_version()}') - - with open(path, 'wt') as fh: - contents = dict( - version=version, - inventories=inventories, - ) - - yaml.dump(contents, fh) + config.inventories = inventories + config.version = version + config.save() @before_startup -def import_inventories_file(path: str = None): - path: Path = Path(path) if path else DEFAULT_SAVE_PATH - - if not path.is_file(): - logger.warning(f'No inventories config file at {path}') - return - - with open(path, 'rt') as fh: - contents = yaml.load(fh, Loader=yaml.Loader) - - if not contents or 'inventories' not in contents: - raise ValueError('Inventories file does not contain the expected "inventories" list.') +def import_inventories_file(): + config = get_inventories_config() inventories = get_inventories() inventories_names = {i.name for i in inventories} - new_inventories = [i for i in contents['inventories'] if i['name'] not in inventories_names] + new_inventories = [i for i in config.inventories if i['name'] not in inventories_names] with get_db_session(commit=True) as session: for inventory in new_inventories: items = inventory['items'] diff --git a/modules/inventory/conftest.py b/modules/inventory/conftest.py index d6cc0ab4..e3990494 100644 --- a/modules/inventory/conftest.py +++ b/modules/inventory/conftest.py @@ -1,6 +1,7 @@ import pytest from modules.inventory import Inventory, Item, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES +from modules.inventory.common import set_test_inventories_config @pytest.fixture @@ -23,4 +24,6 @@ def init_test_inventory(test_session): inventory = test_session.query(Inventory).filter_by(name='Food Storage').one() test_session.commit() - return inventory + set_test_inventories_config(True) + yield inventory + set_test_inventories_config(False) diff --git a/modules/inventory/inventory.py b/modules/inventory/inventory.py index cbd8b05b..dceaf13c 100644 --- a/modules/inventory/inventory.py +++ b/modules/inventory/inventory.py @@ -59,7 +59,7 @@ DEFAULT_INVENTORIES = [ sort_categories = partial(sorted, key=itemgetter(1, 0)) -def get_inventories() -> List[Base]: +def get_inventories() -> List[Inventory]: with get_db_session() as session: inventories = session.query(Inventory).filter( Inventory.deleted_at == None, # noqa diff --git a/modules/inventory/schema.py b/modules/inventory/schema.py index 78c502e7..bc2b37bf 100644 --- a/modules/inventory/schema.py +++ b/modules/inventory/schema.py @@ -1,36 +1,51 @@ from dataclasses import dataclass -from datetime import date, datetime from decimal import Decimal +from typing import Union, Optional + +from wrolpi.dates import from_timestamp @dataclass class ItemPostRequest: brand: str name: str - item_size: Decimal + item_size: Union[str, Decimal] unit: str - count: Decimal + count: Union[str, Decimal] category: str subcategory: str - expiration_date: date + expiration_date: Optional[float] + + def __post_init__(self): + self.item_size = Decimal(self.item_size) if self.item_size else None + self.count = Decimal(self.count) if self.count else None + self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None @dataclass class ItemPutRequest: id: int # Ignored in favor of URL - brand: str - name: str - item_size: Decimal - unit: str - count: Decimal - serving: Decimal - category: str - subcategory: str - expiration_date: date - purchase_date: date - created_at: datetime - deleted_at: datetime + brand: Optional[str] + name: Optional[str] + item_size: Union[str, Decimal] + unit: Optional[str] + count: Union[str, Decimal] + serving: Union[str, Decimal, None] + category: Optional[str] + subcategory: Optional[str] + expiration_date: Optional[float] + purchase_date: Optional[float] inventory_id: int + created_at: Optional[float] + deleted_at: Optional[float] + + def __post_init__(self): + self.item_size = Decimal(self.item_size) if self.item_size else None + self.count = Decimal(self.count) if self.count else None + self.serving = Decimal(self.serving) if self.serving else None + self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None + self.created_at = from_timestamp(self.created_at) if self.created_at else None + self.deleted_at = from_timestamp(self.deleted_at) if self.deleted_at else None @dataclass diff --git a/modules/inventory/test/test_api.py b/modules/inventory/test/test_api.py index e686f039..08d7fb21 100644 --- a/modules/inventory/test/test_api.py +++ b/modules/inventory/test/test_api.py @@ -1,3 +1,4 @@ +import json from http import HTTPStatus from modules.inventory import Item @@ -49,3 +50,47 @@ def test_delete_item(test_session, init_test_inventory, test_client): assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code test_session.refresh(item) assert item.deleted_at + + +def test_item_api(test_session, init_test_inventory, test_client): + """An Item can be added to an Inventory.""" + inventory_id = init_test_inventory.id + item = {'brand': 'Salty', 'name': 'Salt', 'item_size': '1', 'unit': 'lbs', 'count': '5', + 'category': 'cooking ingredients', 'subcategory': 'salt', 'expiration_date': None} + request, response = test_client.post(f'/api/inventory/{inventory_id}/item', content=json.dumps(item)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code + + # Get the Item we just created. + request, response = test_client.get(f'/api/inventory/{inventory_id}/item') + assert response.status_code == HTTPStatus.OK + # assert item == response.json['items'][0] + item = response.json['items'][0] + + expiration_dates = (12345.12345, None) + for expiration_date in expiration_dates: + item['expiration_date'] = expiration_date + request, response = test_client.put(f'/api/inventory/item/{item["id"]}', content=json.dumps(item)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json + + +def test_inventory_api(test_session, test_client): + """An Inventory can be created.""" + inventory = dict(name='foo') + request, response = test_client.post('/api/inventory', content=json.dumps(inventory)) + assert response.status_code == HTTPStatus.CREATED, response.status_code + + request, response = test_client.get('/api/inventory') + assert response.status_code == HTTPStatus.OK + inventory = response.json['inventories'][0] + inventory_id = inventory['id'] + + inventory = dict(name='new name') + request, response = test_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory)) + assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code + + request, response = test_client.delete(f'/api/inventory/{inventory_id}') + assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code + + request, response = test_client.get('/api/inventory') + assert response.status_code == HTTPStatus.OK + assert len(response.json['inventories']) == 0 diff --git a/modules/inventory/test/test_inventory.py b/modules/inventory/test/test_inventory.py index 9da58ceb..ce57f608 100644 --- a/modules/inventory/test/test_inventory.py +++ b/modules/inventory/test/test_inventory.py @@ -1,5 +1,4 @@ -import os -import tempfile +from decimal import Decimal from decimal import Decimal from itertools import zip_longest from typing import List, Iterable @@ -15,7 +14,7 @@ from wrolpi.errors import NoInventories, InventoriesVersionMismatch from wrolpi.test.common import PytestCase from .. import init from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \ - compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file + compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config from ..inventory import unit_registry, \ get_inventories, save_inventory, update_inventory, \ delete_inventory, get_categories @@ -189,75 +188,6 @@ class TestInventory(PytestCase): for idx, (i, j) in enumerate(zip_longest(inventory, expected)): self.assertEqual(i, j, f'named inventory {idx} is not equal') - def test_inventories_file(self, test_session): - self.prepare() - - with tempfile.NamedTemporaryFile() as tf: - # Can't import an empty file. - self.assertRaises(ValueError, import_inventories_file, tf.name) - - save_inventories_file(tf.name) - - # Clear out the DB so the import will be tested - with get_db_session(commit=True) as session: - session.query(Item).delete() - session.query(Inventory).delete() - - import_inventories_file(tf.name) - - inventories = get_inventories() - self.assertEqual(len(inventories), 1) - # ID has increased because we did not reset the sequence when deleting from the table. - self.assertEqual(inventories[0].id, 2) - self.assertEqual(inventories[0].name, 'Food Storage') - - # All items in the DB match those in the test list, except for the "deleted" item. - non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None] - self.assertEqual(len(non_deleted_items), len(TEST_ITEMS) - 1) - test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]} - db_items = {(i.name, i.brand, i.count) for i in non_deleted_items} - self.assertEqual(test_items, db_items) - - def test_inventories_version(self, test_session): - """ - You can't save over a newer version of an inventory. - """ - self.prepare() - with tempfile.NamedTemporaryFile(delete=False) as tf: - pass - - try: - with open(tf.name, 'rt') as fh: - # Version is set to 1 on first save. - save_inventories_file(tf.name) - inventories_contents = yaml.load(fh, Loader=yaml.Loader) - self.assertEqual(inventories_contents['version'], 1) - - with open(tf.name, 'rt') as fh: - # Version is incremented when saving. - save_inventories_file(tf.name) - inventories_contents = yaml.load(fh, Loader=yaml.Loader) - self.assertEqual(inventories_contents['version'], 2) - - with open(tf.name, 'wt') as fh: - # Version is greater than what will be saved. - inventories_contents['version'] = 5 - yaml.dump(inventories_contents, fh) - - self.assertRaises(InventoriesVersionMismatch, save_inventories_file, tf.name) - - finally: - os.unlink(tf.name) - - def test_no_inventories(self, test_session, tmp_path): - """ - Can't save empty inventories. - """ - try: - save_inventories_file(tmp_path.name) - except NoInventories: - pass - def quantity_to_string(quantity: Quantity) -> str: quantity = cleanup_quantity(quantity) @@ -333,3 +263,66 @@ length = gallon.dimensionality def test_sum_by_key(items, expected): items = [Item(**i) for i in items] assert sum_by_key(items, lambda i: (i.category,)) == expected + + +def test_no_inventories(test_session, test_directory): + """Can't save empty inventories.""" + try: + save_inventories_file() + except NoInventories: + pass + + +def test_inventories_version(test_session, test_directory, init_test_inventory): + """You can't save over a newer version of an inventory.""" + for item in TEST_ITEMS: + item = Item(**item) + test_session.add(item) + test_session.commit() + + # Version is set to 1 on first save. + save_inventories_file() + config = get_inventories_config() + assert config.version == 1 + + # Version is incremented when saving. + save_inventories_file() + config = get_inventories_config() + assert config.version == 2 + + # Version is greater than what will be saved. + with config.get_file().open('wt') as fh: + config._config['version'] = 5 + yaml.dump(config._config, fh) + with pytest.raises(InventoriesVersionMismatch): + save_inventories_file() + + +def test_inventories_config(test_session, test_directory, init_test_inventory): + for item in TEST_ITEMS: + item = Item(**item) + test_session.add(item) + test_session.commit() + + # Save the Inventories/Items that were created above. + save_inventories_file() + + # Clear out the DB so the import will be tested + test_session.query(Item).delete() + test_session.query(Inventory).delete() + test_session.commit() + + import_inventories_file() + + inventories = get_inventories() + assert len(inventories) == 1 + # ID has increased because we did not reset the sequence when deleting from the table. + assert inventories[0].id == 2 + assert inventories[0].name == 'Food Storage' + + # All items in the DB match those in the test list, except for the "deleted" item. + non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None] + assert len(non_deleted_items) == len(TEST_ITEMS) - 1 + test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]} + db_items = {(i.name, i.brand, i.count) for i in non_deleted_items} + assert test_items == db_items diff --git a/modules/otp/api.py b/modules/otp/api.py index f786f4c6..47cd4220 100644 --- a/modules/otp/api.py +++ b/modules/otp/api.py @@ -19,8 +19,8 @@ bp = get_blueprint('OTP', '/api/otp') body=schema.EncryptOTPRequest, ) @validate(schema.EncryptOTPRequest) -async def post_encrypt_otp(_: Request, data: dict): - data = lib.encrypt_otp(data['otp'], data['plaintext']) +async def post_encrypt_otp(_: Request, body: schema.EncryptOTPRequest): + data = lib.encrypt_otp(body.otp, body.plaintext) return response.json(data) @@ -30,8 +30,8 @@ async def post_encrypt_otp(_: Request, data: dict): body=schema.DecryptOTPRequest, ) @validate(schema.DecryptOTPRequest) -async def post_decrypt_otp(_: Request, data: dict): - data = lib.decrypt_otp(data['otp'], data['ciphertext']) +async def post_decrypt_otp(_: Request, body: schema.DecryptOTPRequest): + data = lib.decrypt_otp(body.otp, body.ciphertext) return response.json(data) diff --git a/modules/otp/test/test_api.py b/modules/otp/test/test_api.py new file mode 100644 index 00000000..23f6bd6f --- /dev/null +++ b/modules/otp/test/test_api.py @@ -0,0 +1,29 @@ +import json +from http import HTTPStatus + + +def test_encrypt(test_client): + body = dict(otp='asdf', plaintext='asdf') + request, response = test_client.post('/api/otp/encrypt_otp', content=json.dumps(body)) + assert response.status_code == HTTPStatus.OK + assert response.json == dict( + ciphertext='AAGK', + otp='ASDF', + plaintext='ASDF', + ) + + +def test_decrypt(test_client): + body = dict(otp='asdf', ciphertext='aagk') + request, response = test_client.post('/api/otp/decrypt_otp', content=json.dumps(body)) + assert response.status_code == HTTPStatus.OK + assert response.json == dict( + ciphertext='AAGK', + otp='ASDF', + plaintext='ASDF', + ) + + +def test_get_pdf(test_client): + request, response = test_client.get('/api/otp/pdf') + assert response.status_code == HTTPStatus.OK diff --git a/modules/videos/api.py b/modules/videos/api.py index 91a8aa7a..a07a6366 100644 --- a/modules/videos/api.py +++ b/modules/videos/api.py @@ -12,11 +12,11 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \ from wrolpi.common import logger from wrolpi.root_api import add_blueprint, json_response from wrolpi.schema import JSONErrorResponse +from . import lib, schema from .channel import lib as channel_lib from .channel.api import channel_bp from .video import lib as video_lib from .video.api import video_bp -from . import lib, schema content_bp = Blueprint('VideoContent', '/api/videos') bp = Blueprint('Videos', '/api/videos').group( diff --git a/modules/videos/channel/api.py b/modules/videos/channel/api.py index a372ec5f..6e942537 100644 --- a/modules/videos/channel/api.py +++ b/modules/videos/channel/api.py @@ -72,7 +72,7 @@ def channel_post(_: Request, body: schema.ChannelPostRequest): @validate(schema.ChannelPutRequest) @wrol_mode_check def channel_update(_: Request, link: str, body: schema.ChannelPutRequest): - channel = lib.update_channel(body.__dict__, link) # TODO don't use the dataclass as a dict. + channel = lib.update_channel(body.__dict__, link) return response.raw('', HTTPStatus.NO_CONTENT, headers={'Location': f'/api/videos/channels/{channel.link}'}) diff --git a/modules/videos/common.py b/modules/videos/common.py index 6dcdcec8..e773ec26 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -14,7 +14,7 @@ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from wrolpi import before_startup -from wrolpi.common import logger, get_config, iterify, get_media_directory, \ +from wrolpi.common import logger, iterify, get_media_directory, \ minimize_dict, api_param_limiter from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \ @@ -36,19 +36,6 @@ VIDEO_QUERY_MAX_LIMIT = 100 # These are the supported video formats. These are in order of their preference. VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv') - -def load_downloader_config() -> dict: - config = get_config() - downloader = config['downloader'] - return downloader - - -def load_channels_config() -> dict: - config = get_config() - channels = config['channels'] - return channels - - video_limiter = api_param_limiter(100) @@ -81,8 +68,9 @@ def import_videos_config(): """Import channel settings to the DB. Existing channels will be updated.""" logger.info('Importing videos config') try: - config = get_config() - config, favorites = config['channels'], config.get('favorites', {}) + from .lib import get_channels_config + config = get_channels_config() + config, favorites = config.channels, config.favorites with get_db_session(commit=True) as session: for link in config: diff --git a/modules/videos/configs/channels.yaml b/modules/videos/configs/channels.yaml new file mode 100644 index 00000000..c39ef457 --- /dev/null +++ b/modules/videos/configs/channels.yaml @@ -0,0 +1,8 @@ +# This is an example config. Do not edit this file. Edit the config in your media directory! +channels: + wrolpi: + name: WROLPi + url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos + directory: videos/wrolpi + download_frequency: 604800 +favorites: [ ] diff --git a/modules/videos/configs/downloader.yaml b/modules/videos/configs/downloader.yaml new file mode 100644 index 00000000..49377c92 --- /dev/null +++ b/modules/videos/configs/downloader.yaml @@ -0,0 +1,12 @@ +# Most of these names were decided by YT-DLP project, we pass these to any calls to that project. +continue_dl: true +dateafter: 19900101 +file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s" +format: 22/135/18/43 +nooverwrites: true +quiet: false +writeautomaticsub: true +writeinfojson: true +writesubtitles: true +writethumbnail: true +youtube_include_dash_manifest: false diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index b25edee4..03ecb4ae 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -8,6 +8,7 @@ import pytest from PIL import Image from modules.videos.downloader import VideoDownloader, ChannelDownloader +from modules.videos.lib import set_test_channels_config from modules.videos.models import Channel, Video from wrolpi.common import sanitize_link from wrolpi.downloader import DownloadFrequency, DownloadManager, Download @@ -152,3 +153,12 @@ def video_download_manager(test_download_manager) -> DownloadManager: with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: # prevent real downloads mock_extract_info.side_effect = Exception('You must patch modules.videos.downloader.YDL.extract_info!') yield test_download_manager + + +@pytest.fixture +def test_channels_config(test_directory): + (test_directory / 'config').mkdir(exist_ok=True) + config_path = test_directory / 'config/channels.yaml' + set_test_channels_config(True) + yield config_path + set_test_channels_config(False) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index baf6d71b..44eb1f47 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -18,9 +18,9 @@ from wrolpi.downloader import Downloader, Download from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel -from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \ +from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \ get_videos_directory -from .lib import upsert_video, refresh_channel_videos +from .lib import upsert_video, refresh_channel_videos, get_downloader_config from .models import Video, Channel from .video_url_resolver import video_url_resolver @@ -217,11 +217,11 @@ class VideoDownloader(Downloader, ABC): raise ValueError(f'Output directory does not exist! {out_dir=}') # YoutubeDL expects specific options, add onto the default options - config = load_downloader_config() + config = get_downloader_config().dict() options = config.copy() options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}' options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION - # We don't need format when getting the filename. + # We don't need "format" when getting the filename. del options['format'] logger.debug(f'Downloading {url} to {out_dir}') diff --git a/modules/videos/lib.py b/modules/videos/lib.py index a10fb93a..c7c2e395 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -8,8 +8,7 @@ from uuid import uuid1 from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session -from wrolpi.common import logger, chunks, get_config -from wrolpi.common import save_settings_config +from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath @@ -289,10 +288,186 @@ def refresh_videos(channel_links: list = None): logger.info('Refresh of video files complete') -def get_channels_config(session: Session) -> dict: - """ - Create a dictionary that contains all the Channels from the DB. - """ +class ChannelsConfig(ConfigFile): + file_name = 'channels.yaml' + default_config = dict( + channels={ + 'wrolpi': dict( + name='WROLPi', + url='https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos', + directory='videos/wrolpi', + download_frequency=604800, + ) + }, + favorites=dict(), + ) + + @property + def channels(self) -> dict: + return self._config['channels'] + + @channels.setter + def channels(self, value: dict): + self.update({'channels': value}) + + @property + def favorites(self) -> dict: + return self._config['favorites'] + + @favorites.setter + def favorites(self, value: dict): + self.update({'favorites': value}) + + +CHANNELS_CONFIG: ChannelsConfig = None +TEST_CHANNELS_CONFIG: ChannelsConfig = None + + +def get_channels_config(): + global TEST_CHANNELS_CONFIG + if isinstance(TEST_CHANNELS_CONFIG, ConfigFile): + return TEST_CHANNELS_CONFIG + + global CHANNELS_CONFIG + if isinstance(CHANNELS_CONFIG, ConfigFile): + return CHANNELS_CONFIG + + CHANNELS_CONFIG = ChannelsConfig() + return CHANNELS_CONFIG + + +def set_test_channels_config(enable: bool): + global TEST_CHANNELS_CONFIG + if enable is True: + TEST_CHANNELS_CONFIG = ChannelsConfig() + else: + TEST_CHANNELS_CONFIG = None + + +class DownloaderConfig(ConfigFile): + file_name = 'downloader.yaml' + default_config = dict( + continue_dl=True, + dateafter='19900101', + file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s', + format='22 / 135 / 18 / 43', + nooverwrites=True, + quiet=False, + writeautomaticsub=True, + writeinfojson=True, + writesubtitles=True, + writethumbnail=True, + youtube_include_dash_manifest=False, + ) + + @property + def continue_dl(self) -> bool: + return self._config['continue_dl'] + + @continue_dl.setter + def continue_dl(self, value: bool): + self.update({'continue_dl': value}) + + @property + def dateafter(self) -> str: + return self._config['dateafter'] + + @dateafter.setter + def dateafter(self, value: str): + self.update({'dateafter': value}) + + @property + def file_name_format(self) -> str: + return self._config['file_name_format'] + + @file_name_format.setter + def file_name_format(self, value: str): + self.update({'file_name_format': value}) + + @property + def format(self) -> str: + return self._config['format'] + + @format.setter + def format(self, value: str): + self.update({'format': value}) + + @property + def nooverwrites(self) -> bool: + return self._config['nooverwrites'] + + @nooverwrites.setter + def nooverwrites(self, value: bool): + self.update({'nooverwrites': value}) + + @property + def quiet(self) -> bool: + return self._config['quiet'] + + @quiet.setter + def quiet(self, value: bool): + self.update({'quiet': value}) + + @property + def writeautomaticsub(self) -> bool: + return self._config['writeautomaticsub'] + + @writeautomaticsub.setter + def writeautomaticsub(self, value: bool): + self.update({'writeautomaticsub': value}) + + @property + def writeinfojson(self) -> bool: + return self._config['writeinfojson'] + + @writeinfojson.setter + def writeinfojson(self, value: bool): + self.update({'writeinfojson': value}) + + @property + def writesubtitles(self) -> bool: + return self._config['writesubtitles'] + + @writesubtitles.setter + def writesubtitles(self, value: bool): + self.update({'writesubtitles': value}) + + @property + def writethumbnail(self) -> bool: + return self._config['writethumbnail'] + + @writethumbnail.setter + def writethumbnail(self, value: bool): + self.update({'writethumbnail': value}) + + @property + def youtube_include_dash_manifest(self) -> bool: + return self._config['youtube_include_dash_manifest'] + + @youtube_include_dash_manifest.setter + def youtube_include_dash_manifest(self, value: bool): + self.update({'youtube_include_dash_manifest': value}) + + +DOWNLOADER_CONFIG: DownloaderConfig = None +TEST_DOWNLOADER_CONFIG: DownloaderConfig = None + + +def get_downloader_config(): + global TEST_DOWNLOADER_CONFIG + if isinstance(TEST_DOWNLOADER_CONFIG, WROLPiConfig): + return TEST_DOWNLOADER_CONFIG + + global DOWNLOADER_CONFIG + if isinstance(DOWNLOADER_CONFIG, WROLPiConfig): + return DOWNLOADER_CONFIG + + DOWNLOADER_CONFIG = DownloaderConfig() + return DOWNLOADER_CONFIG + + +def get_channels_config_from_db(session: Session) -> dict: + """Create a dictionary that contains all the Channels from the DB.""" channels = session.query(Channel).order_by(Channel.link).all() channels = {i.link: i.config_view() for i in channels} @@ -315,10 +490,11 @@ def save_channels_config(session=None, preserve_favorites: bool = True): """ Pull the Channel information from the DB, save it to the config. """ - config = get_channels_config(session) + config = get_channels_config_from_db(session) + channels_config = get_channels_config() if preserve_favorites: - config['favorites'].update(get_config().get('favorites', {})) - save_settings_config(config) + config['favorites'].update(channels_config.favorites or {}) + channels_config.update(config) async def get_statistics(): diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index aff14313..0a2ef799 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -11,7 +11,7 @@ from PIL import Image from modules.videos.models import Channel, Video from modules.videos.test.common import create_channel_structure -from wrolpi.common import get_absolute_media_path, get_config +from wrolpi.common import get_absolute_media_path from wrolpi.dates import local_timezone from wrolpi.db import get_db_session from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI @@ -19,7 +19,7 @@ from wrolpi.vars import PROJECT_DIR from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \ apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \ import_videos_config -from ..lib import save_channels_config +from ..lib import save_channels_config, get_channels_config class TestCommon(TestAPI): @@ -309,7 +309,7 @@ def test_update_censored_videos(test_session, video_factory, simple_channel): check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)]) -def test_import_favorites(test_session, simple_channel, video_factory): +def test_import_favorites(test_session, simple_channel, video_factory, test_channels_config): """ A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be favorited again. @@ -331,8 +331,8 @@ def test_import_favorites(test_session, simple_channel, video_factory): # Save config, verify that favorite is set. save_channels_config() - config = get_config() - assert config['favorites'] == favorites + config = get_channels_config() + assert config.favorites == favorites def import_and_verify(favorited_ids: List[int]): with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'): @@ -351,8 +351,8 @@ def test_import_favorites(test_session, simple_channel, video_factory): test_session.query(Video).filter_by(id=vid2.id).delete() test_session.commit() save_channels_config() - config = get_config() - assert config['favorites'] == favorites + config = get_channels_config() + assert config.favorites == favorites import_and_verify([vid3.id]) # Add vid2 again, it should be favorited on import. @@ -363,6 +363,6 @@ def test_import_favorites(test_session, simple_channel, video_factory): # Deleting the Video in the model really removes the favorite status. vid3.delete() - config = get_config() - assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}} + config = get_channels_config() + assert config.favorites == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}} import_and_verify([vid2.id]) diff --git a/requirements.txt b/requirements.txt index 12a12dc3..cdc47a63 100644 --- a/requirements.txt +++ b/requirements.txt @@ -7,6 +7,7 @@ Pint==0.18 psycopg2-binary==2.9.3 pytest-asyncio==0.17.2 pytest==6.2.5 +python-dotenv==0.19.2 python-magic==0.4.24 pytz==2021.3 PyYAML==5.3.1 # this is the only version available on RPi diff --git a/wrolpi/admin.py b/wrolpi/admin.py index 14232bbf..a7545800 100644 --- a/wrolpi/admin.py +++ b/wrolpi/admin.py @@ -3,25 +3,25 @@ import pathlib import subprocess from wrolpi.common import logger -from wrolpi.vars import PYTEST, DEFAULT_CPU_FREQUENCY +from wrolpi.vars import PYTEST, DEFAULT_CPU_FREQUENCY, DOCKERIZED logger = logger.getChild(__name__) # Location on an RPi SUDO_BIN = pathlib.Path('/usr/bin/sudo') -if not SUDO_BIN.is_file() and not PYTEST: +if not SUDO_BIN.is_file() and not PYTEST and not DOCKERIZED: logger.error('COULD NOT FIND sudo!!!') NMCLI_BIN = pathlib.Path('/usr/bin/nmcli') -if not NMCLI_BIN.is_file() and not PYTEST: +if not NMCLI_BIN.is_file() and not PYTEST and not DOCKERIZED: logger.error('COULD NOT FIND nmcli!!!') CPUFREQ_INFO_BIN = pathlib.Path('/usr/bin/cpufreq-info') -if not CPUFREQ_INFO_BIN.is_file() and not PYTEST: +if not CPUFREQ_INFO_BIN.is_file() and not PYTEST and not DOCKERIZED: logger.error('COULD NOT FIND cpufreq-info!!!') CPUFREQ_SET_BIN = pathlib.Path('/usr/bin/cpufreq-set') -if not CPUFREQ_SET_BIN.is_file() and not PYTEST: +if not CPUFREQ_SET_BIN.is_file() and not PYTEST and not DOCKERIZED: logger.error('COULD NOT FIND cpufreq-set!!!') POWER_SAVE_FREQ = 'powersave' # noqa @@ -31,11 +31,17 @@ class HotspotStatus(enum.Enum): disconnected = enum.auto() # Radio is on, but Hotspot is not connected. unavailable = enum.auto() # Radio is off. connected = enum.auto() # Radio is on, Hotspot is on. + unknown = enum.auto() # Unknown status. Hotspot may not be supported on this hardware. def hotspot_status() -> HotspotStatus: cmd = (NMCLI_BIN,) - output = subprocess.check_output(cmd).decode().strip() + try: + output = subprocess.check_output(cmd).decode().strip() + except FileNotFoundError as e: + if not DOCKERIZED: + logger.debug(f'Could not get hotspot status', exc_info=e) + return HotspotStatus.unknown for line in output.splitlines(): if line.startswith('wlan0: connected'): @@ -45,12 +51,13 @@ def hotspot_status() -> HotspotStatus: elif line.startswith('wlan0: unavailable'): return HotspotStatus.unavailable - raise Exception(f'Unknown status! {output=}') + return HotspotStatus.unknown def enable_hotspot(): - """ - Turn the wlan0 interface into a hotspot. If wlan0 is already running, replace that with a hotspot. + """Turn the wlan0 interface into a hotspot. + + If wlan0 is already running, replace that with a hotspot. """ status = hotspot_status() logger.warning(f'Hotspot status: {status}') @@ -69,13 +76,19 @@ def enable_hotspot(): cmd = (SUDO_BIN, NMCLI_BIN, 'radio', 'wifi', 'on') subprocess.check_call(cmd) return enable_hotspot() + elif status == HotspotStatus.unknown: + logger.error('Cannot enable hotspot with unknown status!') def disable_hotspot(): """Turn off the wlan0 interface.""" cmd = (SUDO_BIN, NMCLI_BIN, 'radio', 'wifi', 'off') - subprocess.check_call(cmd) - return True + try: + subprocess.check_call(cmd) + return True + except Exception as e: + logger.error('Failed to disable hotspot', exc_info=e) + return False class GovernorStatus(enum.Enum): @@ -90,14 +103,19 @@ class GovernorStatus(enum.Enum): GOVERNOR_MAP = { - 'The governor "ondemand" may decide ': GovernorStatus.ondemand, - 'The governor "powersave" may decide ': GovernorStatus.powersave, + 'governor "ondemand"': GovernorStatus.ondemand, + 'governor "powersave"': GovernorStatus.powersave, } def throttle_status() -> GovernorStatus: cmd = (CPUFREQ_INFO_BIN,) - output = subprocess.check_output(cmd) + try: + output = subprocess.check_output(cmd) + except FileNotFoundError as e: + if not DOCKERIZED: + logger.debug(f'Could not get CPU throttle info', exc_info=e) + return GovernorStatus.unknown output = output.decode().strip() for line in output.splitlines(): @@ -109,12 +127,22 @@ def throttle_status() -> GovernorStatus: def throttle_cpu_on() -> bool: - cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', POWER_SAVE_FREQ) - subprocess.check_call(cmd) - return True + """Call cpufreq-set to throttle CPU.""" + try: + cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', POWER_SAVE_FREQ) + subprocess.check_call(cmd) + return True + except FileNotFoundError: + if not DOCKERIZED: + logger.error('Could not enable CPU throttle', exc_info=True) def throttle_cpu_off() -> bool: - cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', DEFAULT_CPU_FREQUENCY) - subprocess.check_call(cmd) - return True + """Call cpufreq-set to un-throttle CPU.""" + try: + cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', DEFAULT_CPU_FREQUENCY) + subprocess.check_call(cmd) + return True + except FileNotFoundError: + if not DOCKERIZED: + logger.error('Could not disable CPU throttle', exc_info=True) diff --git a/wrolpi/common.py b/wrolpi/common.py index df217dd6..a7e8cded 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -10,26 +10,27 @@ import queue import re import string import tempfile +from copy import deepcopy from datetime import datetime, date from decimal import Decimal from functools import wraps from itertools import islice, filterfalse, tee -from multiprocessing import Event, Queue +from multiprocessing import Event, Queue, Lock, Manager from pathlib import Path -from typing import Union, Callable, Tuple, Dict, Mapping, List, Iterable, Optional, Generator +from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any from urllib.parse import urlunsplit, urlparse import yaml -from cachetools import cached, TTLCache from sanic import Blueprint from sanic.request import Request from sqlalchemy import types from sqlalchemy.ext.compiler import compiles from sqlalchemy.ext.declarative import declarative_base +from wrolpi.dates import DEFAULT_TIMEZONE from wrolpi.errors import WROLModeEnabled, NativeOnly -from wrolpi.vars import CONFIG_PATH, EXAMPLE_CONFIG_PATH, PUBLIC_HOST, PUBLIC_PORT, PROJECT_DIR, PYTEST, MODULES_DIR, \ - DOCKERIZED +from wrolpi.vars import PUBLIC_HOST, PUBLIC_PORT, PYTEST, MODULES_DIR, \ + DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY logger = logging.getLogger() ch = logging.StreamHandler() @@ -167,94 +168,169 @@ def get_sanic_url(scheme: str = 'http', path: str = None, query: list = None, fr return unparsed -def get_example_config() -> dict: - config_path = EXAMPLE_CONFIG_PATH - with open(str(config_path), 'rt') as fh: - config = yaml.load(fh, Loader=yaml.Loader) - return dict(config) - - -def get_local_config() -> dict: - config_path = CONFIG_PATH - with open(str(config_path), 'rt') as fh: - config = yaml.load(fh, Loader=yaml.Loader) - return dict(config) - - -def get_config() -> dict: - try: - return get_local_config() - except FileNotFoundError: - return get_example_config() - - -def combine_dicts(*dicts: dict) -> dict: +class ConfigFile: """ - Recursively combine dictionaries, preserving the leftmost value. - - >>> a = dict(a='b', c=dict(d='e')) - >>> b = dict(a='c', e='f') - >>> combine_dicts(a, b) - dict(a='b', c=dict(d='e'), e='f') + This class keeps track of the contents of a config file. You can update the config by calling + .update(), or by creating a property that does so. Save your changes using .save(). """ - if len(dicts) == 0: - raise IndexError('No dictionaries to iterate through') - elif len(dicts) == 1: - return dicts[0] - a, b = dicts[-2:] - c = dicts[:-2] - new = {} - keys = set(a.keys()) - keys = keys.union(b.keys()) - for k in keys: - if k in b and k in a and isinstance(b[k], Mapping): - value = combine_dicts(a[k], b[k]) - else: - value = a.get(k, b.get(k)) - new[k] = value - if c: - return combine_dicts(*c, new) - return new + file_name: str = None + default_config: dict = None + + def __init__(self): + config_file = self.get_file() + self.file_lock = Lock() + self._config = Manager().dict() + # Use the default settings to initialize the config. + self._config.update(deepcopy(self.default_config)) + if config_file.is_file(): + # Use the config file to get the values the user set. + with config_file.open('rt') as fh: + self._config.update(yaml.load(fh, Loader=yaml.Loader)) + + def save(self): + """ + Write this config to its file. + + Use the existing config file as a template; if any values are missing in the new config, use the values from the + config file. + """ + config_file = self.get_file() + # Don't overwrite a real config while testing. + if PYTEST and not str(config_file).startswith('/tmp'): + raise ValueError(f'Refusing to save config file while testing: {config_file}') + + # Only one process can write to the file. + self.file_lock.acquire(block=True, timeout=5.0) + + try: + # Config directory may not exist while testing. + if PYTEST and not config_file.parent.is_dir(): + config_file.parent.mkdir() + + # Read the existing config, replace all values, then save. + if config_file.is_file(): + with config_file.open('rt') as fh: + config = yaml.load(fh, Loader=yaml.Loader) + else: + # Config file does not yet exist. + config = dict() + + config.update({k: v for k, v in self._config.items() if v is not None}) + with config_file.open('wt') as fh: + yaml.dump(config, fh) + finally: + self.file_lock.release() + + def get_file(self) -> Path: + if not self.file_name: + raise NotImplementedError(f'You must define a file name for this {self.__class__.__name__} config.') + + if PYTEST: + return get_media_directory() / f'config/{self.file_name}' + + return CONFIG_DIR / self.file_name + + def update(self, config: dict): + """ + Update any values of this config. Save the config to its file. + """ + config = {k: v for k, v in config.items() if k in self._config} + self._config.update(config) + self.save() + + def dict(self): + """ + Get a deepcopy of this config. + """ + return deepcopy(self._config) -def save_settings_config(config=None): - """ - Save new settings to local.yaml, overwriting what is there. This function updates the config file from three - sources: the config object argument, the local config, then the example config; in that order. - """ - config = config or {} - example_config = get_example_config() - # Remove the example channel, that shouldn't be saved to local - example_config.pop('channels') - try: - local_config = get_local_config() - except FileNotFoundError: - # Local config does not yet exist, lets create it - local_config = {} +class WROLPiConfig(ConfigFile): + file_name = 'wrolpi.yaml' + default_config = dict( + download_on_startup=True, + hotspot_on_startup=True, + throttle_on_startup=False, + timezone=str(DEFAULT_TIMEZONE), + wrol_mode=False, + ) - if 'channels' in config and 'channels' in local_config: - del local_config['channels'] + @property + def download_on_startup(self) -> bool: + return self._config['download_on_startup'] - if ('channels' in config or 'channels' in local_config) and 'channels' in example_config: - del example_config['channels'] + @download_on_startup.setter + def download_on_startup(self, value: bool): + self.update({'download_on_startup': value}) - # Create a new config using the values in order of: config -> local_config -> example_config - all_keys = list(config.keys()) + list(local_config.keys()) + list(example_config.keys()) - new_config = {i: config.get(i, local_config.get(i, example_config.get(i))) for i in all_keys} + @property + def hotspot_on_startup(self) -> bool: + return self._config['hotspot_on_startup'] - logger.info(f'Writing config to file: {CONFIG_PATH}') - with open(str(CONFIG_PATH), 'wt') as fh: - yaml.dump(new_config, fh) + @hotspot_on_startup.setter + def hotspot_on_startup(self, value: bool): + self.update({'hotspot_on_startup': value}) + + @property + def throttle_on_startup(self) -> bool: + return self._config['throttle_on_startup'] + + @throttle_on_startup.setter + def throttle_on_startup(self, value: bool): + self.update({'throttle_on_startup': value}) + + @property + def timezone(self) -> str: + return self._config['timezone'] + + @timezone.setter + def timezone(self, value: str): + self.update({'timezone': value}) + + @property + def wrol_mode(self) -> bool: + return self._config['wrol_mode'] + + @wrol_mode.setter + def wrol_mode(self, value: bool): + self.update({'wrol_mode': value}) + + +WROLPI_CONFIG: WROLPiConfig = None +TEST_WROLPI_CONFIG: WROLPiConfig = None + + +def get_config() -> WROLPiConfig: + """Read the global WROLPi yaml config file.""" + global TEST_WROLPI_CONFIG + if isinstance(TEST_WROLPI_CONFIG, WROLPiConfig): + return TEST_WROLPI_CONFIG + + global WROLPI_CONFIG + if isinstance(WROLPI_CONFIG, WROLPiConfig): + return WROLPI_CONFIG + + # Create the config instance once. + WROLPI_CONFIG = WROLPiConfig() + return WROLPI_CONFIG + + +def set_test_config(enable: bool): + global TEST_WROLPI_CONFIG + if enable: + TEST_WROLPI_CONFIG = WROLPiConfig() + else: + TEST_WROLPI_CONFIG = None -@cached(cache=TTLCache(maxsize=1, ttl=30)) def wrol_mode_enabled() -> bool: - """ - Return the boolean value of the `wrol_mode` in the config. - """ - config = get_config() - enabled = config.get('wrol_mode', False) - return bool(enabled) + """Return True if WROL Mode is enabled.""" + return get_config().wrol_mode + + +def set_wrol_mode(enable: bool): + """Enable or disable WROL Mode for all processes. This also updates the config file.""" + get_config().wrol_mode = enable def wrol_mode_check(func): @@ -327,13 +403,6 @@ def remove_whitespace(s: str) -> str: return WHITESPACE.sub('', s) -def remove_dict_value_whitespace(d: Dict) -> Dict: - """ - Remove the whitespace around a the values in a dictionary. Handles if the value isn't a string. - """ - return {k: v.strip() if hasattr(v, 'strip') else v for k, v in d.items()} - - def run_after(after: callable, *args, **kwargs) -> callable: """ Run the `after` function sometime in the future ofter the wrapped function returns. @@ -369,7 +438,6 @@ def run_after(after: callable, *args, **kwargs) -> callable: TEST_MEDIA_DIRECTORY = None -MEDIA_DIRECTORY = None def set_test_media_directory(path): @@ -383,11 +451,13 @@ def set_test_media_directory(path): def get_media_directory() -> Path: - """ - Get the media directory configured in local.yaml. + """Get the media directory. + + This will typically be /opt/wrolpi, or something else from the .env. + + During testing, this function returns TEST_MEDIA_DIRECTORY. """ global TEST_MEDIA_DIRECTORY - global MEDIA_DIRECTORY if PYTEST and not TEST_MEDIA_DIRECTORY: raise ValueError('No test media directory set during testing!!') @@ -395,16 +465,6 @@ def get_media_directory() -> Path: if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path): return TEST_MEDIA_DIRECTORY - if isinstance(MEDIA_DIRECTORY, pathlib.Path): - return MEDIA_DIRECTORY - - config = get_config() - media_directory = config['media_directory'] - media_directory = Path(media_directory) - if not media_directory.is_absolute(): - # Media directory is relative. Assume that is relative to the project directory. - media_directory = PROJECT_DIR / media_directory - MEDIA_DIRECTORY = media_directory.absolute() return MEDIA_DIRECTORY @@ -660,6 +720,7 @@ def escape_file_name(name: str) -> str: def native_only(func: callable): """Wraps a function. Raises NativeOnly if run while Dockerized.""" + @wraps(func) def wrapped(*a, **kw): if DOCKERIZED: @@ -667,3 +728,28 @@ def native_only(func: callable): return func(*a, **kw) return wrapped + + +def read_yaml(path: Path): + with path.open('rt') as fh: + return yaml.load(fh, Loader=yaml.Loader) + + +def recursive_map(obj: Any, func: callable): + """ + Apply `func` to all values of any dictionaries; or items in any list/set/tuple. + + >>> recursive_map({'foo ': ' bar '}, lambda i: i.strip() if hasattr(i, 'strip') else i) + {'foo ': 'bar'} + >>> recursive_map(['foo ', ' bar '], lambda i: i.strip() if hasattr(i, 'strip') else i) + ['foo', 'bar'] + """ + if isinstance(obj, dict): + obj = obj.copy() + for key, value in obj.items(): + obj[key] = recursive_map(value, func) + return obj + elif isinstance(obj, (list, set, tuple)): + type_ = type(obj) + return type_(map(lambda i: recursive_map(i, func), obj)) + return func(obj) diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 74a425a9..cf6c62d0 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -9,12 +9,11 @@ from unittest import mock from uuid import uuid1 import pytest -import yaml from sanic_testing.testing import SanicTestClient from sqlalchemy.engine import Engine, create_engine from sqlalchemy.orm import Session, sessionmaker -from wrolpi.common import set_test_media_directory, Base, get_example_config +from wrolpi.common import set_test_media_directory, Base, set_test_config from wrolpi.dates import set_test_now from wrolpi.db import postgres_engine, get_db_args from wrolpi.downloader import DownloadManager @@ -87,17 +86,14 @@ def test_directory() -> pathlib.Path: @pytest.fixture(autouse=True) -def test_config(): +def test_config(test_directory): """ Create a test config based off the example config. """ - test_config_path = tempfile.NamedTemporaryFile(mode='rt') - with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name), \ - mock.patch('wrolpi.common.CONFIG_PATH', test_config_path.name): - config = get_example_config() - with open(test_config_path.name, 'wt') as fh: - fh.write(yaml.dump(config)) - yield pathlib.Path(test_config_path.name) + config_path = test_directory / 'config/wrolpi.yaml' + set_test_config(True) + yield config_path + set_test_config(False) ROUTES_ATTACHED = False diff --git a/wrolpi/errors.py b/wrolpi/errors.py index 0d5eff62..22a03e9d 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -144,6 +144,10 @@ class NativeOnly(APIError): pass +class HotspotError(APIError): + pass + + error_codes = iter(range(1, 1000)) API_ERRORS = { @@ -317,4 +321,9 @@ API_ERRORS = { 'message': 'This functionality is only supported outside a docker container', 'status': HTTPStatus.BAD_REQUEST, }, + HotspotError: { + 'code': next(error_codes), + 'message': 'Updating/accessing Hotspot encountered an error', + 'status': HTTPStatus.INTERNAL_SERVER_ERROR, + } } diff --git a/wrolpi/root_api.py b/wrolpi/root_api.py index 48987ff6..345070dc 100644 --- a/wrolpi/root_api.py +++ b/wrolpi/root_api.py @@ -15,11 +15,11 @@ from sanic_ext import validate from sanic_ext.extensions.openapi import openapi from wrolpi import admin -from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, save_settings_config, \ - Base, get_media_directory, wrol_mode_check, native_only +from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, Base, get_media_directory, \ + wrol_mode_check, native_only, set_wrol_mode from wrolpi.dates import set_timezone from wrolpi.downloader import download_manager -from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError +from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError, HotspotError from wrolpi.media_path import MediaPath from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse from wrolpi.vars import DOCKERIZED @@ -112,30 +112,63 @@ async def echo(request: Request): @openapi.description('Get WROLPi settings') @openapi.response(HTTPStatus.OK, SettingsResponse) def get_settings(_: Request): - config = dict(get_config()) - return json_response({'config': config, 'version': __version__}) + config = get_config() + + settings = { + 'download_on_startup': config.download_on_startup, + 'hotspot_on_startup': config.hotspot_on_startup, + 'hotspot_status': admin.hotspot_status().name, + 'media_directory': get_media_directory(), + 'throttle_on_startup': config.throttle_on_startup, + 'throttle_status': admin.throttle_status().name, + 'timezone': config.timezone, + 'version': __version__, + 'wrol_mode': config.wrol_mode, + } + return json_response(settings) @root_api.patch('/settings') @openapi.description('Update WROLPi settings') @validate(json=SettingsRequest) -def update_settings(_: Request, data: dict): - if wrol_mode_enabled() and 'wrol_mode' not in data: +def update_settings(_: Request, body: SettingsRequest): + if wrol_mode_enabled() and body.wrol_mode is None: # Cannot update settings while WROL Mode is enabled, unless you want to disable WROL Mode. raise WROLModeEnabled() + if body.wrol_mode is False: + # Disable WROL Mode + set_wrol_mode(False) + return response.empty() + elif body.wrol_mode is True: + # Enable WROL Mode + set_wrol_mode(True) + return response.empty() + try: - if data.get('timezone'): - set_timezone(data['timezone']) + if body.timezone: + set_timezone(body.timezone) except UnknownTimeZoneError: - raise InvalidTimezone(f'Invalid timezone: {data["timezone"]}') + raise InvalidTimezone(f'Invalid timezone: {body.timezone}') - save_settings_config(data) + # Remove any keys with None values, then save the config. + config = {k: v for k, v in body.__dict__.items() if v is not None} + wrolpi_config = get_config() + wrolpi_config.update(config) - if data.get('wrolpi_mode'): + if body.wrol_mode: download_manager.kill() - return response.raw('', HTTPStatus.NO_CONTENT) + if body.hotspot_status is True: + # Turn on Hotspot + if admin.enable_hotspot() is False: + raise HotspotError('Could not turn on hotspot') + elif body.hotspot_status is False: + # Turn off Hotspot + if admin.disable_hotspot() is False: + raise HotspotError('Could not turn off hotspot') + + return response.empty() @root_api.post('/valid_regex') @@ -211,18 +244,6 @@ async def get_downloaders(_: Request): return json_response(ret) -@root_api.get('/hotspot') -@native_only -@openapi.description('Get the status of the hotspot') -async def hotspot(_: Request): - status = admin.hotspot_status() - # Get the name of the enum. - status = status.name - - ret = dict(status=status) - return json_response(ret) - - @root_api.post('/hotspot/on') @openapi.description('Turn on the hotspot') @native_only @@ -243,18 +264,6 @@ async def hotspot_off(_: Request): return response.empty(HTTPStatus.INTERNAL_SERVER_ERROR) -@root_api.get('/throttle') -@openapi.description('Get status of CPU throttling') -@native_only -async def throttle_status(_: Request): - status = admin.throttle_status() - # Get the name of the enum. - status = status.name - - ret = dict(status=status) - return json_response(ret) - - @root_api.post('/throttle/on') @openapi.description('Turn on CPU throttling') @native_only diff --git a/wrolpi/schema.py b/wrolpi/schema.py index 2a471686..2d098150 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -14,9 +14,14 @@ class SettingsResponse: @dataclass class SettingsRequest: - media_directory: str - wrol_mode: bool - timezone: str + download_on_startup: Optional[bool] = None + hotspot_on_startup: Optional[bool] = None + hotspot_status: Optional[bool] = None + media_directory: Optional[str] = None + throttle_on: Optional[bool] = None + throttle_on_startup: Optional[bool] = None + timezone: Optional[str] = None + wrol_mode: Optional[bool] = None @dataclass diff --git a/wrolpi/test/common.py b/wrolpi/test/common.py index efbdd4a5..bb531f69 100644 --- a/wrolpi/test/common.py +++ b/wrolpi/test/common.py @@ -1,4 +1,3 @@ -import json import pathlib import tempfile import unittest @@ -6,18 +5,14 @@ from contextlib import contextmanager from functools import partialmethod from http import HTTPStatus from itertools import zip_longest -from shutil import copyfile -from typing import List, Optional +from typing import List import mock -import websockets -import yaml from requests import Response -from wrolpi.common import EXAMPLE_CONFIG_PATH, get_config, set_test_media_directory, get_media_directory +from wrolpi.common import set_test_media_directory, get_media_directory from wrolpi.conftest import ROUTES_ATTACHED, test_db, test_client # noqa from wrolpi.db import postgres_engine -from wrolpi.vars import PROJECT_DIR TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False) @@ -120,21 +115,11 @@ class ExtendedTestCase(PytestCase, unittest.TestCase): class TestAPI(ExtendedTestCase): def setUp(self) -> None: - self.config_path_patch = mock.patch('wrolpi.vars.CONFIG_PATH', TEST_CONFIG_PATH.name) - self.config_path_patch.start() self.tmp_dir = tempfile.TemporaryDirectory() self.tmp_path = pathlib.Path(self.tmp_dir.name) - set_test_media_directory(self.tmp_dir.name) - # Copy the example config to test against - copyfile(str(EXAMPLE_CONFIG_PATH), TEST_CONFIG_PATH.name) - # Setup the testing video root directory - config = get_config() - config['media_directory'] = str(PROJECT_DIR / 'test') - with open(TEST_CONFIG_PATH.name, 'wt') as fh: - fh.write(yaml.dump(config)) + set_test_media_directory(self.tmp_path) def tearDown(self) -> None: - self.config_path_patch.stop() set_test_media_directory(None) def assertHTTPStatus(self, response: Response, status: int): @@ -145,22 +130,6 @@ class TestAPI(ExtendedTestCase): assertNO_CONTENT = partialmethod(assertHTTPStatus, status=HTTPStatus.NO_CONTENT) -@contextmanager -def wrap_media_directory(path: Optional[pathlib.Path] = None): - cleanup = False - if not path: - path = tempfile.TemporaryDirectory() - cleanup = True - - set_test_media_directory(path.name) - try: - yield - finally: - set_test_media_directory(None) - if cleanup: - path.cleanup() - - @contextmanager def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pathlib.Path: """ @@ -196,17 +165,6 @@ def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pa yield tmp_dir.absolute() -async def get_all_ws_messages(ws) -> List[dict]: - messages = [] - while True: - try: - message = await ws.recv() - except websockets.exceptions.ConnectionClosedOK: - break - messages.append(json.loads(message)) - return messages - - def assert_dict_contains(d1: dict, d2: dict): if hasattr(d1, '__dict__'): d1 = d1.__dict__ diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py index 5b37a51c..f602a253 100644 --- a/wrolpi/test/test_common.py +++ b/wrolpi/test/test_common.py @@ -7,91 +7,50 @@ from decimal import Decimal import pytest -from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \ +from wrolpi.common import insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \ escape_file_name from wrolpi.dates import set_timezone, now from wrolpi.errors import InvalidTimezone -from wrolpi.test.common import build_test_directories, wrap_media_directory +from wrolpi.test.common import build_test_directories -@pytest.mark.parametrize( - 'data,expected', - ( - ( - [dict()], - dict() - ), ( - [dict(a='b')], - dict(a='b') - ), ( - [dict(a='b'), dict(b='c')], - dict(a='b', b='c') - ), ( - [dict(b='c'), dict(a='b')], - dict(a='b', b='c'), - ), ( - [dict(a=dict(b='c'), d='e'), dict(a=(dict(f='g')))], - dict(a=dict(b='c', f='g'), d='e'), - ), ( - [dict(a=dict(b='c'), d='e'), dict(a=(dict(b='g')))], - dict(a=dict(b='c'), d='e'), - ), ( - [dict(a=dict(b='c'), d=[1, 2, 3]), dict(a=(dict(b='g')))], - dict(a=dict(b='c'), d=[1, 2, 3]), - ), ( - [dict(a='b', c=dict(d='e')), dict(a='c', e='f')], - dict(a='b', c=dict(d='e'), e='f') - ), ( - [dict(a='b'), dict(a='c', d='e'), dict(e='f', a='d')], - dict(a='b', d='e', e='f') - ), ( - [dict(a='b'), dict(), dict(e='f', a='d')], - dict(a='b', e='f') - ) - ) -) -def test_combine_dicts(data, expected): - assert combine_dicts(*data) == expected +def test_build_video_directories(test_directory): + structure = [ + 'channel1/vid1.mp4', + ] + with build_test_directories(structure) as tempdir: + assert (tempdir / 'channel1').is_dir() + assert (tempdir / 'channel1/vid1.mp4').is_file() + structure = [ + 'channel2/', + 'channel2.1/channel2.2/', + ] + with build_test_directories(structure) as tempdir: + assert (tempdir / 'channel2').is_dir() + assert (tempdir / 'channel2.1/channel2.2').is_dir() -def test_build_video_directories(): - with wrap_media_directory(): - structure = [ - 'channel1/vid1.mp4', - ] - with build_test_directories(structure) as tempdir: - assert (tempdir / 'channel1').is_dir() - assert (tempdir / 'channel1/vid1.mp4').is_file() + structure = [ + 'channel3/vid1.mp4', + 'channel3/vid2.mp4', + 'channel4/vid1.mp4', + 'channel4/vid1.en.vtt', + 'channel5/', + ] + with build_test_directories(structure) as tempdir: + assert (tempdir / 'channel3/vid1.mp4').is_file() + assert (tempdir / 'channel3').is_dir() + assert (tempdir / 'channel3/vid2.mp4').is_file() + assert (tempdir / 'channel4/vid1.mp4').is_file() + assert (tempdir / 'channel4/vid1.en.vtt').is_file() + assert (tempdir / 'channel5').is_dir() - structure = [ - 'channel2/', - 'channel2.1/channel2.2/', - ] - with build_test_directories(structure) as tempdir: - assert (tempdir / 'channel2').is_dir() - assert (tempdir / 'channel2.1/channel2.2').is_dir() - - structure = [ - 'channel3/vid1.mp4', - 'channel3/vid2.mp4', - 'channel4/vid1.mp4', - 'channel4/vid1.en.vtt', - 'channel5/', - ] - with build_test_directories(structure) as tempdir: - assert (tempdir / 'channel3/vid1.mp4').is_file() - assert (tempdir / 'channel3').is_dir() - assert (tempdir / 'channel3/vid2.mp4').is_file() - assert (tempdir / 'channel4/vid1.mp4').is_file() - assert (tempdir / 'channel4/vid1.en.vtt').is_file() - assert (tempdir / 'channel5').is_dir() - - structure = [ - 'channel6/subdirectory/vid1.mp4', - ] - with build_test_directories(structure) as tempdir: - assert (tempdir / 'channel6/subdirectory').is_dir() - assert (tempdir / 'channel6/subdirectory/vid1.mp4').is_file() + structure = [ + 'channel6/subdirectory/vid1.mp4', + ] + with build_test_directories(structure) as tempdir: + assert (tempdir / 'channel6/subdirectory').is_dir() + assert (tempdir / 'channel6/subdirectory/vid1.mp4').is_file() def test_insert_parameter(): diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 23dc82be..35e0095b 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -2,6 +2,9 @@ import json from http import HTTPStatus from itertools import zip_longest +from mock import mock + +from wrolpi.common import get_config from wrolpi.dates import strptime from wrolpi.db import get_db_session from wrolpi.downloader import Download @@ -154,3 +157,57 @@ class TestRootAPI(TestAPI): recurring_downloads = response.json['recurring_downloads'] for d1, d2 in zip_longest(expected, recurring_downloads): self.assertDictContains(d2, d1) + + +def test_hotspot_settings(test_session, test_client, test_config): + """ + The User can toggle the Hotspot via /settings. The Hotspot can be automatically started on startup. + """ + config = get_config() + assert config.hotspot_on_startup is True + + with mock.patch('wrolpi.root_api.admin') as mock_admin: + # Turning on the hotspot succeeds. + mock_admin.enable_hotspot.return_value = True + request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': True})) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json + mock_admin.enable_hotspot.assert_called_once() + mock_admin.reset_mock() + + # Turning on the hotspot fails. + mock_admin.enable_hotspot.return_value = False + request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': True})) + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, response.json + assert response.json['code'] == 35 + mock_admin.enable_hotspot.assert_called_once() + + # Turning off the hotspot succeeds. + mock_admin.disable_hotspot.return_value = True + request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': False})) + assert response.status_code == HTTPStatus.NO_CONTENT, response.json + mock_admin.disable_hotspot.assert_called_once() + mock_admin.reset_mock() + + # Turning off the hotspot succeeds. + mock_admin.disable_hotspot.return_value = False + request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': False})) + assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, response.json + assert response.json['code'] == 35 + mock_admin.disable_hotspot.assert_called_once() + + config = get_config() + assert config.hotspot_on_startup is True + + +def test_throttle_toggle(test_session, test_client, test_config): + with mock.patch('wrolpi.admin.subprocess') as mock_subprocess: + mock_subprocess.check_output.side_effect = [ + b'wlan0: unavailable', + b'The governor "ondemand" may decide ', + ] + request, response = test_client.get('/api/settings') + + # Throttle is off by default. + assert response.status_code == HTTPStatus.OK + assert response.json['throttle_on_startup'] is False + assert response.json['throttle_status'] == 'ondemand' diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 4905aba9..bf97f020 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -1,19 +1,31 @@ import os -import pathlib import sys +from pathlib import Path -PROJECT_DIR: pathlib.Path = pathlib.Path(__file__).parents[1].absolute() -CONFIG_DIR: pathlib.Path = PROJECT_DIR / 'config' -CONFIG_PATH: pathlib.Path = CONFIG_DIR / 'local.yaml' -EXAMPLE_CONFIG_PATH: pathlib.Path = CONFIG_DIR / 'example.yaml' -MODULES_DIR: pathlib.Path = PROJECT_DIR / 'modules' -PUBLIC_HOST = os.environ.get('PUBLIC_HOST') -PUBLIC_PORT = os.environ.get('PUBLIC_PORT') +from dotenv import load_dotenv +# The root of WROLPi. Typically, /opt/wrolpi +PROJECT_DIR: Path = Path(__file__).parents[1].absolute() + +# Load any .env file. This may contain our MEDIA_DIRECTORY. +load_dotenv(PROJECT_DIR / '.env') + +# Special environment variable set in the docker/api/Dockerfile. DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False # tests are running PYTEST = 'pytest' in sys.modules +# Get the media directory from the environment. +DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi') +MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY)) +if not MEDIA_DIRECTORY.is_dir() and not PYTEST: + print(f'Media directory does not exist! {MEDIA_DIRECTORY}') + +CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config' +MODULES_DIR: Path = PROJECT_DIR / 'modules' +PUBLIC_HOST = os.environ.get('PUBLIC_HOST') +PUBLIC_PORT = os.environ.get('PUBLIC_PORT') + DEFAULT_TIMEZONE_STR = 'America/Boise' LAST_MODIFIED_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT' DATE_FORMAT = '%Y-%M-%d' @@ -28,3 +40,10 @@ DB_PORT = int(os.environ.get('DB_PORT', 5432)) DB_NAME = os.environ.get('DB_NAME', 'wrolpi') DB_USER = os.environ.get('DB_USER', 'wrolpi') DB_PASSWORD = os.environ.get('DB_PASSWORD', 'wrolpi') + +EXAMPLE_CONFIG = { + 'hotspot_on_startup': True, + 'throttle_on_startup': False, + 'timezone': 'America/Boise', + 'wrol_mode': False, +}