rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 10:51:36 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Squashing config rework.

Browse source
This commit is contained in:
Roland 2022-02-16 21:57:27 -07:00
parent 5425142adb
commit 08065155cf
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
38 changed files with 1046 additions and 633 deletions
Download patch file Download diff file Expand all files Collapse all files

6
.gitignore vendored
View file

@ -109,10 +109,8 @@ venv.bak/
test/videos
test/archive
# local configs
config/local.yaml
config/local.yaml.bak
config/inventories.yaml
# testing configs
test/config/*
app/package-lock.json
modules/node_modules/

66
app/src/api.js
View file

@ -1,4 +1,12 @@
import {API_URI, ARCHIVES_API, DEFAULT_LIMIT, OTP_API, secondsToDateTime, VIDEOS_API} from "./components/Common";
import {
API_URI,
ARCHIVES_API,
DEFAULT_LIMIT,
emptyToNull,
OTP_API,
secondsToDateTime,
VIDEOS_API
} from "./components/Common";
import {toast} from 'react-semantic-toasts';
function timeoutPromise(ms, promise) {
@ -87,12 +95,13 @@ let apiPatch = async (url, body) => {
return await apiCall(url, 'PATCH', body)
};
export async function updateChannel(link, channel) {
channel = emptyToNull(channel);
return await apiPut(`${VIDEOS_API}/channels/${link}`, channel);
}
export async function createChannel(channel) {
channel = emptyToNull(channel);
return await apiPost(`${VIDEOS_API}/channels`, channel);
}
@ -160,14 +169,13 @@ export async function getDirectories(search_str) {
return [];
}
export async function getConfig() {
export async function getSettings() {
let response = await apiGet(`${API_URI}/settings`);
let data = await response.json();
return data['config'];
return await response.json();
}
export async function saveConfig(config) {
return await apiPatch(`${API_URI}/settings`, config);
export async function saveSettings(settings) {
return await apiPatch(`${API_URI}/settings`, settings);
}
export async function getDownloads() {
@ -291,10 +299,12 @@ export async function getItems(inventoryId) {
}
export async function saveItem(inventoryId, item) {
item = emptyToNull(item);
return await apiPost(`${API_URI}/inventory/${inventoryId}/item`, item);
}
export async function updateItem(itemId, item) {
item = emptyToNull(item);
return await apiPut(`${API_URI}/inventory/item/${itemId}`, item);
}
@ -438,25 +448,8 @@ export async function getVersion() {
}
export async function getHotspotStatus() {
let response = await apiGet(`${API_URI}/hotspot`);
if (response.status === 200) {
let {status} = await response.json();
return status;
} else {
let code = (await response.json())['code'];
if (response.status === 400 && code === 34) {
// Must not be running on a RPi. Don't warn the user.
return null;
} else {
toast({
type: 'error',
title: "Can't get Hotspot status",
description: 'Failed to get Hotspot status. Check client and server logs.',
time: 5000,
});
}
}
return null;
let response = await getSettings();
return response['hotspot_status'];
}
export async function setHotspot(on) {
@ -489,25 +482,8 @@ export async function setHotspot(on) {
}
export async function getThrottleStatus() {
let response = await apiGet(`${API_URI}/throttle`);
if (response.status === 200) {
let {status} = await response.json();
return status;
} else {
let code = (await response.json())['code'];
if (response.status === 400 && code === 34) {
// Must not be running on a RPi. Don't warn the user.
return null;
} else {
toast({
type: 'error',
title: "Can't get throttle status",
description: 'Failed to get CPU Throttle status. Check client and server logs.',
time: 5000,
});
}
}
return null;
let response = await getSettings();
return response['throttle_status'];
}
export async function setThrottle(on) {

111
app/src/components/Admin.js
View file

@ -11,7 +11,7 @@ import {
Placeholder,
Table
} from "semantic-ui-react";
import {getConfig, getDownloads, killDownload, postDownload, saveConfig} from "../api";
import {getDownloads, getSettings, killDownload, postDownload, saveSettings} from "../api";
import TimezoneSelect from 'react-timezone-select';
import {
DisableDownloadsToggle,
@ -33,42 +33,51 @@ class Settings extends React.Component {
this.state = {
disabled: false,
ready: false,
validTimezone: true,
timezone: '',
download_on_startup: null,
hotspot_on_startup: null,
hotspot_status: null,
throttle_on_startup: null,
throttle_status: null,
timezone: {timezone: null},
}
this.mediaDirectory = React.createRef();
this.handleSubmit = this.handleSubmit.bind(this);
}
async componentDidMount() {
let config = await getConfig();
const settings = await getSettings();
this.setState({
ready: true,
disabled: config.wrol_mode,
timezone: config.timezone || '',
disabled: settings.wrol_mode,
download_on_startup: settings.download_on_startup,
hotspot_on_startup: settings.hotspot_on_startup,
hotspot_status: settings.hotspot_status,
throttle_on_startup: settings.throttle_on_startup,
throttle_status: settings.throttle_status,
timezone: {value: settings.timezone, label: settings.timezone},
});
this.mediaDirectory.current.value = config.media_directory;
}
async handleSubmit(e) {
e.preventDefault();
let config = {
media_directory: this.mediaDirectory.current.value,
let settings = {
download_on_startup: this.state.download_on_startup,
hotspot_on_startup: this.state.hotspot_on_startup,
throttle_on_startup: this.state.throttle_on_startup,
timezone: this.state.timezone.value,
};
this.setState({validTimezone: true});
let response = await saveConfig(config);
if (response.status !== 200) {
let json = await response.json();
if (json.api_error === 'Invalid timezone') {
this.setState({validTimezone: false});
}
}
console.log(settings);
await saveSettings(settings);
}
handleTimezoneChange = async (timezone) => {
this.setState({timezone});
this.setState({timezone: timezone});
}
handleInputChange = async (e, name, checked) => {
e.preventDefault();
this.setState({[name]: checked});
}
render() {
@ -76,6 +85,8 @@ class Settings extends React.Component {
return <Loader active inline='centered'/>
}
let {disabled, download_on_startup, hotspot_on_startup, throttle_on_startup, timezone} = this.state;
return (
<Container fluid>
<WROLModeMessage content='Settings are disabled because WROL Mode is enabled.'/>
@ -83,46 +94,52 @@ class Settings extends React.Component {
<HotspotToggle/>
<br/>
<ThrottleToggle style={{marginTop: '0.5em'}}/>
<Divider/>
<p>
The global settings for your server.
</p>
<Header as='h3'>
The settings for your WROLPi.
</Header>
<p className="text-muted">
When saved, these will be written to your <i>local.yaml</i> file. Storing these settings in your
local configuration file (rather than the database) allows your WROLPi to be rebuilt and save your
settings.
Any changes will be written to <i>config/wrolpi.yaml</i>.
</p>
<Divider/>
<Form id="settings" onSubmit={this.handleSubmit}>
<Checkbox toggle
style={{marginTop: '0.5em', marginBottom: '0.5em'}}
label='Download on Startup'
disabled={disabled || download_on_startup === null}
checked={download_on_startup === true}
onChange={(e, d) => this.handleInputChange(e, 'download_on_startup', d.checked)}
/>
<Form.Field>
<label>Media Directory</label>
<input
disabled={this.state.disabled}
type="text"
placeholder="/some/absolute/path"
ref={this.mediaDirectory}
/>
</Form.Field>
<br/>
<Checkbox toggle
style={{marginTop: '0.5em', marginBottom: '0.5em'}}
label='WiFi Hotspot on Startup'
disabled={disabled || hotspot_on_startup === null}
checked={hotspot_on_startup === true}
onChange={(e, d) => this.handleInputChange(e, 'hotspot_on_startup', d.checked)}
/>
<p>
<b>This directory must already exist</b>.
The directory in which your media will be stored. Typically, this will be some external
drive like <i>/media/wrolpi</i>.
</p>
<br/>
<Divider/>
<Checkbox toggle
style={{marginTop: '0.5em', marginBottom: '0.5em'}}
label='CPU Power-save on Startup'
disabled={disabled || throttle_on_startup === null}
checked={throttle_on_startup === true}
onChange={(e, d) => this.handleInputChange(e, 'throttle_on_startup', d.checked)}
/>
<Form.Field>
<label>Timezone</label>
<TimezoneSelect
value={this.state.timezone}
value={timezone}
onChange={this.handleTimezoneChange}
/>
</Form.Field>
<Button color="blue" type="submit" disabled={this.state.disabled}>
<Button color="blue" type="submit" disabled={disabled}>
Save
</Button>
@ -142,8 +159,8 @@ class WROLMode extends React.Component {
}
async componentDidMount() {
let config = await getConfig();
this.setState({ready: true, WROLMode: config.wrol_mode});
let settings = await getSettings();
this.setState({ready: true, WROLMode: settings.wrol_mode});
}
toggleWROLMode = async () => {
@ -151,7 +168,7 @@ class WROLMode extends React.Component {
let config = {
wrol_mode: !this.state.WROLMode,
}
await saveConfig(config);
await saveSettings(config);
this.setState({disabled: !this.state.WROLMode, WROLMode: !this.state.WROLMode});
}

7
app/src/components/Channels.js
View file

@ -18,8 +18,7 @@ import {
downloadChannel,
getChannel,
getChannels,
getConfig,
getDirectories,
getDirectories, getSettings,
refreshChannel,
updateChannel,
validateRegex
@ -57,11 +56,11 @@ class DirectoryInput extends React.Component {
}
async componentDidMount() {
let global_config = await getConfig();
let settings = await getSettings();
let directories = await getDirectories(this.state.directory);
this.setState({
directories,
mediaDirectory: global_config.media_directory,
mediaDirectory: settings.media_directory,
directory: this.props.value,
});
}

17
app/src/components/Common.js
View file

@ -4,7 +4,7 @@ import {Link, NavLink} from "react-router-dom";
import Button from "semantic-ui-react/dist/commonjs/elements/Button";
import Grid from "semantic-ui-react/dist/commonjs/collections/Grid";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {getConfig} from "../api";
import {getConfig, getSettings} from "../api";
import {useDownloaders, useHotspot, useThrottle} from "../hooks/customHooks";
export const API_URI = `http://${window.location.host}/api`;
@ -560,8 +560,8 @@ export const useWROLMode = () => {
const [enabled, setEnabled] = useState(false);
const fetchStatus = async () => {
let config = await getConfig();
setEnabled(config['wrol_mode']);
let settings = await getSettings();
setEnabled(settings.wrol_mode);
}
useEffect(() => {
@ -674,3 +674,14 @@ export function DisableDownloadsToggle() {
onChange={(e, data) => setDownloads(data.checked)}
/>;
}
export function emptyToNull(obj) {
const keys = Object.keys(obj);
for (let i = 0; i < keys.length; i++) {
const key = keys[i];
if (obj[key] === '') {
obj[key] = null;
}
}
return obj;
}

27
config/example.yaml
View file

@ -1,27 +0,0 @@
# This is an example config for your WROLPi. Do NOT modify this file, rather, copy this file to local.yaml and
# modify that file.
media_directory: /media/wrolpi # this should be an absolute directory, for example, /media/wrolpi
downloader:
# Most of these names were decided by YT-DLP project, we pass these to any calls to that project.
continue_dl: true
dateafter: 19900101
file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s"
format: 22/135/18/43
nooverwrites: true
quiet: false
writeautomaticsub: true
writeinfojson: true
writesubtitles: true
writethumbnail: true
youtube_include_dash_manifest: false
channels:
# These are channels or playlists that will be downloaded. You can configure them in the UI, or in your local.yaml.
wrolpi:
name: WROLPi # the pretty name for the channel
url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos
directory: videos/wrolpi # relative to `media_directory`
download_frequency: 604800
timezone: America/Boise

24
main.py
View file

@ -8,8 +8,8 @@ import sys
import pytz
from sanic import Sanic
from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent
from wrolpi.common import logger, get_config, wrol_mode_enabled, import_modules, check_media_directory
from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, after_startup, limit_concurrent, admin
from wrolpi.common import logger, get_config, import_modules, check_media_directory, set_wrol_mode
from wrolpi.dates import set_timezone
from wrolpi.downloader import download_manager
from wrolpi.vars import PROJECT_DIR, DOCKERIZED
@ -89,10 +89,15 @@ async def main(loop):
# Set the Timezone
config = get_config()
if 'timezone' in config:
tz = pytz.timezone(config['timezone'])
if config.timezone:
tz = pytz.timezone(config.timezone)
set_timezone(tz)
if config.hotspot_on_startup:
admin.enable_hotspot()
if config.throttle_on_startup:
admin.throttle_cpu_on()
check_media_directory()
# Import the API in every module. Each API should attach itself to `root_api`.
@ -131,12 +136,15 @@ async def set_log_level(args):
@after_startup
@limit_concurrent(1)
def periodic_downloads(app: Sanic, loop):
"""
A simple function that perpetually calls downloader_manager.do_downloads() after sleeping.
"""
if wrol_mode_enabled():
"""A simple function that perpetually calls downloader_manager.do_downloads() after sleeping."""
config = get_config()
if config.wrol_mode:
logger.warning(f'Not starting download worker because WROL Mode is enabled.')
return
if config.download_on_startup is False:
logger.warning(f'Not starting download worker because Downloads are disabled on startup.')
return
logger.info('Starting download manager.')
# Set all downloads to new.

1
media_directory Normal file
View file

@ -0,0 +1 @@
/media/wrolpi

10
modules/files/lib.py
View file

@ -5,7 +5,6 @@ from pathlib import Path
from typing import List
from uuid import uuid4
import magic
import psycopg2
from sqlalchemy.orm import Session
@ -16,6 +15,13 @@ from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidFile
from wrolpi.vars import PYTEST
try:
import magic
have_magic = True
except ImportError:
logger.error('Could not import magic. Is libmagic installed?')
have_magic = False
logger = logger.getChild(__name__)
__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search']
@ -96,7 +102,7 @@ def upsert_file(path: Path, session: Session) -> File:
if not file:
file = File(path=path)
session.add(file)
if not file.mimetype:
if not file.mimetype and have_magic:
file.mimetype = magic.from_file(str(path), mime=True)
if not file.title:
file.title = split_file_name(path)

26
modules/inventory/api.py
View file

@ -5,7 +5,7 @@ from sanic.request import Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import run_after, remove_dict_value_whitespace
from wrolpi.common import run_after, recursive_map
from wrolpi.errors import ValidationError
from wrolpi.root_api import get_blueprint, json_response
from . import common, inventory, schema
@ -48,9 +48,8 @@ def inventory_get(_: Request, inventory_id: int):
)
@validate(schema.InventoryPostRequest)
@run_after(common.save_inventories_file)
def post_inventory(_: Request, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def post_inventory(_: Request, body: schema.InventoryPostRequest):
data = remove_whitespace(body.__dict__)
inventory.save_inventory(data)
return response.empty(HTTPStatus.CREATED)
@ -63,9 +62,8 @@ def post_inventory(_: Request, data: dict):
)
@validate(schema.InventoryPutRequest)
@run_after(common.save_inventories_file)
def put_inventory(_: Request, inventory_id: int, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def put_inventory(_: Request, inventory_id: int, body: schema.InventoryPutRequest):
data = remove_whitespace(body.__dict__)
inventory.update_inventory(inventory_id, data)
return response.empty()
@ -93,9 +91,8 @@ def items_get(_: Request, inventory_id: int):
)
@validate(schema.ItemPostRequest)
@run_after(common.save_inventories_file)
def post_item(_: Request, inventory_id: int, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def post_item(_: Request, inventory_id: int, body: schema.ItemPostRequest):
data = remove_whitespace(body.__dict__)
inventory.save_item(inventory_id, data)
return response.empty()
@ -108,9 +105,8 @@ def post_item(_: Request, inventory_id: int, data: dict):
)
@validate(schema.ItemPutRequest)
@run_after(common.save_inventories_file)
def put_item(_: Request, item_id: int, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def put_item(_: Request, item_id: int, body: schema.ItemPutRequest):
data = remove_whitespace(body.__dict__)
inventory.update_item(item_id, data)
return response.empty()
@ -131,3 +127,7 @@ def item_delete(_: Request, item_ids: str):
inventory.delete_items(item_ids)
return response.empty()
def remove_whitespace(obj):
return recursive_map(obj, lambda i: i.strip() if hasattr(i, 'strip') else i)

104
modules/inventory/common.py
View file

@ -3,14 +3,12 @@ from functools import partial
from pathlib import Path
from typing import List, Tuple
import yaml
from pint import Quantity
from wrolpi import before_startup
from wrolpi.common import logger, Base
from wrolpi.common import logger, Base, ConfigFile
from wrolpi.db import get_db_session
from wrolpi.errors import NoInventories, InventoriesVersionMismatch
from wrolpi.vars import CONFIG_DIR
from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \
get_inventories_version
from .models import Inventory, Item
@ -98,23 +96,65 @@ def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quanti
def cleanup_quantity(quantity: Quantity) -> Quantity:
"""
Remove trailing zeros from a Quantity.
"""
"""Remove trailing zeros from a Quantity."""
num, unit = quantity_to_tuple(quantity)
num = round(num, UNIT_PRECISION)
num = str(num).rstrip('0').rstrip('.')
return Decimal(num) * unit
DEFAULT_SAVE_PATH = CONFIG_DIR / 'inventories.yaml'
class InventoriesConfig(ConfigFile):
file_name = 'inventories.yaml'
default_config = dict(
inventories=[],
version=1,
)
@property
def inventories(self) -> dict:
return self._config['inventories']
@inventories.setter
def inventories(self, value: dict):
self.update({'inventories': value})
@property
def version(self) -> int:
return self._config['version']
@version.setter
def version(self, value: int):
self.update({'version': value})
def save_inventories_file(path: str = None):
"""
Write all inventories and their respective items to a YAML file.
"""
path: Path = Path(path) if path else DEFAULT_SAVE_PATH
INVENTORIES_CONFIG: InventoriesConfig = None
TEST_INVENTORIES_CONFIG: InventoriesConfig = None
def get_inventories_config():
global TEST_INVENTORIES_CONFIG
if isinstance(TEST_INVENTORIES_CONFIG, ConfigFile):
return TEST_INVENTORIES_CONFIG
global INVENTORIES_CONFIG
if isinstance(INVENTORIES_CONFIG, ConfigFile):
return INVENTORIES_CONFIG
INVENTORIES_CONFIG = InventoriesConfig()
return INVENTORIES_CONFIG
def set_test_inventories_config(enabled: bool):
global TEST_INVENTORIES_CONFIG
if enabled:
TEST_INVENTORIES_CONFIG = InventoriesConfig()
else:
TEST_INVENTORIES_CONFIG = None
def save_inventories_file():
"""Write all inventories and their respective items to a YAML file."""
config = get_inventories_config()
inventories = []
for inventory in get_inventories():
@ -124,44 +164,22 @@ def save_inventories_file(path: str = None):
raise NoInventories('No Inventories are in the database!')
with increment_inventories_version() as version:
if path.is_file():
# Check that we aren't overwriting our inventories with empty inventories.
with open(path, 'rt') as fh:
old = yaml.load(fh, Loader=yaml.Loader)
if old and not inventories:
raise FileExistsError(f'Refusing to overwrite non-empty inventories.yaml with empty inventories.'
f' {path}')
if config.version and config.version > version:
raise InventoriesVersionMismatch(
f'Inventories config version is {config.version} but DB version is {get_inventories_version()}')
if old and old.get('version') > version:
raise InventoriesVersionMismatch(
f'Inventories config version is {old["version"]} but DB version is {get_inventories_version()}')
with open(path, 'wt') as fh:
contents = dict(
version=version,
inventories=inventories,
)
yaml.dump(contents, fh)
config.inventories = inventories
config.version = version
config.save()
@before_startup
def import_inventories_file(path: str = None):
path: Path = Path(path) if path else DEFAULT_SAVE_PATH
if not path.is_file():
logger.warning(f'No inventories config file at {path}')
return
with open(path, 'rt') as fh:
contents = yaml.load(fh, Loader=yaml.Loader)
if not contents or 'inventories' not in contents:
raise ValueError('Inventories file does not contain the expected "inventories" list.')
def import_inventories_file():
config = get_inventories_config()
inventories = get_inventories()
inventories_names = {i.name for i in inventories}
new_inventories = [i for i in contents['inventories'] if i['name'] not in inventories_names]
new_inventories = [i for i in config.inventories if i['name'] not in inventories_names]
with get_db_session(commit=True) as session:
for inventory in new_inventories:
items = inventory['items']

5
modules/inventory/conftest.py
View file

@ -1,6 +1,7 @@
import pytest
from modules.inventory import Inventory, Item, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES
from modules.inventory.common import set_test_inventories_config
@pytest.fixture
@ -23,4 +24,6 @@ def init_test_inventory(test_session):
inventory = test_session.query(Inventory).filter_by(name='Food Storage').one()
test_session.commit()
return inventory
set_test_inventories_config(True)
yield inventory
set_test_inventories_config(False)

2
modules/inventory/inventory.py
View file

@ -59,7 +59,7 @@ DEFAULT_INVENTORIES = [
sort_categories = partial(sorted, key=itemgetter(1, 0))
def get_inventories() -> List[Base]:
def get_inventories() -> List[Inventory]:
with get_db_session() as session:
inventories = session.query(Inventory).filter(
Inventory.deleted_at == None, # noqa

47
modules/inventory/schema.py
View file

@ -1,36 +1,51 @@
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Union, Optional
from wrolpi.dates import from_timestamp
@dataclass
class ItemPostRequest:
brand: str
name: str
item_size: Decimal
item_size: Union[str, Decimal]
unit: str
count: Decimal
count: Union[str, Decimal]
category: str
subcategory: str
expiration_date: date
expiration_date: Optional[float]
def __post_init__(self):
self.item_size = Decimal(self.item_size) if self.item_size else None
self.count = Decimal(self.count) if self.count else None
self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None
@dataclass
class ItemPutRequest:
id: int # Ignored in favor of URL
brand: str
name: str
item_size: Decimal
unit: str
count: Decimal
serving: Decimal
category: str
subcategory: str
expiration_date: date
purchase_date: date
created_at: datetime
deleted_at: datetime
brand: Optional[str]
name: Optional[str]
item_size: Union[str, Decimal]
unit: Optional[str]
count: Union[str, Decimal]
serving: Union[str, Decimal, None]
category: Optional[str]
subcategory: Optional[str]
expiration_date: Optional[float]
purchase_date: Optional[float]
inventory_id: int
created_at: Optional[float]
deleted_at: Optional[float]
def __post_init__(self):
self.item_size = Decimal(self.item_size) if self.item_size else None
self.count = Decimal(self.count) if self.count else None
self.serving = Decimal(self.serving) if self.serving else None
self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None
self.created_at = from_timestamp(self.created_at) if self.created_at else None
self.deleted_at = from_timestamp(self.deleted_at) if self.deleted_at else None
@dataclass

45
modules/inventory/test/test_api.py
View file

@ -1,3 +1,4 @@
import json
from http import HTTPStatus
from modules.inventory import Item
@ -49,3 +50,47 @@ def test_delete_item(test_session, init_test_inventory, test_client):
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
test_session.refresh(item)
assert item.deleted_at
def test_item_api(test_session, init_test_inventory, test_client):
"""An Item can be added to an Inventory."""
inventory_id = init_test_inventory.id
item = {'brand': 'Salty', 'name': 'Salt', 'item_size': '1', 'unit': 'lbs', 'count': '5',
'category': 'cooking ingredients', 'subcategory': 'salt', 'expiration_date': None}
request, response = test_client.post(f'/api/inventory/{inventory_id}/item', content=json.dumps(item))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
# Get the Item we just created.
request, response = test_client.get(f'/api/inventory/{inventory_id}/item')
assert response.status_code == HTTPStatus.OK
# assert item == response.json['items'][0]
item = response.json['items'][0]
expiration_dates = (12345.12345, None)
for expiration_date in expiration_dates:
item['expiration_date'] = expiration_date
request, response = test_client.put(f'/api/inventory/item/{item["id"]}', content=json.dumps(item))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
def test_inventory_api(test_session, test_client):
"""An Inventory can be created."""
inventory = dict(name='foo')
request, response = test_client.post('/api/inventory', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.CREATED, response.status_code
request, response = test_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
inventory = response.json['inventories'][0]
inventory_id = inventory['id']
inventory = dict(name='new name')
request, response = test_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = test_client.delete(f'/api/inventory/{inventory_id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = test_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
assert len(response.json['inventories']) == 0

137
modules/inventory/test/test_inventory.py
View file

@ -1,5 +1,4 @@
import os
import tempfile
from decimal import Decimal
from decimal import Decimal
from itertools import zip_longest
from typing import List, Iterable
@ -15,7 +14,7 @@ from wrolpi.errors import NoInventories, InventoriesVersionMismatch
from wrolpi.test.common import PytestCase
from .. import init
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config
from ..inventory import unit_registry, \
get_inventories, save_inventory, update_inventory, \
delete_inventory, get_categories
@ -189,75 +188,6 @@ class TestInventory(PytestCase):
for idx, (i, j) in enumerate(zip_longest(inventory, expected)):
self.assertEqual(i, j, f'named inventory {idx} is not equal')
def test_inventories_file(self, test_session):
self.prepare()
with tempfile.NamedTemporaryFile() as tf:
# Can't import an empty file.
self.assertRaises(ValueError, import_inventories_file, tf.name)
save_inventories_file(tf.name)
# Clear out the DB so the import will be tested
with get_db_session(commit=True) as session:
session.query(Item).delete()
session.query(Inventory).delete()
import_inventories_file(tf.name)
inventories = get_inventories()
self.assertEqual(len(inventories), 1)
# ID has increased because we did not reset the sequence when deleting from the table.
self.assertEqual(inventories[0].id, 2)
self.assertEqual(inventories[0].name, 'Food Storage')
# All items in the DB match those in the test list, except for the "deleted" item.
non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None]
self.assertEqual(len(non_deleted_items), len(TEST_ITEMS) - 1)
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
self.assertEqual(test_items, db_items)
def test_inventories_version(self, test_session):
"""
You can't save over a newer version of an inventory.
"""
self.prepare()
with tempfile.NamedTemporaryFile(delete=False) as tf:
pass
try:
with open(tf.name, 'rt') as fh:
# Version is set to 1 on first save.
save_inventories_file(tf.name)
inventories_contents = yaml.load(fh, Loader=yaml.Loader)
self.assertEqual(inventories_contents['version'], 1)
with open(tf.name, 'rt') as fh:
# Version is incremented when saving.
save_inventories_file(tf.name)
inventories_contents = yaml.load(fh, Loader=yaml.Loader)
self.assertEqual(inventories_contents['version'], 2)
with open(tf.name, 'wt') as fh:
# Version is greater than what will be saved.
inventories_contents['version'] = 5
yaml.dump(inventories_contents, fh)
self.assertRaises(InventoriesVersionMismatch, save_inventories_file, tf.name)
finally:
os.unlink(tf.name)
def test_no_inventories(self, test_session, tmp_path):
"""
Can't save empty inventories.
"""
try:
save_inventories_file(tmp_path.name)
except NoInventories:
pass
def quantity_to_string(quantity: Quantity) -> str:
quantity = cleanup_quantity(quantity)
@ -333,3 +263,66 @@ length = gallon.dimensionality
def test_sum_by_key(items, expected):
items = [Item(**i) for i in items]
assert sum_by_key(items, lambda i: (i.category,)) == expected
def test_no_inventories(test_session, test_directory):
"""Can't save empty inventories."""
try:
save_inventories_file()
except NoInventories:
pass
def test_inventories_version(test_session, test_directory, init_test_inventory):
"""You can't save over a newer version of an inventory."""
for item in TEST_ITEMS:
item = Item(**item)
test_session.add(item)
test_session.commit()
# Version is set to 1 on first save.
save_inventories_file()
config = get_inventories_config()
assert config.version == 1
# Version is incremented when saving.
save_inventories_file()
config = get_inventories_config()
assert config.version == 2
# Version is greater than what will be saved.
with config.get_file().open('wt') as fh:
config._config['version'] = 5
yaml.dump(config._config, fh)
with pytest.raises(InventoriesVersionMismatch):
save_inventories_file()
def test_inventories_config(test_session, test_directory, init_test_inventory):
for item in TEST_ITEMS:
item = Item(**item)
test_session.add(item)
test_session.commit()
# Save the Inventories/Items that were created above.
save_inventories_file()
# Clear out the DB so the import will be tested
test_session.query(Item).delete()
test_session.query(Inventory).delete()
test_session.commit()
import_inventories_file()
inventories = get_inventories()
assert len(inventories) == 1
# ID has increased because we did not reset the sequence when deleting from the table.
assert inventories[0].id == 2
assert inventories[0].name == 'Food Storage'
# All items in the DB match those in the test list, except for the "deleted" item.
non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None]
assert len(non_deleted_items) == len(TEST_ITEMS) - 1
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
assert test_items == db_items

8
modules/otp/api.py
View file

@ -19,8 +19,8 @@ bp = get_blueprint('OTP', '/api/otp')
body=schema.EncryptOTPRequest,
)
@validate(schema.EncryptOTPRequest)
async def post_encrypt_otp(_: Request, data: dict):
data = lib.encrypt_otp(data['otp'], data['plaintext'])
async def post_encrypt_otp(_: Request, body: schema.EncryptOTPRequest):
data = lib.encrypt_otp(body.otp, body.plaintext)
return response.json(data)
@ -30,8 +30,8 @@ async def post_encrypt_otp(_: Request, data: dict):
body=schema.DecryptOTPRequest,
)
@validate(schema.DecryptOTPRequest)
async def post_decrypt_otp(_: Request, data: dict):
data = lib.decrypt_otp(data['otp'], data['ciphertext'])
async def post_decrypt_otp(_: Request, body: schema.DecryptOTPRequest):
data = lib.decrypt_otp(body.otp, body.ciphertext)
return response.json(data)

29
modules/otp/test/test_api.py Normal file
View file

@ -0,0 +1,29 @@
import json
from http import HTTPStatus
def test_encrypt(test_client):
body = dict(otp='asdf', plaintext='asdf')
request, response = test_client.post('/api/otp/encrypt_otp', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json == dict(
ciphertext='AAGK',
otp='ASDF',
plaintext='ASDF',
)
def test_decrypt(test_client):
body = dict(otp='asdf', ciphertext='aagk')
request, response = test_client.post('/api/otp/decrypt_otp', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json == dict(
ciphertext='AAGK',
otp='ASDF',
plaintext='ASDF',
)
def test_get_pdf(test_client):
request, response = test_client.get('/api/otp/pdf')
assert response.status_code == HTTPStatus.OK

2
modules/videos/api.py
View file

@ -12,11 +12,11 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \
from wrolpi.common import logger
from wrolpi.root_api import add_blueprint, json_response
from wrolpi.schema import JSONErrorResponse
from . import lib, schema
from .channel import lib as channel_lib
from .channel.api import channel_bp
from .video import lib as video_lib
from .video.api import video_bp
from . import lib, schema
content_bp = Blueprint('VideoContent', '/api/videos')
bp = Blueprint('Videos', '/api/videos').group(

2
modules/videos/channel/api.py
View file

@ -72,7 +72,7 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@validate(schema.ChannelPutRequest)
@wrol_mode_check
def channel_update(_: Request, link: str, body: schema.ChannelPutRequest):
channel = lib.update_channel(body.__dict__, link) # TODO don't use the dataclass as a dict.
channel = lib.update_channel(body.__dict__, link)
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.link}'})

20
modules/videos/common.py
View file

@ -14,7 +14,7 @@ from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.common import logger, get_config, iterify, get_media_directory, \
from wrolpi.common import logger, iterify, get_media_directory, \
minimize_dict, api_param_limiter
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
@ -36,19 +36,6 @@ VIDEO_QUERY_MAX_LIMIT = 100
# These are the supported video formats. These are in order of their preference.
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
def load_downloader_config() -> dict:
config = get_config()
downloader = config['downloader']
return downloader
def load_channels_config() -> dict:
config = get_config()
channels = config['channels']
return channels
video_limiter = api_param_limiter(100)
@ -81,8 +68,9 @@ def import_videos_config():
"""Import channel settings to the DB. Existing channels will be updated."""
logger.info('Importing videos config')
try:
config = get_config()
config, favorites = config['channels'], config.get('favorites', {})
from .lib import get_channels_config
config = get_channels_config()
config, favorites = config.channels, config.favorites
with get_db_session(commit=True) as session:
for link in config:

8
modules/videos/configs/channels.yaml Normal file
View file

@ -0,0 +1,8 @@
# This is an example config. Do not edit this file. Edit the config in your media directory!
channels:
wrolpi:
name: WROLPi
url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos
directory: videos/wrolpi
download_frequency: 604800
favorites: [ ]

12
modules/videos/configs/downloader.yaml Normal file
View file

@ -0,0 +1,12 @@
# Most of these names were decided by YT-DLP project, we pass these to any calls to that project.
continue_dl: true
dateafter: 19900101
file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s"
format: 22/135/18/43
nooverwrites: true
quiet: false
writeautomaticsub: true
writeinfojson: true
writesubtitles: true
writethumbnail: true
youtube_include_dash_manifest: false

10
modules/videos/conftest.py
View file

@ -8,6 +8,7 @@ import pytest
from PIL import Image
from modules.videos.downloader import VideoDownloader, ChannelDownloader
from modules.videos.lib import set_test_channels_config
from modules.videos.models import Channel, Video
from wrolpi.common import sanitize_link
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
@ -152,3 +153,12 @@ def video_download_manager(test_download_manager) -> DownloadManager:
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: # prevent real downloads
mock_extract_info.side_effect = Exception('You must patch modules.videos.downloader.YDL.extract_info!')
yield test_download_manager
@pytest.fixture
def test_channels_config(test_directory):
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/channels.yaml'
set_test_channels_config(True)
yield config_path
set_test_channels_config(False)

8
modules/videos/downloader.py
View file

@ -18,9 +18,9 @@ from wrolpi.downloader import Downloader, Download
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \
from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \
get_videos_directory
from .lib import upsert_video, refresh_channel_videos
from .lib import upsert_video, refresh_channel_videos, get_downloader_config
from .models import Video, Channel
from .video_url_resolver import video_url_resolver
@ -217,11 +217,11 @@ class VideoDownloader(Downloader, ABC):
raise ValueError(f'Output directory does not exist! {out_dir=}')
# YoutubeDL expects specific options, add onto the default options
config = load_downloader_config()
config = get_downloader_config().dict()
options = config.copy()
options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
# We don't need format when getting the filename.
# We don't need "format" when getting the filename.
del options['format']
logger.debug(f'Downloading {url} to {out_dir}')

194
modules/videos/lib.py
View file

@ -8,8 +8,7 @@ from uuid import uuid1
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.orm import Session
from wrolpi.common import logger, chunks, get_config
from wrolpi.common import save_settings_config
from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.media_path import MediaPath
@ -289,10 +288,186 @@ def refresh_videos(channel_links: list = None):
logger.info('Refresh of video files complete')
def get_channels_config(session: Session) -> dict:
"""
Create a dictionary that contains all the Channels from the DB.
"""
class ChannelsConfig(ConfigFile):
file_name = 'channels.yaml'
default_config = dict(
channels={
'wrolpi': dict(
name='WROLPi',
url='https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos',
directory='videos/wrolpi',
download_frequency=604800,
)
},
favorites=dict(),
)
@property
def channels(self) -> dict:
return self._config['channels']
@channels.setter
def channels(self, value: dict):
self.update({'channels': value})
@property
def favorites(self) -> dict:
return self._config['favorites']
@favorites.setter
def favorites(self, value: dict):
self.update({'favorites': value})
CHANNELS_CONFIG: ChannelsConfig = None
TEST_CHANNELS_CONFIG: ChannelsConfig = None
def get_channels_config():
global TEST_CHANNELS_CONFIG
if isinstance(TEST_CHANNELS_CONFIG, ConfigFile):
return TEST_CHANNELS_CONFIG
global CHANNELS_CONFIG
if isinstance(CHANNELS_CONFIG, ConfigFile):
return CHANNELS_CONFIG
CHANNELS_CONFIG = ChannelsConfig()
return CHANNELS_CONFIG
def set_test_channels_config(enable: bool):
global TEST_CHANNELS_CONFIG
if enable is True:
TEST_CHANNELS_CONFIG = ChannelsConfig()
else:
TEST_CHANNELS_CONFIG = None
class DownloaderConfig(ConfigFile):
file_name = 'downloader.yaml'
default_config = dict(
continue_dl=True,
dateafter='19900101',
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
format='22 / 135 / 18 / 43',
nooverwrites=True,
quiet=False,
writeautomaticsub=True,
writeinfojson=True,
writesubtitles=True,
writethumbnail=True,
youtube_include_dash_manifest=False,
)
@property
def continue_dl(self) -> bool:
return self._config['continue_dl']
@continue_dl.setter
def continue_dl(self, value: bool):
self.update({'continue_dl': value})
@property
def dateafter(self) -> str:
return self._config['dateafter']
@dateafter.setter
def dateafter(self, value: str):
self.update({'dateafter': value})
@property
def file_name_format(self) -> str:
return self._config['file_name_format']
@file_name_format.setter
def file_name_format(self, value: str):
self.update({'file_name_format': value})
@property
def format(self) -> str:
return self._config['format']
@format.setter
def format(self, value: str):
self.update({'format': value})
@property
def nooverwrites(self) -> bool:
return self._config['nooverwrites']
@nooverwrites.setter
def nooverwrites(self, value: bool):
self.update({'nooverwrites': value})
@property
def quiet(self) -> bool:
return self._config['quiet']
@quiet.setter
def quiet(self, value: bool):
self.update({'quiet': value})
@property
def writeautomaticsub(self) -> bool:
return self._config['writeautomaticsub']
@writeautomaticsub.setter
def writeautomaticsub(self, value: bool):
self.update({'writeautomaticsub': value})
@property
def writeinfojson(self) -> bool:
return self._config['writeinfojson']
@writeinfojson.setter
def writeinfojson(self, value: bool):
self.update({'writeinfojson': value})
@property
def writesubtitles(self) -> bool:
return self._config['writesubtitles']
@writesubtitles.setter
def writesubtitles(self, value: bool):
self.update({'writesubtitles': value})
@property
def writethumbnail(self) -> bool:
return self._config['writethumbnail']
@writethumbnail.setter
def writethumbnail(self, value: bool):
self.update({'writethumbnail': value})
@property
def youtube_include_dash_manifest(self) -> bool:
return self._config['youtube_include_dash_manifest']
@youtube_include_dash_manifest.setter
def youtube_include_dash_manifest(self, value: bool):
self.update({'youtube_include_dash_manifest': value})
DOWNLOADER_CONFIG: DownloaderConfig = None
TEST_DOWNLOADER_CONFIG: DownloaderConfig = None
def get_downloader_config():
global TEST_DOWNLOADER_CONFIG
if isinstance(TEST_DOWNLOADER_CONFIG, WROLPiConfig):
return TEST_DOWNLOADER_CONFIG
global DOWNLOADER_CONFIG
if isinstance(DOWNLOADER_CONFIG, WROLPiConfig):
return DOWNLOADER_CONFIG
DOWNLOADER_CONFIG = DownloaderConfig()
return DOWNLOADER_CONFIG
def get_channels_config_from_db(session: Session) -> dict:
"""Create a dictionary that contains all the Channels from the DB."""
channels = session.query(Channel).order_by(Channel.link).all()
channels = {i.link: i.config_view() for i in channels}
@ -315,10 +490,11 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
"""
Pull the Channel information from the DB, save it to the config.
"""
config = get_channels_config(session)
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
if preserve_favorites:
config['favorites'].update(get_config().get('favorites', {}))
save_settings_config(config)
config['favorites'].update(channels_config.favorites or {})
channels_config.update(config)
async def get_statistics():

18
modules/videos/test/test_common.py
View file

@ -11,7 +11,7 @@ from PIL import Image
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.common import get_absolute_media_path, get_config
from wrolpi.common import get_absolute_media_path
from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
@ -19,7 +19,7 @@ from wrolpi.vars import PROJECT_DIR
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
import_videos_config
from ..lib import save_channels_config
from ..lib import save_channels_config, get_channels_config
class TestCommon(TestAPI):
@ -309,7 +309,7 @@ def test_update_censored_videos(test_session, video_factory, simple_channel):
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
def test_import_favorites(test_session, simple_channel, video_factory):
def test_import_favorites(test_session, simple_channel, video_factory, test_channels_config):
"""
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
favorited again.
@ -331,8 +331,8 @@ def test_import_favorites(test_session, simple_channel, video_factory):
# Save config, verify that favorite is set.
save_channels_config()
config = get_config()
assert config['favorites'] == favorites
config = get_channels_config()
assert config.favorites == favorites
def import_and_verify(favorited_ids: List[int]):
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
@ -351,8 +351,8 @@ def test_import_favorites(test_session, simple_channel, video_factory):
test_session.query(Video).filter_by(id=vid2.id).delete()
test_session.commit()
save_channels_config()
config = get_config()
assert config['favorites'] == favorites
config = get_channels_config()
assert config.favorites == favorites
import_and_verify([vid3.id])
# Add vid2 again, it should be favorited on import.
@ -363,6 +363,6 @@ def test_import_favorites(test_session, simple_channel, video_factory):
# Deleting the Video in the model really removes the favorite status.
vid3.delete()
config = get_config()
assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
config = get_channels_config()
assert config.favorites == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
import_and_verify([vid2.id])

1
requirements.txt
View file

@ -7,6 +7,7 @@ Pint==0.18
psycopg2-binary==2.9.3
pytest-asyncio==0.17.2
pytest==6.2.5
python-dotenv==0.19.2
python-magic==0.4.24
pytz==2021.3
PyYAML==5.3.1 # this is the only version available on RPi

68
wrolpi/admin.py
View file

@ -3,25 +3,25 @@ import pathlib
import subprocess
from wrolpi.common import logger
from wrolpi.vars import PYTEST, DEFAULT_CPU_FREQUENCY
from wrolpi.vars import PYTEST, DEFAULT_CPU_FREQUENCY, DOCKERIZED
logger = logger.getChild(__name__)
# Location on an RPi
SUDO_BIN = pathlib.Path('/usr/bin/sudo')
if not SUDO_BIN.is_file() and not PYTEST:
if not SUDO_BIN.is_file() and not PYTEST and not DOCKERIZED:
logger.error('COULD NOT FIND sudo!!!')
NMCLI_BIN = pathlib.Path('/usr/bin/nmcli')
if not NMCLI_BIN.is_file() and not PYTEST:
if not NMCLI_BIN.is_file() and not PYTEST and not DOCKERIZED:
logger.error('COULD NOT FIND nmcli!!!')
CPUFREQ_INFO_BIN = pathlib.Path('/usr/bin/cpufreq-info')
if not CPUFREQ_INFO_BIN.is_file() and not PYTEST:
if not CPUFREQ_INFO_BIN.is_file() and not PYTEST and not DOCKERIZED:
logger.error('COULD NOT FIND cpufreq-info!!!')
CPUFREQ_SET_BIN = pathlib.Path('/usr/bin/cpufreq-set')
if not CPUFREQ_SET_BIN.is_file() and not PYTEST:
if not CPUFREQ_SET_BIN.is_file() and not PYTEST and not DOCKERIZED:
logger.error('COULD NOT FIND cpufreq-set!!!')
POWER_SAVE_FREQ = 'powersave' # noqa
@ -31,11 +31,17 @@ class HotspotStatus(enum.Enum):
disconnected = enum.auto() # Radio is on, but Hotspot is not connected.
unavailable = enum.auto() # Radio is off.
connected = enum.auto() # Radio is on, Hotspot is on.
unknown = enum.auto() # Unknown status. Hotspot may not be supported on this hardware.
def hotspot_status() -> HotspotStatus:
cmd = (NMCLI_BIN,)
output = subprocess.check_output(cmd).decode().strip()
try:
output = subprocess.check_output(cmd).decode().strip()
except FileNotFoundError as e:
if not DOCKERIZED:
logger.debug(f'Could not get hotspot status', exc_info=e)
return HotspotStatus.unknown
for line in output.splitlines():
if line.startswith('wlan0: connected'):
@ -45,12 +51,13 @@ def hotspot_status() -> HotspotStatus:
elif line.startswith('wlan0: unavailable'):
return HotspotStatus.unavailable
raise Exception(f'Unknown status! {output=}')
return HotspotStatus.unknown
def enable_hotspot():
"""
Turn the wlan0 interface into a hotspot. If wlan0 is already running, replace that with a hotspot.
"""Turn the wlan0 interface into a hotspot.
If wlan0 is already running, replace that with a hotspot.
"""
status = hotspot_status()
logger.warning(f'Hotspot status: {status}')
@ -69,13 +76,19 @@ def enable_hotspot():
cmd = (SUDO_BIN, NMCLI_BIN, 'radio', 'wifi', 'on')
subprocess.check_call(cmd)
return enable_hotspot()
elif status == HotspotStatus.unknown:
logger.error('Cannot enable hotspot with unknown status!')
def disable_hotspot():
"""Turn off the wlan0 interface."""
cmd = (SUDO_BIN, NMCLI_BIN, 'radio', 'wifi', 'off')
subprocess.check_call(cmd)
return True
try:
subprocess.check_call(cmd)
return True
except Exception as e:
logger.error('Failed to disable hotspot', exc_info=e)
return False
class GovernorStatus(enum.Enum):
@ -90,14 +103,19 @@ class GovernorStatus(enum.Enum):
GOVERNOR_MAP = {
'The governor "ondemand" may decide ': GovernorStatus.ondemand,
'The governor "powersave" may decide ': GovernorStatus.powersave,
'governor "ondemand"': GovernorStatus.ondemand,
'governor "powersave"': GovernorStatus.powersave,
}
def throttle_status() -> GovernorStatus:
cmd = (CPUFREQ_INFO_BIN,)
output = subprocess.check_output(cmd)
try:
output = subprocess.check_output(cmd)
except FileNotFoundError as e:
if not DOCKERIZED:
logger.debug(f'Could not get CPU throttle info', exc_info=e)
return GovernorStatus.unknown
output = output.decode().strip()
for line in output.splitlines():
@ -109,12 +127,22 @@ def throttle_status() -> GovernorStatus:
def throttle_cpu_on() -> bool:
cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', POWER_SAVE_FREQ)
subprocess.check_call(cmd)
return True
"""Call cpufreq-set to throttle CPU."""
try:
cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', POWER_SAVE_FREQ)
subprocess.check_call(cmd)
return True
except FileNotFoundError:
if not DOCKERIZED:
logger.error('Could not enable CPU throttle', exc_info=True)
def throttle_cpu_off() -> bool:
cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', DEFAULT_CPU_FREQUENCY)
subprocess.check_call(cmd)
return True
"""Call cpufreq-set to un-throttle CPU."""
try:
cmd = (SUDO_BIN, CPUFREQ_SET_BIN, '-g', DEFAULT_CPU_FREQUENCY)
subprocess.check_call(cmd)
return True
except FileNotFoundError:
if not DOCKERIZED:
logger.error('Could not disable CPU throttle', exc_info=True)

292
wrolpi/common.py
View file

@ -10,26 +10,27 @@ import queue
import re
import string
import tempfile
from copy import deepcopy
from datetime import datetime, date
from decimal import Decimal
from functools import wraps
from itertools import islice, filterfalse, tee
from multiprocessing import Event, Queue
from multiprocessing import Event, Queue, Lock, Manager
from pathlib import Path
from typing import Union, Callable, Tuple, Dict, Mapping, List, Iterable, Optional, Generator
from typing import Union, Callable, Tuple, Dict, List, Iterable, Optional, Generator, Any
from urllib.parse import urlunsplit, urlparse
import yaml
from cachetools import cached, TTLCache
from sanic import Blueprint
from sanic.request import Request
from sqlalchemy import types
from sqlalchemy.ext.compiler import compiles
from sqlalchemy.ext.declarative import declarative_base
from wrolpi.dates import DEFAULT_TIMEZONE
from wrolpi.errors import WROLModeEnabled, NativeOnly
from wrolpi.vars import CONFIG_PATH, EXAMPLE_CONFIG_PATH, PUBLIC_HOST, PUBLIC_PORT, PROJECT_DIR, PYTEST, MODULES_DIR, \
DOCKERIZED
from wrolpi.vars import PUBLIC_HOST, PUBLIC_PORT, PYTEST, MODULES_DIR, \
DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY
logger = logging.getLogger()
ch = logging.StreamHandler()
@ -167,94 +168,169 @@ def get_sanic_url(scheme: str = 'http', path: str = None, query: list = None, fr
return unparsed
def get_example_config() -> dict:
config_path = EXAMPLE_CONFIG_PATH
with open(str(config_path), 'rt') as fh:
config = yaml.load(fh, Loader=yaml.Loader)
return dict(config)
def get_local_config() -> dict:
config_path = CONFIG_PATH
with open(str(config_path), 'rt') as fh:
config = yaml.load(fh, Loader=yaml.Loader)
return dict(config)
def get_config() -> dict:
try:
return get_local_config()
except FileNotFoundError:
return get_example_config()
def combine_dicts(*dicts: dict) -> dict:
class ConfigFile:
"""
Recursively combine dictionaries, preserving the leftmost value.
>>> a = dict(a='b', c=dict(d='e'))
>>> b = dict(a='c', e='f')
>>> combine_dicts(a, b)
dict(a='b', c=dict(d='e'), e='f')
This class keeps track of the contents of a config file. You can update the config by calling
.update(), or by creating a property that does so. Save your changes using .save().
"""
if len(dicts) == 0:
raise IndexError('No dictionaries to iterate through')
elif len(dicts) == 1:
return dicts[0]
a, b = dicts[-2:]
c = dicts[:-2]
new = {}
keys = set(a.keys())
keys = keys.union(b.keys())
for k in keys:
if k in b and k in a and isinstance(b[k], Mapping):
value = combine_dicts(a[k], b[k])
else:
value = a.get(k, b.get(k))
new[k] = value
if c:
return combine_dicts(*c, new)
return new
file_name: str = None
default_config: dict = None
def __init__(self):
config_file = self.get_file()
self.file_lock = Lock()
self._config = Manager().dict()
# Use the default settings to initialize the config.
self._config.update(deepcopy(self.default_config))
if config_file.is_file():
# Use the config file to get the values the user set.
with config_file.open('rt') as fh:
self._config.update(yaml.load(fh, Loader=yaml.Loader))
def save(self):
"""
Write this config to its file.
Use the existing config file as a template; if any values are missing in the new config, use the values from the
config file.
"""
config_file = self.get_file()
# Don't overwrite a real config while testing.
if PYTEST and not str(config_file).startswith('/tmp'):
raise ValueError(f'Refusing to save config file while testing: {config_file}')
# Only one process can write to the file.
self.file_lock.acquire(block=True, timeout=5.0)
try:
# Config directory may not exist while testing.
if PYTEST and not config_file.parent.is_dir():
config_file.parent.mkdir()
# Read the existing config, replace all values, then save.
if config_file.is_file():
with config_file.open('rt') as fh:
config = yaml.load(fh, Loader=yaml.Loader)
else:
# Config file does not yet exist.
config = dict()
config.update({k: v for k, v in self._config.items() if v is not None})
with config_file.open('wt') as fh:
yaml.dump(config, fh)
finally:
self.file_lock.release()
def get_file(self) -> Path:
if not self.file_name:
raise NotImplementedError(f'You must define a file name for this {self.__class__.__name__} config.')
if PYTEST:
return get_media_directory() / f'config/{self.file_name}'
return CONFIG_DIR / self.file_name
def update(self, config: dict):
"""
Update any values of this config. Save the config to its file.
"""
config = {k: v for k, v in config.items() if k in self._config}
self._config.update(config)
self.save()
def dict(self):
"""
Get a deepcopy of this config.
"""
return deepcopy(self._config)
def save_settings_config(config=None):
"""
Save new settings to local.yaml, overwriting what is there. This function updates the config file from three
sources: the config object argument, the local config, then the example config; in that order.
"""
config = config or {}
example_config = get_example_config()
# Remove the example channel, that shouldn't be saved to local
example_config.pop('channels')
try:
local_config = get_local_config()
except FileNotFoundError:
# Local config does not yet exist, lets create it
local_config = {}
class WROLPiConfig(ConfigFile):
file_name = 'wrolpi.yaml'
default_config = dict(
download_on_startup=True,
hotspot_on_startup=True,
throttle_on_startup=False,
timezone=str(DEFAULT_TIMEZONE),
wrol_mode=False,
)
if 'channels' in config and 'channels' in local_config:
del local_config['channels']
@property
def download_on_startup(self) -> bool:
return self._config['download_on_startup']
if ('channels' in config or 'channels' in local_config) and 'channels' in example_config:
del example_config['channels']
@download_on_startup.setter
def download_on_startup(self, value: bool):
self.update({'download_on_startup': value})
# Create a new config using the values in order of: config -> local_config -> example_config
all_keys = list(config.keys()) + list(local_config.keys()) + list(example_config.keys())
new_config = {i: config.get(i, local_config.get(i, example_config.get(i))) for i in all_keys}
@property
def hotspot_on_startup(self) -> bool:
return self._config['hotspot_on_startup']
logger.info(f'Writing config to file: {CONFIG_PATH}')
with open(str(CONFIG_PATH), 'wt') as fh:
yaml.dump(new_config, fh)
@hotspot_on_startup.setter
def hotspot_on_startup(self, value: bool):
self.update({'hotspot_on_startup': value})
@property
def throttle_on_startup(self) -> bool:
return self._config['throttle_on_startup']
@throttle_on_startup.setter
def throttle_on_startup(self, value: bool):
self.update({'throttle_on_startup': value})
@property
def timezone(self) -> str:
return self._config['timezone']
@timezone.setter
def timezone(self, value: str):
self.update({'timezone': value})
@property
def wrol_mode(self) -> bool:
return self._config['wrol_mode']
@wrol_mode.setter
def wrol_mode(self, value: bool):
self.update({'wrol_mode': value})
WROLPI_CONFIG: WROLPiConfig = None
TEST_WROLPI_CONFIG: WROLPiConfig = None
def get_config() -> WROLPiConfig:
"""Read the global WROLPi yaml config file."""
global TEST_WROLPI_CONFIG
if isinstance(TEST_WROLPI_CONFIG, WROLPiConfig):
return TEST_WROLPI_CONFIG
global WROLPI_CONFIG
if isinstance(WROLPI_CONFIG, WROLPiConfig):
return WROLPI_CONFIG
# Create the config instance once.
WROLPI_CONFIG = WROLPiConfig()
return WROLPI_CONFIG
def set_test_config(enable: bool):
global TEST_WROLPI_CONFIG
if enable:
TEST_WROLPI_CONFIG = WROLPiConfig()
else:
TEST_WROLPI_CONFIG = None
@cached(cache=TTLCache(maxsize=1, ttl=30))
def wrol_mode_enabled() -> bool:
"""
Return the boolean value of the `wrol_mode` in the config.
"""
config = get_config()
enabled = config.get('wrol_mode', False)
return bool(enabled)
"""Return True if WROL Mode is enabled."""
return get_config().wrol_mode
def set_wrol_mode(enable: bool):
"""Enable or disable WROL Mode for all processes. This also updates the config file."""
get_config().wrol_mode = enable
def wrol_mode_check(func):
@ -327,13 +403,6 @@ def remove_whitespace(s: str) -> str:
return WHITESPACE.sub('', s)
def remove_dict_value_whitespace(d: Dict) -> Dict:
"""
Remove the whitespace around a the values in a dictionary. Handles if the value isn't a string.
"""
return {k: v.strip() if hasattr(v, 'strip') else v for k, v in d.items()}
def run_after(after: callable, *args, **kwargs) -> callable:
"""
Run the `after` function sometime in the future ofter the wrapped function returns.
@ -369,7 +438,6 @@ def run_after(after: callable, *args, **kwargs) -> callable:
TEST_MEDIA_DIRECTORY = None
MEDIA_DIRECTORY = None
def set_test_media_directory(path):
@ -383,11 +451,13 @@ def set_test_media_directory(path):
def get_media_directory() -> Path:
"""
Get the media directory configured in local.yaml.
"""Get the media directory.
This will typically be /opt/wrolpi, or something else from the .env.
During testing, this function returns TEST_MEDIA_DIRECTORY.
"""
global TEST_MEDIA_DIRECTORY
global MEDIA_DIRECTORY
if PYTEST and not TEST_MEDIA_DIRECTORY:
raise ValueError('No test media directory set during testing!!')
@ -395,16 +465,6 @@ def get_media_directory() -> Path:
if isinstance(TEST_MEDIA_DIRECTORY, pathlib.Path):
return TEST_MEDIA_DIRECTORY
if isinstance(MEDIA_DIRECTORY, pathlib.Path):
return MEDIA_DIRECTORY
config = get_config()
media_directory = config['media_directory']
media_directory = Path(media_directory)
if not media_directory.is_absolute():
# Media directory is relative. Assume that is relative to the project directory.
media_directory = PROJECT_DIR / media_directory
MEDIA_DIRECTORY = media_directory.absolute()
return MEDIA_DIRECTORY
@ -660,6 +720,7 @@ def escape_file_name(name: str) -> str:
def native_only(func: callable):
"""Wraps a function. Raises NativeOnly if run while Dockerized."""
@wraps(func)
def wrapped(*a, **kw):
if DOCKERIZED:
@ -667,3 +728,28 @@ def native_only(func: callable):
return func(*a, **kw)
return wrapped
def read_yaml(path: Path):
with path.open('rt') as fh:
return yaml.load(fh, Loader=yaml.Loader)
def recursive_map(obj: Any, func: callable):
"""
Apply `func` to all values of any dictionaries; or items in any list/set/tuple.
>>> recursive_map({'foo ': ' bar '}, lambda i: i.strip() if hasattr(i, 'strip') else i)
{'foo ': 'bar'}
>>> recursive_map(['foo ', ' bar '], lambda i: i.strip() if hasattr(i, 'strip') else i)
['foo', 'bar']
"""
if isinstance(obj, dict):
obj = obj.copy()
for key, value in obj.items():
obj[key] = recursive_map(value, func)
return obj
elif isinstance(obj, (list, set, tuple)):
type_ = type(obj)
return type_(map(lambda i: recursive_map(i, func), obj))
return func(obj)

16
wrolpi/conftest.py
View file

@ -9,12 +9,11 @@ from unittest import mock
from uuid import uuid1
import pytest
import yaml
from sanic_testing.testing import SanicTestClient
from sqlalchemy.engine import Engine, create_engine
from sqlalchemy.orm import Session, sessionmaker
from wrolpi.common import set_test_media_directory, Base, get_example_config
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.dates import set_test_now
from wrolpi.db import postgres_engine, get_db_args
from wrolpi.downloader import DownloadManager
@ -87,17 +86,14 @@ def test_directory() -> pathlib.Path:
@pytest.fixture(autouse=True)
def test_config():
def test_config(test_directory):
"""
Create a test config based off the example config.
"""
test_config_path = tempfile.NamedTemporaryFile(mode='rt')
with mock.patch('wrolpi.vars.CONFIG_PATH', test_config_path.name), \
mock.patch('wrolpi.common.CONFIG_PATH', test_config_path.name):
config = get_example_config()
with open(test_config_path.name, 'wt') as fh:
fh.write(yaml.dump(config))
yield pathlib.Path(test_config_path.name)
config_path = test_directory / 'config/wrolpi.yaml'
set_test_config(True)
yield config_path
set_test_config(False)
ROUTES_ATTACHED = False

9
wrolpi/errors.py
View file

@ -144,6 +144,10 @@ class NativeOnly(APIError):
pass
class HotspotError(APIError):
pass
error_codes = iter(range(1, 1000))
API_ERRORS = {
@ -317,4 +321,9 @@ API_ERRORS = {
'message': 'This functionality is only supported outside a docker container',
'status': HTTPStatus.BAD_REQUEST,
},
HotspotError: {
'code': next(error_codes),
'message': 'Updating/accessing Hotspot encountered an error',
'status': HTTPStatus.INTERNAL_SERVER_ERROR,
}
}

83
wrolpi/root_api.py
View file

@ -15,11 +15,11 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi import admin
from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, save_settings_config, \
Base, get_media_directory, wrol_mode_check, native_only
from wrolpi.common import set_sanic_url_parts, logger, get_config, wrol_mode_enabled, Base, get_media_directory, \
wrol_mode_check, native_only, set_wrol_mode
from wrolpi.dates import set_timezone
from wrolpi.downloader import download_manager
from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError
from wrolpi.errors import WROLModeEnabled, InvalidTimezone, API_ERRORS, APIError, ValidationError, HotspotError
from wrolpi.media_path import MediaPath
from wrolpi.schema import RegexRequest, RegexResponse, SettingsRequest, SettingsResponse, DownloadRequest, EchoResponse
from wrolpi.vars import DOCKERIZED
@ -112,30 +112,63 @@ async def echo(request: Request):
@openapi.description('Get WROLPi settings')
@openapi.response(HTTPStatus.OK, SettingsResponse)
def get_settings(_: Request):
config = dict(get_config())
return json_response({'config': config, 'version': __version__})
config = get_config()
settings = {
'download_on_startup': config.download_on_startup,
'hotspot_on_startup': config.hotspot_on_startup,
'hotspot_status': admin.hotspot_status().name,
'media_directory': get_media_directory(),
'throttle_on_startup': config.throttle_on_startup,
'throttle_status': admin.throttle_status().name,
'timezone': config.timezone,
'version': __version__,
'wrol_mode': config.wrol_mode,
}
return json_response(settings)
@root_api.patch('/settings')
@openapi.description('Update WROLPi settings')
@validate(json=SettingsRequest)
def update_settings(_: Request, data: dict):
if wrol_mode_enabled() and 'wrol_mode' not in data:
def update_settings(_: Request, body: SettingsRequest):
if wrol_mode_enabled() and body.wrol_mode is None:
# Cannot update settings while WROL Mode is enabled, unless you want to disable WROL Mode.
raise WROLModeEnabled()
if body.wrol_mode is False:
# Disable WROL Mode
set_wrol_mode(False)
return response.empty()
elif body.wrol_mode is True:
# Enable WROL Mode
set_wrol_mode(True)
return response.empty()
try:
if data.get('timezone'):
set_timezone(data['timezone'])
if body.timezone:
set_timezone(body.timezone)
except UnknownTimeZoneError:
raise InvalidTimezone(f'Invalid timezone: {data["timezone"]}')
raise InvalidTimezone(f'Invalid timezone: {body.timezone}')
save_settings_config(data)
# Remove any keys with None values, then save the config.
config = {k: v for k, v in body.__dict__.items() if v is not None}
wrolpi_config = get_config()
wrolpi_config.update(config)
if data.get('wrolpi_mode'):
if body.wrol_mode:
download_manager.kill()
return response.raw('', HTTPStatus.NO_CONTENT)
if body.hotspot_status is True:
# Turn on Hotspot
if admin.enable_hotspot() is False:
raise HotspotError('Could not turn on hotspot')
elif body.hotspot_status is False:
# Turn off Hotspot
if admin.disable_hotspot() is False:
raise HotspotError('Could not turn off hotspot')
return response.empty()
@root_api.post('/valid_regex')
@ -211,18 +244,6 @@ async def get_downloaders(_: Request):
return json_response(ret)
@root_api.get('/hotspot')
@native_only
@openapi.description('Get the status of the hotspot')
async def hotspot(_: Request):
status = admin.hotspot_status()
# Get the name of the enum.
status = status.name
ret = dict(status=status)
return json_response(ret)
@root_api.post('/hotspot/on')
@openapi.description('Turn on the hotspot')
@native_only
@ -243,18 +264,6 @@ async def hotspot_off(_: Request):
return response.empty(HTTPStatus.INTERNAL_SERVER_ERROR)
@root_api.get('/throttle')
@openapi.description('Get status of CPU throttling')
@native_only
async def throttle_status(_: Request):
status = admin.throttle_status()
# Get the name of the enum.
status = status.name
ret = dict(status=status)
return json_response(ret)
@root_api.post('/throttle/on')
@openapi.description('Turn on CPU throttling')
@native_only

11
wrolpi/schema.py
View file

@ -14,9 +14,14 @@ class SettingsResponse:
@dataclass
class SettingsRequest:
media_directory: str
wrol_mode: bool
timezone: str
download_on_startup: Optional[bool] = None
hotspot_on_startup: Optional[bool] = None
hotspot_status: Optional[bool] = None
media_directory: Optional[str] = None
throttle_on: Optional[bool] = None
throttle_on_startup: Optional[bool] = None
timezone: Optional[str] = None
wrol_mode: Optional[bool] = None
@dataclass

48
wrolpi/test/common.py
View file

@ -1,4 +1,3 @@
import json
import pathlib
import tempfile
import unittest
@ -6,18 +5,14 @@ from contextlib import contextmanager
from functools import partialmethod
from http import HTTPStatus
from itertools import zip_longest
from shutil import copyfile
from typing import List, Optional
from typing import List
import mock
import websockets
import yaml
from requests import Response
from wrolpi.common import EXAMPLE_CONFIG_PATH, get_config, set_test_media_directory, get_media_directory
from wrolpi.common import set_test_media_directory, get_media_directory
from wrolpi.conftest import ROUTES_ATTACHED, test_db, test_client # noqa
from wrolpi.db import postgres_engine
from wrolpi.vars import PROJECT_DIR
TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
@ -120,21 +115,11 @@ class ExtendedTestCase(PytestCase, unittest.TestCase):
class TestAPI(ExtendedTestCase):
def setUp(self) -> None:
self.config_path_patch = mock.patch('wrolpi.vars.CONFIG_PATH', TEST_CONFIG_PATH.name)
self.config_path_patch.start()
self.tmp_dir = tempfile.TemporaryDirectory()
self.tmp_path = pathlib.Path(self.tmp_dir.name)
set_test_media_directory(self.tmp_dir.name)
# Copy the example config to test against
copyfile(str(EXAMPLE_CONFIG_PATH), TEST_CONFIG_PATH.name)
# Setup the testing video root directory
config = get_config()
config['media_directory'] = str(PROJECT_DIR / 'test')
with open(TEST_CONFIG_PATH.name, 'wt') as fh:
fh.write(yaml.dump(config))
set_test_media_directory(self.tmp_path)
def tearDown(self) -> None:
self.config_path_patch.stop()
set_test_media_directory(None)
def assertHTTPStatus(self, response: Response, status: int):
@ -145,22 +130,6 @@ class TestAPI(ExtendedTestCase):
assertNO_CONTENT = partialmethod(assertHTTPStatus, status=HTTPStatus.NO_CONTENT)
@contextmanager
def wrap_media_directory(path: Optional[pathlib.Path] = None):
cleanup = False
if not path:
path = tempfile.TemporaryDirectory()
cleanup = True
set_test_media_directory(path.name)
try:
yield
finally:
set_test_media_directory(None)
if cleanup:
path.cleanup()
@contextmanager
def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pathlib.Path:
"""
@ -196,17 +165,6 @@ def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pa
yield tmp_dir.absolute()
async def get_all_ws_messages(ws) -> List[dict]:
messages = []
while True:
try:
message = await ws.recv()
except websockets.exceptions.ConnectionClosedOK:
break
messages.append(json.loads(message))
return messages
def assert_dict_contains(d1: dict, d2: dict):
if hasattr(d1, '__dict__'):
d1 = d1.__dict__

113
wrolpi/test/test_common.py
View file

@ -7,91 +7,50 @@ from decimal import Decimal
import pytest
from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \
from wrolpi.common import insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \
escape_file_name
from wrolpi.dates import set_timezone, now
from wrolpi.errors import InvalidTimezone
from wrolpi.test.common import build_test_directories, wrap_media_directory
from wrolpi.test.common import build_test_directories
@pytest.mark.parametrize(
'data,expected',
(
(
[dict()],
dict()
), (
[dict(a='b')],
dict(a='b')
), (
[dict(a='b'), dict(b='c')],
dict(a='b', b='c')
), (
[dict(b='c'), dict(a='b')],
dict(a='b', b='c'),
), (
[dict(a=dict(b='c'), d='e'), dict(a=(dict(f='g')))],
dict(a=dict(b='c', f='g'), d='e'),
), (
[dict(a=dict(b='c'), d='e'), dict(a=(dict(b='g')))],
dict(a=dict(b='c'), d='e'),
), (
[dict(a=dict(b='c'), d=[1, 2, 3]), dict(a=(dict(b='g')))],
dict(a=dict(b='c'), d=[1, 2, 3]),
), (
[dict(a='b', c=dict(d='e')), dict(a='c', e='f')],
dict(a='b', c=dict(d='e'), e='f')
), (
[dict(a='b'), dict(a='c', d='e'), dict(e='f', a='d')],
dict(a='b', d='e', e='f')
), (
[dict(a='b'), dict(), dict(e='f', a='d')],
dict(a='b', e='f')
)
)
)
def test_combine_dicts(data, expected):
assert combine_dicts(*data) == expected
def test_build_video_directories(test_directory):
structure = [
'channel1/vid1.mp4',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel1').is_dir()
assert (tempdir / 'channel1/vid1.mp4').is_file()
structure = [
'channel2/',
'channel2.1/channel2.2/',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel2').is_dir()
assert (tempdir / 'channel2.1/channel2.2').is_dir()
def test_build_video_directories():
with wrap_media_directory():
structure = [
'channel1/vid1.mp4',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel1').is_dir()
assert (tempdir / 'channel1/vid1.mp4').is_file()
structure = [
'channel3/vid1.mp4',
'channel3/vid2.mp4',
'channel4/vid1.mp4',
'channel4/vid1.en.vtt',
'channel5/',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel3/vid1.mp4').is_file()
assert (tempdir / 'channel3').is_dir()
assert (tempdir / 'channel3/vid2.mp4').is_file()
assert (tempdir / 'channel4/vid1.mp4').is_file()
assert (tempdir / 'channel4/vid1.en.vtt').is_file()
assert (tempdir / 'channel5').is_dir()
structure = [
'channel2/',
'channel2.1/channel2.2/',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel2').is_dir()
assert (tempdir / 'channel2.1/channel2.2').is_dir()
structure = [
'channel3/vid1.mp4',
'channel3/vid2.mp4',
'channel4/vid1.mp4',
'channel4/vid1.en.vtt',
'channel5/',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel3/vid1.mp4').is_file()
assert (tempdir / 'channel3').is_dir()
assert (tempdir / 'channel3/vid2.mp4').is_file()
assert (tempdir / 'channel4/vid1.mp4').is_file()
assert (tempdir / 'channel4/vid1.en.vtt').is_file()
assert (tempdir / 'channel5').is_dir()
structure = [
'channel6/subdirectory/vid1.mp4',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel6/subdirectory').is_dir()
assert (tempdir / 'channel6/subdirectory/vid1.mp4').is_file()
structure = [
'channel6/subdirectory/vid1.mp4',
]
with build_test_directories(structure) as tempdir:
assert (tempdir / 'channel6/subdirectory').is_dir()
assert (tempdir / 'channel6/subdirectory/vid1.mp4').is_file()
def test_insert_parameter():

57
wrolpi/test/test_root_api.py
View file

@ -2,6 +2,9 @@ import json
from http import HTTPStatus
from itertools import zip_longest
from mock import mock
from wrolpi.common import get_config
from wrolpi.dates import strptime
from wrolpi.db import get_db_session
from wrolpi.downloader import Download
@ -154,3 +157,57 @@ class TestRootAPI(TestAPI):
recurring_downloads = response.json['recurring_downloads']
for d1, d2 in zip_longest(expected, recurring_downloads):
self.assertDictContains(d2, d1)
def test_hotspot_settings(test_session, test_client, test_config):
"""
The User can toggle the Hotspot via /settings. The Hotspot can be automatically started on startup.
"""
config = get_config()
assert config.hotspot_on_startup is True
with mock.patch('wrolpi.root_api.admin') as mock_admin:
# Turning on the hotspot succeeds.
mock_admin.enable_hotspot.return_value = True
request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': True}))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
mock_admin.enable_hotspot.assert_called_once()
mock_admin.reset_mock()
# Turning on the hotspot fails.
mock_admin.enable_hotspot.return_value = False
request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': True}))
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, response.json
assert response.json['code'] == 35
mock_admin.enable_hotspot.assert_called_once()
# Turning off the hotspot succeeds.
mock_admin.disable_hotspot.return_value = True
request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': False}))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
mock_admin.disable_hotspot.assert_called_once()
mock_admin.reset_mock()
# Turning off the hotspot succeeds.
mock_admin.disable_hotspot.return_value = False
request, response = test_client.patch('/api/settings', content=json.dumps({'hotspot_status': False}))
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR, response.json
assert response.json['code'] == 35
mock_admin.disable_hotspot.assert_called_once()
config = get_config()
assert config.hotspot_on_startup is True
def test_throttle_toggle(test_session, test_client, test_config):
with mock.patch('wrolpi.admin.subprocess') as mock_subprocess:
mock_subprocess.check_output.side_effect = [
b'wlan0: unavailable',
b'The governor "ondemand" may decide ',
]
request, response = test_client.get('/api/settings')
# Throttle is off by default.
assert response.status_code == HTTPStatus.OK
assert response.json['throttle_on_startup'] is False
assert response.json['throttle_status'] == 'ondemand'

35
wrolpi/vars.py
View file

@ -1,19 +1,31 @@
import os
import pathlib
import sys
from pathlib import Path
PROJECT_DIR: pathlib.Path = pathlib.Path(__file__).parents[1].absolute()
CONFIG_DIR: pathlib.Path = PROJECT_DIR / 'config'
CONFIG_PATH: pathlib.Path = CONFIG_DIR / 'local.yaml'
EXAMPLE_CONFIG_PATH: pathlib.Path = CONFIG_DIR / 'example.yaml'
MODULES_DIR: pathlib.Path = PROJECT_DIR / 'modules'
PUBLIC_HOST = os.environ.get('PUBLIC_HOST')
PUBLIC_PORT = os.environ.get('PUBLIC_PORT')
from dotenv import load_dotenv
# The root of WROLPi. Typically, /opt/wrolpi
PROJECT_DIR: Path = Path(__file__).parents[1].absolute()
# Load any .env file. This may contain our MEDIA_DIRECTORY.
load_dotenv(PROJECT_DIR / '.env')
# Special environment variable set in the docker/api/Dockerfile.
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
# tests are running
PYTEST = 'pytest' in sys.modules
# Get the media directory from the environment.
DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
if not MEDIA_DIRECTORY.is_dir() and not PYTEST:
print(f'Media directory does not exist! {MEDIA_DIRECTORY}')
CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config'
MODULES_DIR: Path = PROJECT_DIR / 'modules'
PUBLIC_HOST = os.environ.get('PUBLIC_HOST')
PUBLIC_PORT = os.environ.get('PUBLIC_PORT')
DEFAULT_TIMEZONE_STR = 'America/Boise'
LAST_MODIFIED_DATE_FORMAT = '%a, %d %b %Y %H:%M:%S GMT'
DATE_FORMAT = '%Y-%M-%d'
@ -28,3 +40,10 @@ DB_PORT = int(os.environ.get('DB_PORT', 5432))
DB_NAME = os.environ.get('DB_NAME', 'wrolpi')
DB_USER = os.environ.get('DB_USER', 'wrolpi')
DB_PASSWORD = os.environ.get('DB_PASSWORD', 'wrolpi')
EXAMPLE_CONFIG = {
'hotspot_on_startup': True,
'throttle_on_startup': False,
'timezone': 'America/Boise',
'wrol_mode': False,
}