mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
Squashing config rework.
This commit is contained in:
parent
5425142adb
commit
08065155cf
38 changed files with 1046 additions and 633 deletions
|
|
@ -5,7 +5,6 @@ from pathlib import Path
|
|||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
import magic
|
||||
import psycopg2
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
|
|
@ -16,6 +15,13 @@ from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
|
|||
from wrolpi.errors import InvalidFile
|
||||
from wrolpi.vars import PYTEST
|
||||
|
||||
try:
|
||||
import magic
|
||||
have_magic = True
|
||||
except ImportError:
|
||||
logger.error('Could not import magic. Is libmagic installed?')
|
||||
have_magic = False
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
||||
__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search']
|
||||
|
|
@ -96,7 +102,7 @@ def upsert_file(path: Path, session: Session) -> File:
|
|||
if not file:
|
||||
file = File(path=path)
|
||||
session.add(file)
|
||||
if not file.mimetype:
|
||||
if not file.mimetype and have_magic:
|
||||
file.mimetype = magic.from_file(str(path), mime=True)
|
||||
if not file.title:
|
||||
file.title = split_file_name(path)
|
||||
|
|
|
|||
|
|
@ -5,7 +5,7 @@ from sanic.request import Request
|
|||
from sanic_ext import validate
|
||||
from sanic_ext.extensions.openapi import openapi
|
||||
|
||||
from wrolpi.common import run_after, remove_dict_value_whitespace
|
||||
from wrolpi.common import run_after, recursive_map
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.root_api import get_blueprint, json_response
|
||||
from . import common, inventory, schema
|
||||
|
|
@ -48,9 +48,8 @@ def inventory_get(_: Request, inventory_id: int):
|
|||
)
|
||||
@validate(schema.InventoryPostRequest)
|
||||
@run_after(common.save_inventories_file)
|
||||
def post_inventory(_: Request, data: dict):
|
||||
# Cleanup the whitespace.
|
||||
data = remove_dict_value_whitespace(data)
|
||||
def post_inventory(_: Request, body: schema.InventoryPostRequest):
|
||||
data = remove_whitespace(body.__dict__)
|
||||
|
||||
inventory.save_inventory(data)
|
||||
return response.empty(HTTPStatus.CREATED)
|
||||
|
|
@ -63,9 +62,8 @@ def post_inventory(_: Request, data: dict):
|
|||
)
|
||||
@validate(schema.InventoryPutRequest)
|
||||
@run_after(common.save_inventories_file)
|
||||
def put_inventory(_: Request, inventory_id: int, data: dict):
|
||||
# Cleanup the whitespace.
|
||||
data = remove_dict_value_whitespace(data)
|
||||
def put_inventory(_: Request, inventory_id: int, body: schema.InventoryPutRequest):
|
||||
data = remove_whitespace(body.__dict__)
|
||||
|
||||
inventory.update_inventory(inventory_id, data)
|
||||
return response.empty()
|
||||
|
|
@ -93,9 +91,8 @@ def items_get(_: Request, inventory_id: int):
|
|||
)
|
||||
@validate(schema.ItemPostRequest)
|
||||
@run_after(common.save_inventories_file)
|
||||
def post_item(_: Request, inventory_id: int, data: dict):
|
||||
# Cleanup the whitespace.
|
||||
data = remove_dict_value_whitespace(data)
|
||||
def post_item(_: Request, inventory_id: int, body: schema.ItemPostRequest):
|
||||
data = remove_whitespace(body.__dict__)
|
||||
|
||||
inventory.save_item(inventory_id, data)
|
||||
return response.empty()
|
||||
|
|
@ -108,9 +105,8 @@ def post_item(_: Request, inventory_id: int, data: dict):
|
|||
)
|
||||
@validate(schema.ItemPutRequest)
|
||||
@run_after(common.save_inventories_file)
|
||||
def put_item(_: Request, item_id: int, data: dict):
|
||||
# Cleanup the whitespace.
|
||||
data = remove_dict_value_whitespace(data)
|
||||
def put_item(_: Request, item_id: int, body: schema.ItemPutRequest):
|
||||
data = remove_whitespace(body.__dict__)
|
||||
|
||||
inventory.update_item(item_id, data)
|
||||
return response.empty()
|
||||
|
|
@ -131,3 +127,7 @@ def item_delete(_: Request, item_ids: str):
|
|||
|
||||
inventory.delete_items(item_ids)
|
||||
return response.empty()
|
||||
|
||||
|
||||
def remove_whitespace(obj):
|
||||
return recursive_map(obj, lambda i: i.strip() if hasattr(i, 'strip') else i)
|
||||
|
|
|
|||
|
|
@ -3,14 +3,12 @@ from functools import partial
|
|||
from pathlib import Path
|
||||
from typing import List, Tuple
|
||||
|
||||
import yaml
|
||||
from pint import Quantity
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import logger, Base
|
||||
from wrolpi.common import logger, Base, ConfigFile
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import NoInventories, InventoriesVersionMismatch
|
||||
from wrolpi.vars import CONFIG_DIR
|
||||
from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \
|
||||
get_inventories_version
|
||||
from .models import Inventory, Item
|
||||
|
|
@ -98,23 +96,65 @@ def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quanti
|
|||
|
||||
|
||||
def cleanup_quantity(quantity: Quantity) -> Quantity:
|
||||
"""
|
||||
Remove trailing zeros from a Quantity.
|
||||
"""
|
||||
"""Remove trailing zeros from a Quantity."""
|
||||
num, unit = quantity_to_tuple(quantity)
|
||||
num = round(num, UNIT_PRECISION)
|
||||
num = str(num).rstrip('0').rstrip('.')
|
||||
return Decimal(num) * unit
|
||||
|
||||
|
||||
DEFAULT_SAVE_PATH = CONFIG_DIR / 'inventories.yaml'
|
||||
class InventoriesConfig(ConfigFile):
|
||||
file_name = 'inventories.yaml'
|
||||
default_config = dict(
|
||||
inventories=[],
|
||||
version=1,
|
||||
)
|
||||
|
||||
@property
|
||||
def inventories(self) -> dict:
|
||||
return self._config['inventories']
|
||||
|
||||
@inventories.setter
|
||||
def inventories(self, value: dict):
|
||||
self.update({'inventories': value})
|
||||
|
||||
@property
|
||||
def version(self) -> int:
|
||||
return self._config['version']
|
||||
|
||||
@version.setter
|
||||
def version(self, value: int):
|
||||
self.update({'version': value})
|
||||
|
||||
|
||||
def save_inventories_file(path: str = None):
|
||||
"""
|
||||
Write all inventories and their respective items to a YAML file.
|
||||
"""
|
||||
path: Path = Path(path) if path else DEFAULT_SAVE_PATH
|
||||
INVENTORIES_CONFIG: InventoriesConfig = None
|
||||
TEST_INVENTORIES_CONFIG: InventoriesConfig = None
|
||||
|
||||
|
||||
def get_inventories_config():
|
||||
global TEST_INVENTORIES_CONFIG
|
||||
if isinstance(TEST_INVENTORIES_CONFIG, ConfigFile):
|
||||
return TEST_INVENTORIES_CONFIG
|
||||
|
||||
global INVENTORIES_CONFIG
|
||||
if isinstance(INVENTORIES_CONFIG, ConfigFile):
|
||||
return INVENTORIES_CONFIG
|
||||
|
||||
INVENTORIES_CONFIG = InventoriesConfig()
|
||||
return INVENTORIES_CONFIG
|
||||
|
||||
|
||||
def set_test_inventories_config(enabled: bool):
|
||||
global TEST_INVENTORIES_CONFIG
|
||||
if enabled:
|
||||
TEST_INVENTORIES_CONFIG = InventoriesConfig()
|
||||
else:
|
||||
TEST_INVENTORIES_CONFIG = None
|
||||
|
||||
|
||||
def save_inventories_file():
|
||||
"""Write all inventories and their respective items to a YAML file."""
|
||||
config = get_inventories_config()
|
||||
|
||||
inventories = []
|
||||
for inventory in get_inventories():
|
||||
|
|
@ -124,44 +164,22 @@ def save_inventories_file(path: str = None):
|
|||
raise NoInventories('No Inventories are in the database!')
|
||||
|
||||
with increment_inventories_version() as version:
|
||||
if path.is_file():
|
||||
# Check that we aren't overwriting our inventories with empty inventories.
|
||||
with open(path, 'rt') as fh:
|
||||
old = yaml.load(fh, Loader=yaml.Loader)
|
||||
if old and not inventories:
|
||||
raise FileExistsError(f'Refusing to overwrite non-empty inventories.yaml with empty inventories.'
|
||||
f' {path}')
|
||||
if config.version and config.version > version:
|
||||
raise InventoriesVersionMismatch(
|
||||
f'Inventories config version is {config.version} but DB version is {get_inventories_version()}')
|
||||
|
||||
if old and old.get('version') > version:
|
||||
raise InventoriesVersionMismatch(
|
||||
f'Inventories config version is {old["version"]} but DB version is {get_inventories_version()}')
|
||||
|
||||
with open(path, 'wt') as fh:
|
||||
contents = dict(
|
||||
version=version,
|
||||
inventories=inventories,
|
||||
)
|
||||
|
||||
yaml.dump(contents, fh)
|
||||
config.inventories = inventories
|
||||
config.version = version
|
||||
config.save()
|
||||
|
||||
|
||||
@before_startup
|
||||
def import_inventories_file(path: str = None):
|
||||
path: Path = Path(path) if path else DEFAULT_SAVE_PATH
|
||||
|
||||
if not path.is_file():
|
||||
logger.warning(f'No inventories config file at {path}')
|
||||
return
|
||||
|
||||
with open(path, 'rt') as fh:
|
||||
contents = yaml.load(fh, Loader=yaml.Loader)
|
||||
|
||||
if not contents or 'inventories' not in contents:
|
||||
raise ValueError('Inventories file does not contain the expected "inventories" list.')
|
||||
def import_inventories_file():
|
||||
config = get_inventories_config()
|
||||
|
||||
inventories = get_inventories()
|
||||
inventories_names = {i.name for i in inventories}
|
||||
new_inventories = [i for i in contents['inventories'] if i['name'] not in inventories_names]
|
||||
new_inventories = [i for i in config.inventories if i['name'] not in inventories_names]
|
||||
with get_db_session(commit=True) as session:
|
||||
for inventory in new_inventories:
|
||||
items = inventory['items']
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import pytest
|
||||
|
||||
from modules.inventory import Inventory, Item, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES
|
||||
from modules.inventory.common import set_test_inventories_config
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
|
|
@ -23,4 +24,6 @@ def init_test_inventory(test_session):
|
|||
|
||||
inventory = test_session.query(Inventory).filter_by(name='Food Storage').one()
|
||||
test_session.commit()
|
||||
return inventory
|
||||
set_test_inventories_config(True)
|
||||
yield inventory
|
||||
set_test_inventories_config(False)
|
||||
|
|
|
|||
|
|
@ -59,7 +59,7 @@ DEFAULT_INVENTORIES = [
|
|||
sort_categories = partial(sorted, key=itemgetter(1, 0))
|
||||
|
||||
|
||||
def get_inventories() -> List[Base]:
|
||||
def get_inventories() -> List[Inventory]:
|
||||
with get_db_session() as session:
|
||||
inventories = session.query(Inventory).filter(
|
||||
Inventory.deleted_at == None, # noqa
|
||||
|
|
|
|||
|
|
@ -1,36 +1,51 @@
|
|||
from dataclasses import dataclass
|
||||
from datetime import date, datetime
|
||||
from decimal import Decimal
|
||||
from typing import Union, Optional
|
||||
|
||||
from wrolpi.dates import from_timestamp
|
||||
|
||||
|
||||
@dataclass
|
||||
class ItemPostRequest:
|
||||
brand: str
|
||||
name: str
|
||||
item_size: Decimal
|
||||
item_size: Union[str, Decimal]
|
||||
unit: str
|
||||
count: Decimal
|
||||
count: Union[str, Decimal]
|
||||
category: str
|
||||
subcategory: str
|
||||
expiration_date: date
|
||||
expiration_date: Optional[float]
|
||||
|
||||
def __post_init__(self):
|
||||
self.item_size = Decimal(self.item_size) if self.item_size else None
|
||||
self.count = Decimal(self.count) if self.count else None
|
||||
self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None
|
||||
|
||||
|
||||
@dataclass
|
||||
class ItemPutRequest:
|
||||
id: int # Ignored in favor of URL
|
||||
brand: str
|
||||
name: str
|
||||
item_size: Decimal
|
||||
unit: str
|
||||
count: Decimal
|
||||
serving: Decimal
|
||||
category: str
|
||||
subcategory: str
|
||||
expiration_date: date
|
||||
purchase_date: date
|
||||
created_at: datetime
|
||||
deleted_at: datetime
|
||||
brand: Optional[str]
|
||||
name: Optional[str]
|
||||
item_size: Union[str, Decimal]
|
||||
unit: Optional[str]
|
||||
count: Union[str, Decimal]
|
||||
serving: Union[str, Decimal, None]
|
||||
category: Optional[str]
|
||||
subcategory: Optional[str]
|
||||
expiration_date: Optional[float]
|
||||
purchase_date: Optional[float]
|
||||
inventory_id: int
|
||||
created_at: Optional[float]
|
||||
deleted_at: Optional[float]
|
||||
|
||||
def __post_init__(self):
|
||||
self.item_size = Decimal(self.item_size) if self.item_size else None
|
||||
self.count = Decimal(self.count) if self.count else None
|
||||
self.serving = Decimal(self.serving) if self.serving else None
|
||||
self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None
|
||||
self.created_at = from_timestamp(self.created_at) if self.created_at else None
|
||||
self.deleted_at = from_timestamp(self.deleted_at) if self.deleted_at else None
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
from modules.inventory import Item
|
||||
|
|
@ -49,3 +50,47 @@ def test_delete_item(test_session, init_test_inventory, test_client):
|
|||
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
||||
test_session.refresh(item)
|
||||
assert item.deleted_at
|
||||
|
||||
|
||||
def test_item_api(test_session, init_test_inventory, test_client):
|
||||
"""An Item can be added to an Inventory."""
|
||||
inventory_id = init_test_inventory.id
|
||||
item = {'brand': 'Salty', 'name': 'Salt', 'item_size': '1', 'unit': 'lbs', 'count': '5',
|
||||
'category': 'cooking ingredients', 'subcategory': 'salt', 'expiration_date': None}
|
||||
request, response = test_client.post(f'/api/inventory/{inventory_id}/item', content=json.dumps(item))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
||||
|
||||
# Get the Item we just created.
|
||||
request, response = test_client.get(f'/api/inventory/{inventory_id}/item')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
# assert item == response.json['items'][0]
|
||||
item = response.json['items'][0]
|
||||
|
||||
expiration_dates = (12345.12345, None)
|
||||
for expiration_date in expiration_dates:
|
||||
item['expiration_date'] = expiration_date
|
||||
request, response = test_client.put(f'/api/inventory/item/{item["id"]}', content=json.dumps(item))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
|
||||
|
||||
|
||||
def test_inventory_api(test_session, test_client):
|
||||
"""An Inventory can be created."""
|
||||
inventory = dict(name='foo')
|
||||
request, response = test_client.post('/api/inventory', content=json.dumps(inventory))
|
||||
assert response.status_code == HTTPStatus.CREATED, response.status_code
|
||||
|
||||
request, response = test_client.get('/api/inventory')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
inventory = response.json['inventories'][0]
|
||||
inventory_id = inventory['id']
|
||||
|
||||
inventory = dict(name='new name')
|
||||
request, response = test_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
||||
|
||||
request, response = test_client.delete(f'/api/inventory/{inventory_id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
|
||||
|
||||
request, response = test_client.get('/api/inventory')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert len(response.json['inventories']) == 0
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
import os
|
||||
import tempfile
|
||||
from decimal import Decimal
|
||||
from decimal import Decimal
|
||||
from itertools import zip_longest
|
||||
from typing import List, Iterable
|
||||
|
|
@ -15,7 +14,7 @@ from wrolpi.errors import NoInventories, InventoriesVersionMismatch
|
|||
from wrolpi.test.common import PytestCase
|
||||
from .. import init
|
||||
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
|
||||
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file
|
||||
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config
|
||||
from ..inventory import unit_registry, \
|
||||
get_inventories, save_inventory, update_inventory, \
|
||||
delete_inventory, get_categories
|
||||
|
|
@ -189,75 +188,6 @@ class TestInventory(PytestCase):
|
|||
for idx, (i, j) in enumerate(zip_longest(inventory, expected)):
|
||||
self.assertEqual(i, j, f'named inventory {idx} is not equal')
|
||||
|
||||
def test_inventories_file(self, test_session):
|
||||
self.prepare()
|
||||
|
||||
with tempfile.NamedTemporaryFile() as tf:
|
||||
# Can't import an empty file.
|
||||
self.assertRaises(ValueError, import_inventories_file, tf.name)
|
||||
|
||||
save_inventories_file(tf.name)
|
||||
|
||||
# Clear out the DB so the import will be tested
|
||||
with get_db_session(commit=True) as session:
|
||||
session.query(Item).delete()
|
||||
session.query(Inventory).delete()
|
||||
|
||||
import_inventories_file(tf.name)
|
||||
|
||||
inventories = get_inventories()
|
||||
self.assertEqual(len(inventories), 1)
|
||||
# ID has increased because we did not reset the sequence when deleting from the table.
|
||||
self.assertEqual(inventories[0].id, 2)
|
||||
self.assertEqual(inventories[0].name, 'Food Storage')
|
||||
|
||||
# All items in the DB match those in the test list, except for the "deleted" item.
|
||||
non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None]
|
||||
self.assertEqual(len(non_deleted_items), len(TEST_ITEMS) - 1)
|
||||
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
|
||||
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
|
||||
self.assertEqual(test_items, db_items)
|
||||
|
||||
def test_inventories_version(self, test_session):
|
||||
"""
|
||||
You can't save over a newer version of an inventory.
|
||||
"""
|
||||
self.prepare()
|
||||
with tempfile.NamedTemporaryFile(delete=False) as tf:
|
||||
pass
|
||||
|
||||
try:
|
||||
with open(tf.name, 'rt') as fh:
|
||||
# Version is set to 1 on first save.
|
||||
save_inventories_file(tf.name)
|
||||
inventories_contents = yaml.load(fh, Loader=yaml.Loader)
|
||||
self.assertEqual(inventories_contents['version'], 1)
|
||||
|
||||
with open(tf.name, 'rt') as fh:
|
||||
# Version is incremented when saving.
|
||||
save_inventories_file(tf.name)
|
||||
inventories_contents = yaml.load(fh, Loader=yaml.Loader)
|
||||
self.assertEqual(inventories_contents['version'], 2)
|
||||
|
||||
with open(tf.name, 'wt') as fh:
|
||||
# Version is greater than what will be saved.
|
||||
inventories_contents['version'] = 5
|
||||
yaml.dump(inventories_contents, fh)
|
||||
|
||||
self.assertRaises(InventoriesVersionMismatch, save_inventories_file, tf.name)
|
||||
|
||||
finally:
|
||||
os.unlink(tf.name)
|
||||
|
||||
def test_no_inventories(self, test_session, tmp_path):
|
||||
"""
|
||||
Can't save empty inventories.
|
||||
"""
|
||||
try:
|
||||
save_inventories_file(tmp_path.name)
|
||||
except NoInventories:
|
||||
pass
|
||||
|
||||
|
||||
def quantity_to_string(quantity: Quantity) -> str:
|
||||
quantity = cleanup_quantity(quantity)
|
||||
|
|
@ -333,3 +263,66 @@ length = gallon.dimensionality
|
|||
def test_sum_by_key(items, expected):
|
||||
items = [Item(**i) for i in items]
|
||||
assert sum_by_key(items, lambda i: (i.category,)) == expected
|
||||
|
||||
|
||||
def test_no_inventories(test_session, test_directory):
|
||||
"""Can't save empty inventories."""
|
||||
try:
|
||||
save_inventories_file()
|
||||
except NoInventories:
|
||||
pass
|
||||
|
||||
|
||||
def test_inventories_version(test_session, test_directory, init_test_inventory):
|
||||
"""You can't save over a newer version of an inventory."""
|
||||
for item in TEST_ITEMS:
|
||||
item = Item(**item)
|
||||
test_session.add(item)
|
||||
test_session.commit()
|
||||
|
||||
# Version is set to 1 on first save.
|
||||
save_inventories_file()
|
||||
config = get_inventories_config()
|
||||
assert config.version == 1
|
||||
|
||||
# Version is incremented when saving.
|
||||
save_inventories_file()
|
||||
config = get_inventories_config()
|
||||
assert config.version == 2
|
||||
|
||||
# Version is greater than what will be saved.
|
||||
with config.get_file().open('wt') as fh:
|
||||
config._config['version'] = 5
|
||||
yaml.dump(config._config, fh)
|
||||
with pytest.raises(InventoriesVersionMismatch):
|
||||
save_inventories_file()
|
||||
|
||||
|
||||
def test_inventories_config(test_session, test_directory, init_test_inventory):
|
||||
for item in TEST_ITEMS:
|
||||
item = Item(**item)
|
||||
test_session.add(item)
|
||||
test_session.commit()
|
||||
|
||||
# Save the Inventories/Items that were created above.
|
||||
save_inventories_file()
|
||||
|
||||
# Clear out the DB so the import will be tested
|
||||
test_session.query(Item).delete()
|
||||
test_session.query(Inventory).delete()
|
||||
test_session.commit()
|
||||
|
||||
import_inventories_file()
|
||||
|
||||
inventories = get_inventories()
|
||||
assert len(inventories) == 1
|
||||
# ID has increased because we did not reset the sequence when deleting from the table.
|
||||
assert inventories[0].id == 2
|
||||
assert inventories[0].name == 'Food Storage'
|
||||
|
||||
# All items in the DB match those in the test list, except for the "deleted" item.
|
||||
non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None]
|
||||
assert len(non_deleted_items) == len(TEST_ITEMS) - 1
|
||||
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
|
||||
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
|
||||
assert test_items == db_items
|
||||
|
|
|
|||
|
|
@ -19,8 +19,8 @@ bp = get_blueprint('OTP', '/api/otp')
|
|||
body=schema.EncryptOTPRequest,
|
||||
)
|
||||
@validate(schema.EncryptOTPRequest)
|
||||
async def post_encrypt_otp(_: Request, data: dict):
|
||||
data = lib.encrypt_otp(data['otp'], data['plaintext'])
|
||||
async def post_encrypt_otp(_: Request, body: schema.EncryptOTPRequest):
|
||||
data = lib.encrypt_otp(body.otp, body.plaintext)
|
||||
return response.json(data)
|
||||
|
||||
|
||||
|
|
@ -30,8 +30,8 @@ async def post_encrypt_otp(_: Request, data: dict):
|
|||
body=schema.DecryptOTPRequest,
|
||||
)
|
||||
@validate(schema.DecryptOTPRequest)
|
||||
async def post_decrypt_otp(_: Request, data: dict):
|
||||
data = lib.decrypt_otp(data['otp'], data['ciphertext'])
|
||||
async def post_decrypt_otp(_: Request, body: schema.DecryptOTPRequest):
|
||||
data = lib.decrypt_otp(body.otp, body.ciphertext)
|
||||
return response.json(data)
|
||||
|
||||
|
||||
|
|
|
|||
29
modules/otp/test/test_api.py
Normal file
29
modules/otp/test/test_api.py
Normal file
|
|
@ -0,0 +1,29 @@
|
|||
import json
|
||||
from http import HTTPStatus
|
||||
|
||||
|
||||
def test_encrypt(test_client):
|
||||
body = dict(otp='asdf', plaintext='asdf')
|
||||
request, response = test_client.post('/api/otp/encrypt_otp', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json == dict(
|
||||
ciphertext='AAGK',
|
||||
otp='ASDF',
|
||||
plaintext='ASDF',
|
||||
)
|
||||
|
||||
|
||||
def test_decrypt(test_client):
|
||||
body = dict(otp='asdf', ciphertext='aagk')
|
||||
request, response = test_client.post('/api/otp/decrypt_otp', content=json.dumps(body))
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json == dict(
|
||||
ciphertext='AAGK',
|
||||
otp='ASDF',
|
||||
plaintext='ASDF',
|
||||
)
|
||||
|
||||
|
||||
def test_get_pdf(test_client):
|
||||
request, response = test_client.get('/api/otp/pdf')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
|
|
@ -12,11 +12,11 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \
|
|||
from wrolpi.common import logger
|
||||
from wrolpi.root_api import add_blueprint, json_response
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from . import lib, schema
|
||||
from .channel import lib as channel_lib
|
||||
from .channel.api import channel_bp
|
||||
from .video import lib as video_lib
|
||||
from .video.api import video_bp
|
||||
from . import lib, schema
|
||||
|
||||
content_bp = Blueprint('VideoContent', '/api/videos')
|
||||
bp = Blueprint('Videos', '/api/videos').group(
|
||||
|
|
|
|||
|
|
@ -72,7 +72,7 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
|
|||
@validate(schema.ChannelPutRequest)
|
||||
@wrol_mode_check
|
||||
def channel_update(_: Request, link: str, body: schema.ChannelPutRequest):
|
||||
channel = lib.update_channel(body.__dict__, link) # TODO don't use the dataclass as a dict.
|
||||
channel = lib.update_channel(body.__dict__, link)
|
||||
return response.raw('', HTTPStatus.NO_CONTENT,
|
||||
headers={'Location': f'/api/videos/channels/{channel.link}'})
|
||||
|
||||
|
|
|
|||
|
|
@ -14,7 +14,7 @@ from sqlalchemy.orm import Session
|
|||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import logger, get_config, iterify, get_media_directory, \
|
||||
from wrolpi.common import logger, iterify, get_media_directory, \
|
||||
minimize_dict, api_param_limiter
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
|
||||
|
|
@ -36,19 +36,6 @@ VIDEO_QUERY_MAX_LIMIT = 100
|
|||
# These are the supported video formats. These are in order of their preference.
|
||||
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
|
||||
|
||||
|
||||
def load_downloader_config() -> dict:
|
||||
config = get_config()
|
||||
downloader = config['downloader']
|
||||
return downloader
|
||||
|
||||
|
||||
def load_channels_config() -> dict:
|
||||
config = get_config()
|
||||
channels = config['channels']
|
||||
return channels
|
||||
|
||||
|
||||
video_limiter = api_param_limiter(100)
|
||||
|
||||
|
||||
|
|
@ -81,8 +68,9 @@ def import_videos_config():
|
|||
"""Import channel settings to the DB. Existing channels will be updated."""
|
||||
logger.info('Importing videos config')
|
||||
try:
|
||||
config = get_config()
|
||||
config, favorites = config['channels'], config.get('favorites', {})
|
||||
from .lib import get_channels_config
|
||||
config = get_channels_config()
|
||||
config, favorites = config.channels, config.favorites
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
for link in config:
|
||||
|
|
|
|||
8
modules/videos/configs/channels.yaml
Normal file
8
modules/videos/configs/channels.yaml
Normal file
|
|
@ -0,0 +1,8 @@
|
|||
# This is an example config. Do not edit this file. Edit the config in your media directory!
|
||||
channels:
|
||||
wrolpi:
|
||||
name: WROLPi
|
||||
url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos
|
||||
directory: videos/wrolpi
|
||||
download_frequency: 604800
|
||||
favorites: [ ]
|
||||
12
modules/videos/configs/downloader.yaml
Normal file
12
modules/videos/configs/downloader.yaml
Normal file
|
|
@ -0,0 +1,12 @@
|
|||
# Most of these names were decided by YT-DLP project, we pass these to any calls to that project.
|
||||
continue_dl: true
|
||||
dateafter: 19900101
|
||||
file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s"
|
||||
format: 22/135/18/43
|
||||
nooverwrites: true
|
||||
quiet: false
|
||||
writeautomaticsub: true
|
||||
writeinfojson: true
|
||||
writesubtitles: true
|
||||
writethumbnail: true
|
||||
youtube_include_dash_manifest: false
|
||||
|
|
@ -8,6 +8,7 @@ import pytest
|
|||
from PIL import Image
|
||||
|
||||
from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
||||
from modules.videos.lib import set_test_channels_config
|
||||
from modules.videos.models import Channel, Video
|
||||
from wrolpi.common import sanitize_link
|
||||
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
|
||||
|
|
@ -152,3 +153,12 @@ def video_download_manager(test_download_manager) -> DownloadManager:
|
|||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: # prevent real downloads
|
||||
mock_extract_info.side_effect = Exception('You must patch modules.videos.downloader.YDL.extract_info!')
|
||||
yield test_download_manager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def test_channels_config(test_directory):
|
||||
(test_directory / 'config').mkdir(exist_ok=True)
|
||||
config_path = test_directory / 'config/channels.yaml'
|
||||
set_test_channels_config(True)
|
||||
yield config_path
|
||||
set_test_channels_config(False)
|
||||
|
|
|
|||
|
|
@ -18,9 +18,9 @@ from wrolpi.downloader import Downloader, Download
|
|||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \
|
||||
from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \
|
||||
get_videos_directory
|
||||
from .lib import upsert_video, refresh_channel_videos
|
||||
from .lib import upsert_video, refresh_channel_videos, get_downloader_config
|
||||
from .models import Video, Channel
|
||||
from .video_url_resolver import video_url_resolver
|
||||
|
||||
|
|
@ -217,11 +217,11 @@ class VideoDownloader(Downloader, ABC):
|
|||
raise ValueError(f'Output directory does not exist! {out_dir=}')
|
||||
|
||||
# YoutubeDL expects specific options, add onto the default options
|
||||
config = load_downloader_config()
|
||||
config = get_downloader_config().dict()
|
||||
options = config.copy()
|
||||
options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
|
||||
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
||||
# We don't need format when getting the filename.
|
||||
# We don't need "format" when getting the filename.
|
||||
del options['format']
|
||||
|
||||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
|
|
|
|||
|
|
@ -8,8 +8,7 @@ from uuid import uuid1
|
|||
from sqlalchemy.exc import InvalidRequestError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import logger, chunks, get_config
|
||||
from wrolpi.common import save_settings_config
|
||||
from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig
|
||||
from wrolpi.dates import from_timestamp, Seconds
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.media_path import MediaPath
|
||||
|
|
@ -289,10 +288,186 @@ def refresh_videos(channel_links: list = None):
|
|||
logger.info('Refresh of video files complete')
|
||||
|
||||
|
||||
def get_channels_config(session: Session) -> dict:
|
||||
"""
|
||||
Create a dictionary that contains all the Channels from the DB.
|
||||
"""
|
||||
class ChannelsConfig(ConfigFile):
|
||||
file_name = 'channels.yaml'
|
||||
default_config = dict(
|
||||
channels={
|
||||
'wrolpi': dict(
|
||||
name='WROLPi',
|
||||
url='https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos',
|
||||
directory='videos/wrolpi',
|
||||
download_frequency=604800,
|
||||
)
|
||||
},
|
||||
favorites=dict(),
|
||||
)
|
||||
|
||||
@property
|
||||
def channels(self) -> dict:
|
||||
return self._config['channels']
|
||||
|
||||
@channels.setter
|
||||
def channels(self, value: dict):
|
||||
self.update({'channels': value})
|
||||
|
||||
@property
|
||||
def favorites(self) -> dict:
|
||||
return self._config['favorites']
|
||||
|
||||
@favorites.setter
|
||||
def favorites(self, value: dict):
|
||||
self.update({'favorites': value})
|
||||
|
||||
|
||||
CHANNELS_CONFIG: ChannelsConfig = None
|
||||
TEST_CHANNELS_CONFIG: ChannelsConfig = None
|
||||
|
||||
|
||||
def get_channels_config():
|
||||
global TEST_CHANNELS_CONFIG
|
||||
if isinstance(TEST_CHANNELS_CONFIG, ConfigFile):
|
||||
return TEST_CHANNELS_CONFIG
|
||||
|
||||
global CHANNELS_CONFIG
|
||||
if isinstance(CHANNELS_CONFIG, ConfigFile):
|
||||
return CHANNELS_CONFIG
|
||||
|
||||
CHANNELS_CONFIG = ChannelsConfig()
|
||||
return CHANNELS_CONFIG
|
||||
|
||||
|
||||
def set_test_channels_config(enable: bool):
|
||||
global TEST_CHANNELS_CONFIG
|
||||
if enable is True:
|
||||
TEST_CHANNELS_CONFIG = ChannelsConfig()
|
||||
else:
|
||||
TEST_CHANNELS_CONFIG = None
|
||||
|
||||
|
||||
class DownloaderConfig(ConfigFile):
|
||||
file_name = 'downloader.yaml'
|
||||
default_config = dict(
|
||||
continue_dl=True,
|
||||
dateafter='19900101',
|
||||
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
|
||||
format='22 / 135 / 18 / 43',
|
||||
nooverwrites=True,
|
||||
quiet=False,
|
||||
writeautomaticsub=True,
|
||||
writeinfojson=True,
|
||||
writesubtitles=True,
|
||||
writethumbnail=True,
|
||||
youtube_include_dash_manifest=False,
|
||||
)
|
||||
|
||||
@property
|
||||
def continue_dl(self) -> bool:
|
||||
return self._config['continue_dl']
|
||||
|
||||
@continue_dl.setter
|
||||
def continue_dl(self, value: bool):
|
||||
self.update({'continue_dl': value})
|
||||
|
||||
@property
|
||||
def dateafter(self) -> str:
|
||||
return self._config['dateafter']
|
||||
|
||||
@dateafter.setter
|
||||
def dateafter(self, value: str):
|
||||
self.update({'dateafter': value})
|
||||
|
||||
@property
|
||||
def file_name_format(self) -> str:
|
||||
return self._config['file_name_format']
|
||||
|
||||
@file_name_format.setter
|
||||
def file_name_format(self, value: str):
|
||||
self.update({'file_name_format': value})
|
||||
|
||||
@property
|
||||
def format(self) -> str:
|
||||
return self._config['format']
|
||||
|
||||
@format.setter
|
||||
def format(self, value: str):
|
||||
self.update({'format': value})
|
||||
|
||||
@property
|
||||
def nooverwrites(self) -> bool:
|
||||
return self._config['nooverwrites']
|
||||
|
||||
@nooverwrites.setter
|
||||
def nooverwrites(self, value: bool):
|
||||
self.update({'nooverwrites': value})
|
||||
|
||||
@property
|
||||
def quiet(self) -> bool:
|
||||
return self._config['quiet']
|
||||
|
||||
@quiet.setter
|
||||
def quiet(self, value: bool):
|
||||
self.update({'quiet': value})
|
||||
|
||||
@property
|
||||
def writeautomaticsub(self) -> bool:
|
||||
return self._config['writeautomaticsub']
|
||||
|
||||
@writeautomaticsub.setter
|
||||
def writeautomaticsub(self, value: bool):
|
||||
self.update({'writeautomaticsub': value})
|
||||
|
||||
@property
|
||||
def writeinfojson(self) -> bool:
|
||||
return self._config['writeinfojson']
|
||||
|
||||
@writeinfojson.setter
|
||||
def writeinfojson(self, value: bool):
|
||||
self.update({'writeinfojson': value})
|
||||
|
||||
@property
|
||||
def writesubtitles(self) -> bool:
|
||||
return self._config['writesubtitles']
|
||||
|
||||
@writesubtitles.setter
|
||||
def writesubtitles(self, value: bool):
|
||||
self.update({'writesubtitles': value})
|
||||
|
||||
@property
|
||||
def writethumbnail(self) -> bool:
|
||||
return self._config['writethumbnail']
|
||||
|
||||
@writethumbnail.setter
|
||||
def writethumbnail(self, value: bool):
|
||||
self.update({'writethumbnail': value})
|
||||
|
||||
@property
|
||||
def youtube_include_dash_manifest(self) -> bool:
|
||||
return self._config['youtube_include_dash_manifest']
|
||||
|
||||
@youtube_include_dash_manifest.setter
|
||||
def youtube_include_dash_manifest(self, value: bool):
|
||||
self.update({'youtube_include_dash_manifest': value})
|
||||
|
||||
|
||||
DOWNLOADER_CONFIG: DownloaderConfig = None
|
||||
TEST_DOWNLOADER_CONFIG: DownloaderConfig = None
|
||||
|
||||
|
||||
def get_downloader_config():
|
||||
global TEST_DOWNLOADER_CONFIG
|
||||
if isinstance(TEST_DOWNLOADER_CONFIG, WROLPiConfig):
|
||||
return TEST_DOWNLOADER_CONFIG
|
||||
|
||||
global DOWNLOADER_CONFIG
|
||||
if isinstance(DOWNLOADER_CONFIG, WROLPiConfig):
|
||||
return DOWNLOADER_CONFIG
|
||||
|
||||
DOWNLOADER_CONFIG = DownloaderConfig()
|
||||
return DOWNLOADER_CONFIG
|
||||
|
||||
|
||||
def get_channels_config_from_db(session: Session) -> dict:
|
||||
"""Create a dictionary that contains all the Channels from the DB."""
|
||||
channels = session.query(Channel).order_by(Channel.link).all()
|
||||
channels = {i.link: i.config_view() for i in channels}
|
||||
|
||||
|
|
@ -315,10 +490,11 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
|
|||
"""
|
||||
Pull the Channel information from the DB, save it to the config.
|
||||
"""
|
||||
config = get_channels_config(session)
|
||||
config = get_channels_config_from_db(session)
|
||||
channels_config = get_channels_config()
|
||||
if preserve_favorites:
|
||||
config['favorites'].update(get_config().get('favorites', {}))
|
||||
save_settings_config(config)
|
||||
config['favorites'].update(channels_config.favorites or {})
|
||||
channels_config.update(config)
|
||||
|
||||
|
||||
async def get_statistics():
|
||||
|
|
|
|||
|
|
@ -11,7 +11,7 @@ from PIL import Image
|
|||
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.test.common import create_channel_structure
|
||||
from wrolpi.common import get_absolute_media_path, get_config
|
||||
from wrolpi.common import get_absolute_media_path
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
|
||||
|
|
@ -19,7 +19,7 @@ from wrolpi.vars import PROJECT_DIR
|
|||
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
|
||||
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
|
||||
import_videos_config
|
||||
from ..lib import save_channels_config
|
||||
from ..lib import save_channels_config, get_channels_config
|
||||
|
||||
|
||||
class TestCommon(TestAPI):
|
||||
|
|
@ -309,7 +309,7 @@ def test_update_censored_videos(test_session, video_factory, simple_channel):
|
|||
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
|
||||
def test_import_favorites(test_session, simple_channel, video_factory):
|
||||
def test_import_favorites(test_session, simple_channel, video_factory, test_channels_config):
|
||||
"""
|
||||
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
|
||||
favorited again.
|
||||
|
|
@ -331,8 +331,8 @@ def test_import_favorites(test_session, simple_channel, video_factory):
|
|||
|
||||
# Save config, verify that favorite is set.
|
||||
save_channels_config()
|
||||
config = get_config()
|
||||
assert config['favorites'] == favorites
|
||||
config = get_channels_config()
|
||||
assert config.favorites == favorites
|
||||
|
||||
def import_and_verify(favorited_ids: List[int]):
|
||||
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
|
||||
|
|
@ -351,8 +351,8 @@ def test_import_favorites(test_session, simple_channel, video_factory):
|
|||
test_session.query(Video).filter_by(id=vid2.id).delete()
|
||||
test_session.commit()
|
||||
save_channels_config()
|
||||
config = get_config()
|
||||
assert config['favorites'] == favorites
|
||||
config = get_channels_config()
|
||||
assert config.favorites == favorites
|
||||
import_and_verify([vid3.id])
|
||||
|
||||
# Add vid2 again, it should be favorited on import.
|
||||
|
|
@ -363,6 +363,6 @@ def test_import_favorites(test_session, simple_channel, video_factory):
|
|||
|
||||
# Deleting the Video in the model really removes the favorite status.
|
||||
vid3.delete()
|
||||
config = get_config()
|
||||
assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
|
||||
config = get_channels_config()
|
||||
assert config.favorites == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
|
||||
import_and_verify([vid2.id])
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue