Squashing config rework.

This commit is contained in:
Roland 2022-02-16 21:57:27 -07:00
parent 5425142adb
commit 08065155cf
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
38 changed files with 1046 additions and 633 deletions

View file

@ -5,7 +5,6 @@ from pathlib import Path
from typing import List
from uuid import uuid4
import magic
import psycopg2
from sqlalchemy.orm import Session
@ -16,6 +15,13 @@ from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
from wrolpi.errors import InvalidFile
from wrolpi.vars import PYTEST
try:
import magic
have_magic = True
except ImportError:
logger.error('Could not import magic. Is libmagic installed?')
have_magic = False
logger = logger.getChild(__name__)
__all__ = ['list_files', 'delete_file', 'split_file_name', 'upsert_file', 'refresh_files', 'search']
@ -96,7 +102,7 @@ def upsert_file(path: Path, session: Session) -> File:
if not file:
file = File(path=path)
session.add(file)
if not file.mimetype:
if not file.mimetype and have_magic:
file.mimetype = magic.from_file(str(path), mime=True)
if not file.title:
file.title = split_file_name(path)

View file

@ -5,7 +5,7 @@ from sanic.request import Request
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.common import run_after, remove_dict_value_whitespace
from wrolpi.common import run_after, recursive_map
from wrolpi.errors import ValidationError
from wrolpi.root_api import get_blueprint, json_response
from . import common, inventory, schema
@ -48,9 +48,8 @@ def inventory_get(_: Request, inventory_id: int):
)
@validate(schema.InventoryPostRequest)
@run_after(common.save_inventories_file)
def post_inventory(_: Request, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def post_inventory(_: Request, body: schema.InventoryPostRequest):
data = remove_whitespace(body.__dict__)
inventory.save_inventory(data)
return response.empty(HTTPStatus.CREATED)
@ -63,9 +62,8 @@ def post_inventory(_: Request, data: dict):
)
@validate(schema.InventoryPutRequest)
@run_after(common.save_inventories_file)
def put_inventory(_: Request, inventory_id: int, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def put_inventory(_: Request, inventory_id: int, body: schema.InventoryPutRequest):
data = remove_whitespace(body.__dict__)
inventory.update_inventory(inventory_id, data)
return response.empty()
@ -93,9 +91,8 @@ def items_get(_: Request, inventory_id: int):
)
@validate(schema.ItemPostRequest)
@run_after(common.save_inventories_file)
def post_item(_: Request, inventory_id: int, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def post_item(_: Request, inventory_id: int, body: schema.ItemPostRequest):
data = remove_whitespace(body.__dict__)
inventory.save_item(inventory_id, data)
return response.empty()
@ -108,9 +105,8 @@ def post_item(_: Request, inventory_id: int, data: dict):
)
@validate(schema.ItemPutRequest)
@run_after(common.save_inventories_file)
def put_item(_: Request, item_id: int, data: dict):
# Cleanup the whitespace.
data = remove_dict_value_whitespace(data)
def put_item(_: Request, item_id: int, body: schema.ItemPutRequest):
data = remove_whitespace(body.__dict__)
inventory.update_item(item_id, data)
return response.empty()
@ -131,3 +127,7 @@ def item_delete(_: Request, item_ids: str):
inventory.delete_items(item_ids)
return response.empty()
def remove_whitespace(obj):
return recursive_map(obj, lambda i: i.strip() if hasattr(i, 'strip') else i)

View file

@ -3,14 +3,12 @@ from functools import partial
from pathlib import Path
from typing import List, Tuple
import yaml
from pint import Quantity
from wrolpi import before_startup
from wrolpi.common import logger, Base
from wrolpi.common import logger, Base, ConfigFile
from wrolpi.db import get_db_session
from wrolpi.errors import NoInventories, InventoriesVersionMismatch
from wrolpi.vars import CONFIG_DIR
from .inventory import unit_registry, get_items, get_inventories, increment_inventories_version, \
get_inventories_version
from .models import Inventory, Item
@ -98,23 +96,65 @@ def quantity_to_tuple(quantity: unit_registry.Quantity) -> Tuple[Decimal, Quanti
def cleanup_quantity(quantity: Quantity) -> Quantity:
"""
Remove trailing zeros from a Quantity.
"""
"""Remove trailing zeros from a Quantity."""
num, unit = quantity_to_tuple(quantity)
num = round(num, UNIT_PRECISION)
num = str(num).rstrip('0').rstrip('.')
return Decimal(num) * unit
DEFAULT_SAVE_PATH = CONFIG_DIR / 'inventories.yaml'
class InventoriesConfig(ConfigFile):
file_name = 'inventories.yaml'
default_config = dict(
inventories=[],
version=1,
)
@property
def inventories(self) -> dict:
return self._config['inventories']
@inventories.setter
def inventories(self, value: dict):
self.update({'inventories': value})
@property
def version(self) -> int:
return self._config['version']
@version.setter
def version(self, value: int):
self.update({'version': value})
def save_inventories_file(path: str = None):
"""
Write all inventories and their respective items to a YAML file.
"""
path: Path = Path(path) if path else DEFAULT_SAVE_PATH
INVENTORIES_CONFIG: InventoriesConfig = None
TEST_INVENTORIES_CONFIG: InventoriesConfig = None
def get_inventories_config():
global TEST_INVENTORIES_CONFIG
if isinstance(TEST_INVENTORIES_CONFIG, ConfigFile):
return TEST_INVENTORIES_CONFIG
global INVENTORIES_CONFIG
if isinstance(INVENTORIES_CONFIG, ConfigFile):
return INVENTORIES_CONFIG
INVENTORIES_CONFIG = InventoriesConfig()
return INVENTORIES_CONFIG
def set_test_inventories_config(enabled: bool):
global TEST_INVENTORIES_CONFIG
if enabled:
TEST_INVENTORIES_CONFIG = InventoriesConfig()
else:
TEST_INVENTORIES_CONFIG = None
def save_inventories_file():
"""Write all inventories and their respective items to a YAML file."""
config = get_inventories_config()
inventories = []
for inventory in get_inventories():
@ -124,44 +164,22 @@ def save_inventories_file(path: str = None):
raise NoInventories('No Inventories are in the database!')
with increment_inventories_version() as version:
if path.is_file():
# Check that we aren't overwriting our inventories with empty inventories.
with open(path, 'rt') as fh:
old = yaml.load(fh, Loader=yaml.Loader)
if old and not inventories:
raise FileExistsError(f'Refusing to overwrite non-empty inventories.yaml with empty inventories.'
f' {path}')
if config.version and config.version > version:
raise InventoriesVersionMismatch(
f'Inventories config version is {config.version} but DB version is {get_inventories_version()}')
if old and old.get('version') > version:
raise InventoriesVersionMismatch(
f'Inventories config version is {old["version"]} but DB version is {get_inventories_version()}')
with open(path, 'wt') as fh:
contents = dict(
version=version,
inventories=inventories,
)
yaml.dump(contents, fh)
config.inventories = inventories
config.version = version
config.save()
@before_startup
def import_inventories_file(path: str = None):
path: Path = Path(path) if path else DEFAULT_SAVE_PATH
if not path.is_file():
logger.warning(f'No inventories config file at {path}')
return
with open(path, 'rt') as fh:
contents = yaml.load(fh, Loader=yaml.Loader)
if not contents or 'inventories' not in contents:
raise ValueError('Inventories file does not contain the expected "inventories" list.')
def import_inventories_file():
config = get_inventories_config()
inventories = get_inventories()
inventories_names = {i.name for i in inventories}
new_inventories = [i for i in contents['inventories'] if i['name'] not in inventories_names]
new_inventories = [i for i in config.inventories if i['name'] not in inventories_names]
with get_db_session(commit=True) as session:
for inventory in new_inventories:
items = inventory['items']

View file

@ -1,6 +1,7 @@
import pytest
from modules.inventory import Inventory, Item, DEFAULT_CATEGORIES, DEFAULT_INVENTORIES
from modules.inventory.common import set_test_inventories_config
@pytest.fixture
@ -23,4 +24,6 @@ def init_test_inventory(test_session):
inventory = test_session.query(Inventory).filter_by(name='Food Storage').one()
test_session.commit()
return inventory
set_test_inventories_config(True)
yield inventory
set_test_inventories_config(False)

View file

@ -59,7 +59,7 @@ DEFAULT_INVENTORIES = [
sort_categories = partial(sorted, key=itemgetter(1, 0))
def get_inventories() -> List[Base]:
def get_inventories() -> List[Inventory]:
with get_db_session() as session:
inventories = session.query(Inventory).filter(
Inventory.deleted_at == None, # noqa

View file

@ -1,36 +1,51 @@
from dataclasses import dataclass
from datetime import date, datetime
from decimal import Decimal
from typing import Union, Optional
from wrolpi.dates import from_timestamp
@dataclass
class ItemPostRequest:
brand: str
name: str
item_size: Decimal
item_size: Union[str, Decimal]
unit: str
count: Decimal
count: Union[str, Decimal]
category: str
subcategory: str
expiration_date: date
expiration_date: Optional[float]
def __post_init__(self):
self.item_size = Decimal(self.item_size) if self.item_size else None
self.count = Decimal(self.count) if self.count else None
self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None
@dataclass
class ItemPutRequest:
id: int # Ignored in favor of URL
brand: str
name: str
item_size: Decimal
unit: str
count: Decimal
serving: Decimal
category: str
subcategory: str
expiration_date: date
purchase_date: date
created_at: datetime
deleted_at: datetime
brand: Optional[str]
name: Optional[str]
item_size: Union[str, Decimal]
unit: Optional[str]
count: Union[str, Decimal]
serving: Union[str, Decimal, None]
category: Optional[str]
subcategory: Optional[str]
expiration_date: Optional[float]
purchase_date: Optional[float]
inventory_id: int
created_at: Optional[float]
deleted_at: Optional[float]
def __post_init__(self):
self.item_size = Decimal(self.item_size) if self.item_size else None
self.count = Decimal(self.count) if self.count else None
self.serving = Decimal(self.serving) if self.serving else None
self.expiration_date = from_timestamp(self.expiration_date) if self.expiration_date else None
self.created_at = from_timestamp(self.created_at) if self.created_at else None
self.deleted_at = from_timestamp(self.deleted_at) if self.deleted_at else None
@dataclass

View file

@ -1,3 +1,4 @@
import json
from http import HTTPStatus
from modules.inventory import Item
@ -49,3 +50,47 @@ def test_delete_item(test_session, init_test_inventory, test_client):
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
test_session.refresh(item)
assert item.deleted_at
def test_item_api(test_session, init_test_inventory, test_client):
"""An Item can be added to an Inventory."""
inventory_id = init_test_inventory.id
item = {'brand': 'Salty', 'name': 'Salt', 'item_size': '1', 'unit': 'lbs', 'count': '5',
'category': 'cooking ingredients', 'subcategory': 'salt', 'expiration_date': None}
request, response = test_client.post(f'/api/inventory/{inventory_id}/item', content=json.dumps(item))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
# Get the Item we just created.
request, response = test_client.get(f'/api/inventory/{inventory_id}/item')
assert response.status_code == HTTPStatus.OK
# assert item == response.json['items'][0]
item = response.json['items'][0]
expiration_dates = (12345.12345, None)
for expiration_date in expiration_dates:
item['expiration_date'] = expiration_date
request, response = test_client.put(f'/api/inventory/item/{item["id"]}', content=json.dumps(item))
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
def test_inventory_api(test_session, test_client):
"""An Inventory can be created."""
inventory = dict(name='foo')
request, response = test_client.post('/api/inventory', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.CREATED, response.status_code
request, response = test_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
inventory = response.json['inventories'][0]
inventory_id = inventory['id']
inventory = dict(name='new name')
request, response = test_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = test_client.delete(f'/api/inventory/{inventory_id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = test_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
assert len(response.json['inventories']) == 0

View file

@ -1,5 +1,4 @@
import os
import tempfile
from decimal import Decimal
from decimal import Decimal
from itertools import zip_longest
from typing import List, Iterable
@ -15,7 +14,7 @@ from wrolpi.errors import NoInventories, InventoriesVersionMismatch
from wrolpi.test.common import PytestCase
from .. import init
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file
compact_unit, cleanup_quantity, save_inventories_file, import_inventories_file, get_inventories_config
from ..inventory import unit_registry, \
get_inventories, save_inventory, update_inventory, \
delete_inventory, get_categories
@ -189,75 +188,6 @@ class TestInventory(PytestCase):
for idx, (i, j) in enumerate(zip_longest(inventory, expected)):
self.assertEqual(i, j, f'named inventory {idx} is not equal')
def test_inventories_file(self, test_session):
self.prepare()
with tempfile.NamedTemporaryFile() as tf:
# Can't import an empty file.
self.assertRaises(ValueError, import_inventories_file, tf.name)
save_inventories_file(tf.name)
# Clear out the DB so the import will be tested
with get_db_session(commit=True) as session:
session.query(Item).delete()
session.query(Inventory).delete()
import_inventories_file(tf.name)
inventories = get_inventories()
self.assertEqual(len(inventories), 1)
# ID has increased because we did not reset the sequence when deleting from the table.
self.assertEqual(inventories[0].id, 2)
self.assertEqual(inventories[0].name, 'Food Storage')
# All items in the DB match those in the test list, except for the "deleted" item.
non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None]
self.assertEqual(len(non_deleted_items), len(TEST_ITEMS) - 1)
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
self.assertEqual(test_items, db_items)
def test_inventories_version(self, test_session):
"""
You can't save over a newer version of an inventory.
"""
self.prepare()
with tempfile.NamedTemporaryFile(delete=False) as tf:
pass
try:
with open(tf.name, 'rt') as fh:
# Version is set to 1 on first save.
save_inventories_file(tf.name)
inventories_contents = yaml.load(fh, Loader=yaml.Loader)
self.assertEqual(inventories_contents['version'], 1)
with open(tf.name, 'rt') as fh:
# Version is incremented when saving.
save_inventories_file(tf.name)
inventories_contents = yaml.load(fh, Loader=yaml.Loader)
self.assertEqual(inventories_contents['version'], 2)
with open(tf.name, 'wt') as fh:
# Version is greater than what will be saved.
inventories_contents['version'] = 5
yaml.dump(inventories_contents, fh)
self.assertRaises(InventoriesVersionMismatch, save_inventories_file, tf.name)
finally:
os.unlink(tf.name)
def test_no_inventories(self, test_session, tmp_path):
"""
Can't save empty inventories.
"""
try:
save_inventories_file(tmp_path.name)
except NoInventories:
pass
def quantity_to_string(quantity: Quantity) -> str:
quantity = cleanup_quantity(quantity)
@ -333,3 +263,66 @@ length = gallon.dimensionality
def test_sum_by_key(items, expected):
items = [Item(**i) for i in items]
assert sum_by_key(items, lambda i: (i.category,)) == expected
def test_no_inventories(test_session, test_directory):
"""Can't save empty inventories."""
try:
save_inventories_file()
except NoInventories:
pass
def test_inventories_version(test_session, test_directory, init_test_inventory):
"""You can't save over a newer version of an inventory."""
for item in TEST_ITEMS:
item = Item(**item)
test_session.add(item)
test_session.commit()
# Version is set to 1 on first save.
save_inventories_file()
config = get_inventories_config()
assert config.version == 1
# Version is incremented when saving.
save_inventories_file()
config = get_inventories_config()
assert config.version == 2
# Version is greater than what will be saved.
with config.get_file().open('wt') as fh:
config._config['version'] = 5
yaml.dump(config._config, fh)
with pytest.raises(InventoriesVersionMismatch):
save_inventories_file()
def test_inventories_config(test_session, test_directory, init_test_inventory):
for item in TEST_ITEMS:
item = Item(**item)
test_session.add(item)
test_session.commit()
# Save the Inventories/Items that were created above.
save_inventories_file()
# Clear out the DB so the import will be tested
test_session.query(Item).delete()
test_session.query(Inventory).delete()
test_session.commit()
import_inventories_file()
inventories = get_inventories()
assert len(inventories) == 1
# ID has increased because we did not reset the sequence when deleting from the table.
assert inventories[0].id == 2
assert inventories[0].name == 'Food Storage'
# All items in the DB match those in the test list, except for the "deleted" item.
non_deleted_items = [i for i in inventories[0].items if i.deleted_at is None]
assert len(non_deleted_items) == len(TEST_ITEMS) - 1
test_items = {(i['name'], i['brand'], i['count']) for i in TEST_ITEMS[:-1]}
db_items = {(i.name, i.brand, i.count) for i in non_deleted_items}
assert test_items == db_items

View file

@ -19,8 +19,8 @@ bp = get_blueprint('OTP', '/api/otp')
body=schema.EncryptOTPRequest,
)
@validate(schema.EncryptOTPRequest)
async def post_encrypt_otp(_: Request, data: dict):
data = lib.encrypt_otp(data['otp'], data['plaintext'])
async def post_encrypt_otp(_: Request, body: schema.EncryptOTPRequest):
data = lib.encrypt_otp(body.otp, body.plaintext)
return response.json(data)
@ -30,8 +30,8 @@ async def post_encrypt_otp(_: Request, data: dict):
body=schema.DecryptOTPRequest,
)
@validate(schema.DecryptOTPRequest)
async def post_decrypt_otp(_: Request, data: dict):
data = lib.decrypt_otp(data['otp'], data['ciphertext'])
async def post_decrypt_otp(_: Request, body: schema.DecryptOTPRequest):
data = lib.decrypt_otp(body.otp, body.ciphertext)
return response.json(data)

View file

@ -0,0 +1,29 @@
import json
from http import HTTPStatus
def test_encrypt(test_client):
body = dict(otp='asdf', plaintext='asdf')
request, response = test_client.post('/api/otp/encrypt_otp', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json == dict(
ciphertext='AAGK',
otp='ASDF',
plaintext='ASDF',
)
def test_decrypt(test_client):
body = dict(otp='asdf', ciphertext='aagk')
request, response = test_client.post('/api/otp/decrypt_otp', content=json.dumps(body))
assert response.status_code == HTTPStatus.OK
assert response.json == dict(
ciphertext='AAGK',
otp='ASDF',
plaintext='ASDF',
)
def test_get_pdf(test_client):
request, response = test_client.get('/api/otp/pdf')
assert response.status_code == HTTPStatus.OK

View file

@ -12,11 +12,11 @@ from wrolpi.common import create_websocket_feed, get_sanic_url, \
from wrolpi.common import logger
from wrolpi.root_api import add_blueprint, json_response
from wrolpi.schema import JSONErrorResponse
from . import lib, schema
from .channel import lib as channel_lib
from .channel.api import channel_bp
from .video import lib as video_lib
from .video.api import video_bp
from . import lib, schema
content_bp = Blueprint('VideoContent', '/api/videos')
bp = Blueprint('Videos', '/api/videos').group(

View file

@ -72,7 +72,7 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@validate(schema.ChannelPutRequest)
@wrol_mode_check
def channel_update(_: Request, link: str, body: schema.ChannelPutRequest):
channel = lib.update_channel(body.__dict__, link) # TODO don't use the dataclass as a dict.
channel = lib.update_channel(body.__dict__, link)
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.link}'})

View file

@ -14,7 +14,7 @@ from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
from wrolpi import before_startup
from wrolpi.common import logger, get_config, iterify, get_media_directory, \
from wrolpi.common import logger, iterify, get_media_directory, \
minimize_dict, api_param_limiter
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
@ -36,19 +36,6 @@ VIDEO_QUERY_MAX_LIMIT = 100
# These are the supported video formats. These are in order of their preference.
VIDEO_EXTENSIONS = ('mp4', 'ogg', 'webm', 'flv')
def load_downloader_config() -> dict:
config = get_config()
downloader = config['downloader']
return downloader
def load_channels_config() -> dict:
config = get_config()
channels = config['channels']
return channels
video_limiter = api_param_limiter(100)
@ -81,8 +68,9 @@ def import_videos_config():
"""Import channel settings to the DB. Existing channels will be updated."""
logger.info('Importing videos config')
try:
config = get_config()
config, favorites = config['channels'], config.get('favorites', {})
from .lib import get_channels_config
config = get_channels_config()
config, favorites = config.channels, config.favorites
with get_db_session(commit=True) as session:
for link in config:

View file

@ -0,0 +1,8 @@
# This is an example config. Do not edit this file. Edit the config in your media directory!
channels:
wrolpi:
name: WROLPi
url: https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos
directory: videos/wrolpi
download_frequency: 604800
favorites: [ ]

View file

@ -0,0 +1,12 @@
# Most of these names were decided by YT-DLP project, we pass these to any calls to that project.
continue_dl: true
dateafter: 19900101
file_name_format: "%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s"
format: 22/135/18/43
nooverwrites: true
quiet: false
writeautomaticsub: true
writeinfojson: true
writesubtitles: true
writethumbnail: true
youtube_include_dash_manifest: false

View file

@ -8,6 +8,7 @@ import pytest
from PIL import Image
from modules.videos.downloader import VideoDownloader, ChannelDownloader
from modules.videos.lib import set_test_channels_config
from modules.videos.models import Channel, Video
from wrolpi.common import sanitize_link
from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
@ -152,3 +153,12 @@ def video_download_manager(test_download_manager) -> DownloadManager:
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info: # prevent real downloads
mock_extract_info.side_effect = Exception('You must patch modules.videos.downloader.YDL.extract_info!')
yield test_download_manager
@pytest.fixture
def test_channels_config(test_directory):
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/channels.yaml'
set_test_channels_config(True)
yield config_path
set_test_channels_config(False)

View file

@ -18,9 +18,9 @@ from wrolpi.downloader import Downloader, Download
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
from .common import load_downloader_config, apply_info_json, get_channel_source_id, get_no_channel_directory, \
from .common import apply_info_json, get_channel_source_id, get_no_channel_directory, \
get_videos_directory
from .lib import upsert_video, refresh_channel_videos
from .lib import upsert_video, refresh_channel_videos, get_downloader_config
from .models import Video, Channel
from .video_url_resolver import video_url_resolver
@ -217,11 +217,11 @@ class VideoDownloader(Downloader, ABC):
raise ValueError(f'Output directory does not exist! {out_dir=}')
# YoutubeDL expects specific options, add onto the default options
config = load_downloader_config()
config = get_downloader_config().dict()
options = config.copy()
options['outtmpl'] = f'{out_dir}/{config["file_name_format"]}'
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
# We don't need format when getting the filename.
# We don't need "format" when getting the filename.
del options['format']
logger.debug(f'Downloading {url} to {out_dir}')

View file

@ -8,8 +8,7 @@ from uuid import uuid1
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.orm import Session
from wrolpi.common import logger, chunks, get_config
from wrolpi.common import save_settings_config
from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.media_path import MediaPath
@ -289,10 +288,186 @@ def refresh_videos(channel_links: list = None):
logger.info('Refresh of video files complete')
def get_channels_config(session: Session) -> dict:
"""
Create a dictionary that contains all the Channels from the DB.
"""
class ChannelsConfig(ConfigFile):
file_name = 'channels.yaml'
default_config = dict(
channels={
'wrolpi': dict(
name='WROLPi',
url='https://www.youtube.com/channel/UC4t8bw1besFTyjW7ZBCOIrw/videos',
directory='videos/wrolpi',
download_frequency=604800,
)
},
favorites=dict(),
)
@property
def channels(self) -> dict:
return self._config['channels']
@channels.setter
def channels(self, value: dict):
self.update({'channels': value})
@property
def favorites(self) -> dict:
return self._config['favorites']
@favorites.setter
def favorites(self, value: dict):
self.update({'favorites': value})
CHANNELS_CONFIG: ChannelsConfig = None
TEST_CHANNELS_CONFIG: ChannelsConfig = None
def get_channels_config():
global TEST_CHANNELS_CONFIG
if isinstance(TEST_CHANNELS_CONFIG, ConfigFile):
return TEST_CHANNELS_CONFIG
global CHANNELS_CONFIG
if isinstance(CHANNELS_CONFIG, ConfigFile):
return CHANNELS_CONFIG
CHANNELS_CONFIG = ChannelsConfig()
return CHANNELS_CONFIG
def set_test_channels_config(enable: bool):
global TEST_CHANNELS_CONFIG
if enable is True:
TEST_CHANNELS_CONFIG = ChannelsConfig()
else:
TEST_CHANNELS_CONFIG = None
class DownloaderConfig(ConfigFile):
file_name = 'downloader.yaml'
default_config = dict(
continue_dl=True,
dateafter='19900101',
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
format='22 / 135 / 18 / 43',
nooverwrites=True,
quiet=False,
writeautomaticsub=True,
writeinfojson=True,
writesubtitles=True,
writethumbnail=True,
youtube_include_dash_manifest=False,
)
@property
def continue_dl(self) -> bool:
return self._config['continue_dl']
@continue_dl.setter
def continue_dl(self, value: bool):
self.update({'continue_dl': value})
@property
def dateafter(self) -> str:
return self._config['dateafter']
@dateafter.setter
def dateafter(self, value: str):
self.update({'dateafter': value})
@property
def file_name_format(self) -> str:
return self._config['file_name_format']
@file_name_format.setter
def file_name_format(self, value: str):
self.update({'file_name_format': value})
@property
def format(self) -> str:
return self._config['format']
@format.setter
def format(self, value: str):
self.update({'format': value})
@property
def nooverwrites(self) -> bool:
return self._config['nooverwrites']
@nooverwrites.setter
def nooverwrites(self, value: bool):
self.update({'nooverwrites': value})
@property
def quiet(self) -> bool:
return self._config['quiet']
@quiet.setter
def quiet(self, value: bool):
self.update({'quiet': value})
@property
def writeautomaticsub(self) -> bool:
return self._config['writeautomaticsub']
@writeautomaticsub.setter
def writeautomaticsub(self, value: bool):
self.update({'writeautomaticsub': value})
@property
def writeinfojson(self) -> bool:
return self._config['writeinfojson']
@writeinfojson.setter
def writeinfojson(self, value: bool):
self.update({'writeinfojson': value})
@property
def writesubtitles(self) -> bool:
return self._config['writesubtitles']
@writesubtitles.setter
def writesubtitles(self, value: bool):
self.update({'writesubtitles': value})
@property
def writethumbnail(self) -> bool:
return self._config['writethumbnail']
@writethumbnail.setter
def writethumbnail(self, value: bool):
self.update({'writethumbnail': value})
@property
def youtube_include_dash_manifest(self) -> bool:
return self._config['youtube_include_dash_manifest']
@youtube_include_dash_manifest.setter
def youtube_include_dash_manifest(self, value: bool):
self.update({'youtube_include_dash_manifest': value})
DOWNLOADER_CONFIG: DownloaderConfig = None
TEST_DOWNLOADER_CONFIG: DownloaderConfig = None
def get_downloader_config():
global TEST_DOWNLOADER_CONFIG
if isinstance(TEST_DOWNLOADER_CONFIG, WROLPiConfig):
return TEST_DOWNLOADER_CONFIG
global DOWNLOADER_CONFIG
if isinstance(DOWNLOADER_CONFIG, WROLPiConfig):
return DOWNLOADER_CONFIG
DOWNLOADER_CONFIG = DownloaderConfig()
return DOWNLOADER_CONFIG
def get_channels_config_from_db(session: Session) -> dict:
"""Create a dictionary that contains all the Channels from the DB."""
channels = session.query(Channel).order_by(Channel.link).all()
channels = {i.link: i.config_view() for i in channels}
@ -315,10 +490,11 @@ def save_channels_config(session=None, preserve_favorites: bool = True):
"""
Pull the Channel information from the DB, save it to the config.
"""
config = get_channels_config(session)
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
if preserve_favorites:
config['favorites'].update(get_config().get('favorites', {}))
save_settings_config(config)
config['favorites'].update(channels_config.favorites or {})
channels_config.update(config)
async def get_statistics():

View file

@ -11,7 +11,7 @@ from PIL import Image
from modules.videos.models import Channel, Video
from modules.videos.test.common import create_channel_structure
from wrolpi.common import get_absolute_media_path, get_config
from wrolpi.common import get_absolute_media_path
from wrolpi.dates import local_timezone
from wrolpi.db import get_db_session
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
@ -19,7 +19,7 @@ from wrolpi.vars import PROJECT_DIR
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
import_videos_config
from ..lib import save_channels_config
from ..lib import save_channels_config, get_channels_config
class TestCommon(TestAPI):
@ -309,7 +309,7 @@ def test_update_censored_videos(test_session, video_factory, simple_channel):
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
def test_import_favorites(test_session, simple_channel, video_factory):
def test_import_favorites(test_session, simple_channel, video_factory, test_channels_config):
"""
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
favorited again.
@ -331,8 +331,8 @@ def test_import_favorites(test_session, simple_channel, video_factory):
# Save config, verify that favorite is set.
save_channels_config()
config = get_config()
assert config['favorites'] == favorites
config = get_channels_config()
assert config.favorites == favorites
def import_and_verify(favorited_ids: List[int]):
with mock.patch('modules.videos.common.get_channel_source_id', lambda i: 'foo'):
@ -351,8 +351,8 @@ def test_import_favorites(test_session, simple_channel, video_factory):
test_session.query(Video).filter_by(id=vid2.id).delete()
test_session.commit()
save_channels_config()
config = get_config()
assert config['favorites'] == favorites
config = get_channels_config()
assert config.favorites == favorites
import_and_verify([vid3.id])
# Add vid2 again, it should be favorited on import.
@ -363,6 +363,6 @@ def test_import_favorites(test_session, simple_channel, video_factory):
# Deleting the Video in the model really removes the favorite status.
vid3.delete()
config = get_config()
assert config['favorites'] == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
config = get_channels_config()
assert config.favorites == {'NO CHANNEL': {vid2.video_path.path.name: {'favorite': favorite}}}
import_and_verify([vid2.id])