rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Configs can be saved in background using switches.

Browse source
This commit is contained in:
Roland 2024-09-15 15:33:07 -06:00
parent 44d4921c84
commit d54735c4bb
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
53 changed files with 967 additions and 574 deletions
Download patch file Download diff file Expand all files Collapse all files

14
.circleci/config.yml
View file

@ -61,7 +61,7 @@ jobs:
- run: sudo apt-get update
- run: sudo apt-get install -y ffmpeg catdoc
- restore_cache:
key: deps-3.11-{{ checksum "requirements.txt" }}-2
key: deps-3.11-{{ checksum "requirements.txt" }}-4
- run:
name: Install Requirements
command: |
@ -69,7 +69,7 @@ jobs:
. venv/bin/activate
pip install -r requirements.txt
- save_cache:
key: deps-3.11-{{ checksum "requirements.txt" }}-2
key: deps-3.11-{{ checksum "requirements.txt" }}-4
paths:
- "venv"
- run:
@ -88,7 +88,7 @@ jobs:
steps:
- checkout
- restore_cache:
key: deps-3.11-{{ checksum "requirements.txt" }}-2
key: deps-3.11-{{ checksum "requirements.txt" }}-4
- run:
name: Install Requirements
command: |
@ -96,7 +96,7 @@ jobs:
. venv/bin/activate
pip install -r requirements.txt
- save_cache:
key: deps-3.11-{{ checksum "requirements.txt" }}-2
key: deps-3.11-{{ checksum "requirements.txt" }}-4
paths:
- "venv"
- run:
@ -115,7 +115,7 @@ jobs:
- run: sudo apt-get update
- run: sudo apt-get install -y ffmpeg catdoc
- restore_cache:
key: deps-3.12-{{ checksum "requirements.txt" }}-2
key: deps-3.12-{{ checksum "requirements.txt" }}-4
- run:
name: Install Requirements
command: |
@ -123,7 +123,7 @@ jobs:
. venv/bin/activate
pip install -r requirements.txt
- save_cache:
key: deps-3.12-{{ checksum "requirements.txt" }}-2
key: deps-3.12-{{ checksum "requirements.txt" }}-4
paths:
- "venv"
- run:
@ -134,7 +134,7 @@ jobs:
steps:
- checkout
- restore_cache:
key: deps-3.11-{{ checksum "requirements.txt" }}-2
key: deps-3.11-{{ checksum "requirements.txt" }}-4
- run:
name: Install dependencies
command: |

110
alembic/versions/2bfb3b50b178_tag_names_migration.py
View file

@ -1,4 +1,4 @@
"""Tag names migration
"""MTag names migration
Revision ID: 2bfb3b50b178
Revises: 8d0d81bc9c34
@ -6,13 +6,21 @@ Create Date: 2024-07-31 14:58:29.583029
"""
import os
import sys
import pathlib
from datetime import datetime
import yaml
from alembic import op
from sqlalchemy import Column, Integer, BigInteger, ForeignKey, Text, String
from sqlalchemy.ext.declarative import declarative_base
from sqlalchemy.orm import Session
from sqlalchemy.orm.exc import NoResultFound # noqa
from wrolpi.common import escape_file_name
from wrolpi.tags import Tag, get_tags_config
from wrolpi import dates
from wrolpi.common import escape_file_name, get_media_directory, ModelHelper
from wrolpi.common import get_relative_to_media_directory
from wrolpi.dates import TZDateTime
from wrolpi.tags import get_tags_config
# revision identifiers, used by Alembic.
revision = '2bfb3b50b178'
@ -22,13 +30,97 @@ depends_on = None
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
Base = declarative_base()
class MFileGroup(ModelHelper, Base):
__tablename__ = 'file_group'
id: int = Column(BigInteger, primary_key=True)
primary_path: pathlib.Path = Column(String, unique=True, nullable=False)
class MTagFile(ModelHelper, Base):
__tablename__ = 'tag_file'
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
file_group_id = Column(BigInteger, ForeignKey('file_group.id', ondelete='CASCADE'), primary_key=True)
class MTag(ModelHelper, Base):
__tablename__ = 'tag'
id = Column(Integer, primary_key=True)
name = Column(String, unique=True, nullable=False)
color = Column(String)
class MZim(Base):
__tablename__ = 'zim'
id = Column(Integer, primary_key=True)
path: pathlib.Path = Column(String, unique=True, nullable=False)
class MTagZimEntry(Base):
__tablename__ = 'tag_zim'
tag_id = Column(Integer, ForeignKey('tag.id', ondelete='CASCADE'), primary_key=True)
zim_id = Column(Integer, ForeignKey('zim.id', ondelete='CASCADE'), primary_key=True)
zim_entry: str = Column(Text, nullable=False, primary_key=True)
created_at: datetime = Column(TZDateTime, default=dates.now)
def save_tags(session: Session):
media_directory = get_media_directory()
tags = dict()
tag_rows = session.query(MTag)
for tag in tag_rows:
tags[tag.name] = dict(color=tag.color)
results = session.query(MTag, MTagFile, MFileGroup) \
.filter(MTagFile.tag_id == MTag.id, MTagFile.file_group_id == MFileGroup.id) \
.order_by(MFileGroup.primary_path)
tag_files = []
for tag, tag_file, file_group in results:
value = [
tag.name,
str(file_group.primary_path.relative_to(media_directory)),
# Fallback to current time if not set.
tag_file.created_at.isoformat() if tag_file.created_at else dates.now().isoformat(),
]
tag_files.append(value)
results = session.query(MTag, MZim, MTagZimEntry) \
.filter(MTag.id == MTagZimEntry.tag_id, MZim.id == MTagZimEntry.zim_id) \
.order_by(MTagZimEntry.zim_id, MTagZimEntry.zim_entry)
tag_zims = []
for tag, zim, tag_zim_entry in results:
zim: MZim
tag_zim_entry: MTagZimEntry
value = [
tag.name,
str(get_relative_to_media_directory(zim.path)),
tag_zim_entry.zim_entry,
# Fallback to current time if not set.
tag_zim_entry.created_at.isoformat() if tag_zim_entry.created_at else dates.now().isoformat(),
]
tag_zims.append(value)
# Write to the config.
with get_tags_config().get_file().open('wt') as fh:
data = {
'tag_files': tag_files,
'tag_zims': tag_zims,
'tags': tags,
}
yaml.dump(data, fh)
def upgrade():
bind = op.get_bind()
session = Session(bind=bind)
need_commit = False
tags = session.query(Tag).all()
tags = session.query(MTag).all()
for tag in tags:
escaped_name = escape_file_name(tag.name)
if escaped_name != tag.name:
@ -38,13 +130,7 @@ def upgrade():
if need_commit:
session.commit()
try:
config = get_tags_config()
config.initialize()
config.save_tags(session, ignore_lock=True)
except FileNotFoundError:
print(f'Failed to save tags config probably because media directory does not exist', file=sys.stderr)
save_tags(session)
def downgrade():

2
app/src/Tags.js
View file

@ -1,6 +1,6 @@
import React, {useEffect} from "react";
import {deleteTag, getTags, saveTag} from "./api";
import {Dimmer, Divider, Form, Grid, GridColumn, GridRow, Label, TableCell, TableRow,} from "semantic-ui-react";
import {Dimmer, Divider, Form, Grid, GridColumn, GridRow, Input, Label, TableCell, TableRow,} from "semantic-ui-react";
import {APIButton, contrastingColor, ErrorMessage, fuzzyMatch, scrollToTopOfElement} from "./components/Common";
import {
Button,

80
main.py
View file

@ -11,13 +11,12 @@ from sanic.signals import Event
from wrolpi import flags, BEFORE_STARTUP_FUNCTIONS, admin
from wrolpi import root_api # noqa
from wrolpi import tags
from wrolpi.api_utils import api_app
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.common import logger, check_media_directory, set_log_level, limit_concurrent, \
cancel_refresh_tasks, cancel_background_tasks, get_wrolpi_config, can_connect_to_server, wrol_mode_enabled
from wrolpi.contexts import attach_shared_contexts, reset_shared_contexts, initialize_configs_contexts
from wrolpi.dates import Seconds
from wrolpi.downloader import import_downloads_config, download_manager, \
perpetual_download_worker
from wrolpi.downloader import import_downloads_config, download_manager
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, INTERNET_SERVER
from wrolpi.version import get_version_string
@ -145,7 +144,7 @@ def main():
return 1
@api_app.main_process_start
@api_app.main_process_ready
async def startup(app: Sanic):
"""
Initializes multiprocessing tools, flags, etc.
@ -167,7 +166,7 @@ async def startup(app: Sanic):
logger.debug('startup done')
@api_app.listener('before_server_start') # FileConfigs need to be initialized first.
@api_app.listener('after_server_start') # FileConfigs need to be initialized first.
async def initialize_configs(app: Sanic):
"""Each Sanic process runs this once."""
# Each process will have their own FileConfig object, but share the `app.shared_ctx.*config`
@ -177,13 +176,11 @@ async def initialize_configs(app: Sanic):
initialize_configs_contexts(app)
await asyncio.sleep(0.5)
wrol_mode_enabled()
logger.info('initialize_configs succeeded')
logger.info(f'initialize_configs succeeded pid={os.getpid()}')
except Exception as e:
logger.error('initialize_configs failed with', exc_info=e)
raise
logger.debug('initialize_configs done')
@api_app.signal(Event.SERVER_SHUTDOWN_BEFORE)
@api_app.listener('reload_process_stop')
@ -203,7 +200,7 @@ async def handle_server_shutdown_reset(app: Sanic, loop):
# Start periodic tasks after configs are ready.
@api_app.listener('after_server_start')
@api_app.after_server_start
async def start_single_tasks(app: Sanic):
"""Recurring/Single tasks that are started in only one Sanic process."""
# Only allow one child process to perform periodic tasks. See `handle_server_shutdown`
@ -219,14 +216,6 @@ async def start_single_tasks(app: Sanic):
else:
download_manager.disable()
# Start perpetual tasks. DO NOT AWAIT!
app.add_task(perpetual_check_db_is_up_worker()) # noqa
app.add_task(perpetual_download_worker()) # noqa
app.add_task(perpetual_have_internet_worker()) # noqa
await app.dispatch('wrolpi.periodic.start_video_missing_comments_download')
await app.dispatch('wrolpi.periodic.status')
try:
flag_outdated_zim_files()
except Exception as e:
@ -275,14 +264,7 @@ async def start_single_tasks(app: Sanic):
logger.debug(f'start_single_tasks done')
@api_app.listener('after_server_start')
async def start_sanic_worker(app: Sanic):
logger.debug(f'start_sanic_worker')
await app.dispatch('wrolpi.periodic.check_log_level')
@api_app.signal('wrolpi.periodic.check_log_level')
@perpetual_signal(sleep=1)
async def periodic_check_log_level():
"""Copies global log level into this Sanic worker's logger."""
log_level = api_app.shared_ctx.log_level.value
@ -290,49 +272,31 @@ async def periodic_check_log_level():
logger.info(f'changing log level from {logger.getEffectiveLevel()} to {log_level}')
set_log_level(log_level, warn_level=False)
try:
await asyncio.sleep(1)
await api_app.dispatch('wrolpi.periodic.check_log_level')
except asyncio.CancelledError:
# Server is shutting down.
pass
@perpetual_signal(sleep=10)
async def perpetual_check_db_is_up_worker():
try:
while True:
try:
flags.check_db_is_up()
flags.init_flags()
except Exception as e:
logger.error('Failed to check if db is up', exc_info=e)
await asyncio.sleep(10)
except asyncio.CancelledError:
logger.info('periodic_check_db_is_up was cancelled...')
flags.check_db_is_up()
flags.init_flags()
except Exception as e:
logger.error('Failed to check db status', exc_info=e)
@perpetual_signal(sleep=10)
async def perpetual_have_internet_worker():
try:
while True:
try:
if can_connect_to_server(INTERNET_SERVER):
flags.have_internet.set()
# Check hourly once we have internet.
await asyncio.sleep(float(Seconds.hour))
else:
# Check more often until the internet is back.
flags.have_internet.clear()
await asyncio.sleep(10)
except Exception as e:
logger.error('Failed to check if internet is up', exc_info=e)
except asyncio.CancelledError:
logger.info('perpetual_check_have_internet_worker was cancelled...')
if can_connect_to_server(INTERNET_SERVER):
flags.have_internet.set()
# Check hourly once we have internet.
await asyncio.sleep(float(Seconds.hour))
else:
# Check more often until the internet is back.
flags.have_internet.clear()
except Exception as e:
logger.error('Failed to check if internet is up', exc_info=e)
@api_app.signal('wrolpi.periodic.start_video_missing_comments_download')
@perpetual_signal(sleep=30)
async def periodic_start_video_missing_comments_download():
from modules.videos.video.lib import get_missing_videos_comments
@ -358,8 +322,6 @@ async def periodic_start_video_missing_comments_download():
await asyncio.sleep(int(Seconds.hour))
await api_app.dispatch('wrolpi.periodic.start_video_missing_comments_download')
if __name__ == '__main__':
sys.exit(main())

6
modules/archive/test/test_api.py
View file

@ -82,20 +82,20 @@ def test_archives_search(test_session, archive_directory, archive_factory, test_
@pytest.mark.asyncio
async def test_search_archive_tags(test_session, test_async_client, archive_factory, tag_factory):
async def test_search_archive_tags(test_session, async_client, archive_factory, tag_factory):
"""Tagged Archives can be searched."""
tag = await tag_factory()
archive_factory(domain='example.com', tag_names=[tag.name, ])
test_session.commit()
content = {'search_str': '', 'domain': 'example.com', 'tag_names': [tag.name, ]}
request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content))
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups']
assert response.json['totals']['file_groups'] == 1
content = {'search_str': '', 'domain': 'example.com', 'tag_names': ['does not exist', ]}
request, response = await test_async_client.post('/api/archive/search', content=json.dumps(content))
request, response = await async_client.post('/api/archive/search', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == []
assert response.json['totals']['file_groups'] == 0

14
modules/archive/test/test_lib.py
View file

@ -80,7 +80,7 @@ async def test_relationships(test_session, example_singlefile):
@pytest.mark.asyncio
async def test_archive_title(test_async_client, test_session, archive_factory, singlefile_contents_factory):
async def test_archive_title(async_client, test_session, archive_factory, singlefile_contents_factory):
"""An Archive's title can be fetched in multiple ways. This tests from most to least reliable."""
# Create some test files, delete all records for a fresh refresh.
archive_factory(
@ -459,7 +459,7 @@ async def test_title_in_filename(test_session, fake_now, test_directory, image_b
@pytest.mark.asyncio
async def test_refresh_archives(test_session, test_directory, test_async_client, make_files_structure):
async def test_refresh_archives(test_session, test_directory, async_client, make_files_structure):
"""Archives can be found and put in the database. A single Archive will have multiple files."""
# The start of a typical singlefile html file.
singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!--
@ -484,7 +484,7 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
})
# The single archive is found.
await test_async_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Cause a re-index of the archive.
@ -493,7 +493,7 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
test_session.commit()
# Running the refresh does not result in a new archive.
await test_async_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
assert test_session.query(Archive).count() == 1
# Archives file format was changed, lets check the new formats are found.
@ -501,13 +501,13 @@ async def test_refresh_archives(test_session, test_directory, test_async_client,
'archive/example.com/2021-10-05-16-20-11_The Title.html': '<html></html>',
'archive/example.com/2021-10-05-16-20-11_The Title.readability.json': '{"url": "https://example.com"}',
})
await test_async_client.post('/api/files/refresh')
await async_client.post('/api/files/refresh')
# The old formatted archive above is renamed.
assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file()
assert test_session.query(Archive).count() == 2
content = json.dumps({'search_str': 'text', 'model': 'archive'})
request, response = await test_async_client.post('/api/files/search', content=content)
request, response = await async_client.post('/api/files/search', content=content)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'], 'No files matched "text"'
assert response.json['file_groups'][0]['model'] == 'archive', 'Returned file was not an archive'
@ -821,7 +821,7 @@ def test_get_url_from_singlefile():
@pytest.mark.asyncio
async def test_get_custom_archive_directory(test_async_client, test_directory, test_config):
async def test_get_custom_archive_directory(async_client, test_directory, test_config):
"""Custom directory can be used for archive directory."""
# Default location.
assert lib.get_archive_directory() == (test_directory / 'archive')

2
modules/archive/test/test_models.py
View file

@ -7,7 +7,7 @@ from wrolpi.common import get_wrolpi_config
@pytest.mark.asyncio
async def test_archive_download_destination(test_async_client, test_session, test_directory, archive_factory, fake_now):
async def test_archive_download_destination(async_client, test_session, test_directory, archive_factory, fake_now):
fake_now(datetime(2000, 1, 2))
wrolpi_config = get_wrolpi_config()

2
modules/inventory/common.py
View file

@ -180,7 +180,7 @@ def save_inventories_file():
config.inventories = inventories
config.version = version
config.save()
config.background_save.activate_switch()
@before_startup

18
modules/inventory/test/test_api.py
View file

@ -76,33 +76,33 @@ def test_item_api(test_session, init_test_inventory, test_client):
@pytest.mark.asyncio
async def test_inventory_api(test_session, test_async_client):
async def test_inventory_api(test_session, async_client):
"""An Inventory can be created."""
inventory = dict(name='foo')
request, response = await test_async_client.post('/api/inventory', content=json.dumps(inventory))
request, response = await async_client.post('/api/inventory', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.CREATED, response.status_code
request, response = await test_async_client.get('/api/inventory')
request, response = await async_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
inventory = response.json['inventories'][0]
inventory_id = inventory['id']
request, response = await test_async_client.get(f'/api/inventory/{inventory_id}')
request, response = await async_client.get(f'/api/inventory/{inventory_id}')
assert response.status_code == HTTPStatus.OK
request, response = await test_async_client.get('/api/inventory/categories')
request, response = await async_client.get('/api/inventory/categories')
assert response.status_code == HTTPStatus.OK
request, response = await test_async_client.get('/api/inventory/brands')
request, response = await async_client.get('/api/inventory/brands')
assert response.status_code == HTTPStatus.OK
inventory = dict(name='new name')
request, response = await test_async_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
request, response = await async_client.put(f'/api/inventory/{inventory_id}', content=json.dumps(inventory))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = await test_async_client.delete(f'/api/inventory/{inventory_id}')
request, response = await async_client.delete(f'/api/inventory/{inventory_id}')
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = await test_async_client.get('/api/inventory')
request, response = await async_client.get('/api/inventory')
assert response.status_code == HTTPStatus.OK
assert len(response.json['inventories']) == 0

8
modules/inventory/test/test_inventory.py
View file

@ -9,6 +9,7 @@ from pint import Quantity
from wrolpi.common import Base
from wrolpi.db import get_db_session
from wrolpi.switches import await_switches
from wrolpi.test.common import PytestCase
from .. import init
from ..common import sum_by_key, get_inventory_by_category, get_inventory_by_subcategory, get_inventory_by_name, \
@ -273,7 +274,8 @@ def test_no_inventories(test_session, test_directory):
pass
def test_inventories_version(test_async_client, test_session, test_directory, init_test_inventory):
@pytest.mark.asyncio
async def test_inventories_version(async_client, test_session, test_directory, init_test_inventory):
"""You can't save over a newer version of an inventory."""
config = get_inventories_config()
@ -284,10 +286,12 @@ def test_inventories_version(test_async_client, test_session, test_directory, in
# Version is set to 1 on first save.
save_inventories_file()
await await_switches()
assert config.version == 1
# Version is incremented when saving.
save_inventories_file()
await await_switches()
assert config.version == 2
# Version is greater than what will be saved.
@ -299,7 +303,7 @@ def test_inventories_version(test_async_client, test_session, test_directory, in
@pytest.mark.asyncio
async def test_inventories_config(test_async_client, test_session, test_directory, init_test_inventory):
async def test_inventories_config(async_client, test_session, test_directory, init_test_inventory):
for item in TEST_ITEMS:
item = Item(**item)
test_session.add(item)

11
modules/videos/__init__.py
View file

@ -1,6 +1,8 @@
import asyncio
from typing import List, Tuple
from sqlalchemy.orm import Session
from modules.videos.models import Video
from wrolpi.common import logger, limit_concurrent, register_modeler, register_refresh_cleanup
from wrolpi.db import get_db_curs, get_db_session
@ -33,14 +35,19 @@ async def video_modeler():
video_id = None
try:
if not video:
video = Video(file_group=file_group)
video = Video(file_group=file_group, file_group_id=file_group.id)
session.add(video)
session.flush([video])
video_id = video.id
if not Session.object_session(video):
session.add(video)
video.flush()
# Extract ffprobe data.
await video.get_ffprobe_json()
video.flush(session)
# Validate and index subtitles.
video.validate()
video.validate(session)
processed += 1
except Exception as e:
if PYTEST:
raise

5
modules/videos/channel/api.py
View file

@ -8,8 +8,7 @@ from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from wrolpi.api_utils import json_response
from wrolpi.common import logger, wrol_mode_check, make_media_directory, \
get_media_directory, run_after, get_relative_to_media_directory
from wrolpi.common import logger, wrol_mode_check, get_media_directory, get_relative_to_media_directory
from wrolpi.downloader import download_manager
from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
@ -74,11 +73,11 @@ def channel_post(_: Request, body: schema.ChannelPostRequest):
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@validate(schema.ChannelPutRequest)
@run_after(save_channels_config)
@wrol_mode_check
async def channel_update(_: Request, channel_id: int, body: schema.ChannelPutRequest):
body.directory = pathlib.Path(body.directory) if body.directory else None
channel = await lib.update_channel(data=body, channel_id=channel_id)
save_channels_config.activate_switch()
return response.raw('', HTTPStatus.NO_CONTENT,
headers={'Location': f'/api/videos/channels/{channel.id}'})

21
modules/videos/channel/lib.py
View file

@ -6,7 +6,7 @@ from sqlalchemy import or_, func, desc, asc
from sqlalchemy.orm import Session
from wrolpi import flags
from wrolpi.common import run_after, logger, \
from wrolpi.common import logger, \
get_media_directory, wrol_mode_check, background_task
from wrolpi.db import get_db_curs, optional_session, get_db_session
from wrolpi.downloader import save_downloads_config, download_manager
@ -114,7 +114,6 @@ def get_channel(session: Session, *, channel_id: int = None, source_id: str = No
return channel
@run_after(save_channels_config)
@optional_session
async def update_channel(session: Session, *, data: schema.ChannelPutRequest, channel_id: int) -> Channel:
"""Update a Channel's DB record"""
@ -150,10 +149,11 @@ async def update_channel(session: Session, *, data: schema.ChannelPutRequest, ch
else:
background_task(_())
save_channels_config.activate_switch()
return channel
@run_after(save_channels_config)
@optional_session
def create_channel(session: Session, data: schema.ChannelPostRequest, return_dict: bool = True) -> Union[Channel, dict]:
"""
@ -180,10 +180,11 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
session.commit()
save_channels_config.activate_switch()
return channel.dict() if return_dict else channel
@run_after(save_channels_config)
@optional_session(commit=True)
def delete_channel(session: Session, *, channel_id: int):
channel = Channel.find_by_id(channel_id, session=session)
@ -191,6 +192,8 @@ def delete_channel(session: Session, *, channel_id: int):
channel_dict = channel.dict()
channel.delete_with_videos()
save_channels_config.activate_switch()
return channel_dict
@ -230,8 +233,8 @@ async def create_channel_download(channel_id: int, url: str, frequency: int, set
download = channel.get_or_create_download(url, frequency, session, reset_attempts=True)
download.settings = settings
save_channels_config()
await save_downloads_config()
save_channels_config.activate_switch()
save_downloads_config.activate_switch()
return download
@ -250,8 +253,8 @@ async def update_channel_download(channel_id: int, download_id: int, url: str, f
download_manager.remove_from_skip_list(url)
save_channels_config()
await save_downloads_config()
save_channels_config.activate_switch()
save_downloads_config.activate_switch()
return download
@ -287,6 +290,8 @@ async def tag_channel(tag_name: str | None, directory: pathlib.Path | None, chan
await coro
else:
background_task(coro)
else:
save_channels_config.activate_switch()
else:
logger.info(f'Tagging {channel} with {tag_name}')

62
modules/videos/channel/test/test_api.py
View file

@ -99,7 +99,7 @@ def test_video_file_name(test_session, simple_video, test_client):
@pytest.mark.asyncio
async def test_channel_conflicts(test_async_client, test_session, test_directory):
async def test_channel_conflicts(async_client, test_session, test_directory):
channel_directory = tempfile.TemporaryDirectory(dir=test_directory).name
pathlib.Path(channel_directory).mkdir()
new_channel = dict(
@ -109,7 +109,7 @@ async def test_channel_conflicts(test_async_client, test_session, test_directory
)
async def _post_channel(channel):
return await test_async_client.post('/api/videos/channels', content=json.dumps(channel))
return await async_client.post('/api/videos/channels', content=json.dumps(channel))
# Create it
request, response = await _post_channel(new_channel)
@ -189,7 +189,7 @@ def test_channel_empty_url_doesnt_conflict(test_client, test_session, test_direc
@pytest.mark.asyncio
async def test_channel_download_requires_refresh(
test_async_client, test_session, mock_video_extract_info, download_channel, video_download_manager,
async_client, test_session, mock_video_extract_info, download_channel, video_download_manager,
video_factory, events_history):
"""A Channel cannot be downloaded until it has been refreshed.
@ -206,7 +206,7 @@ async def test_channel_download_requires_refresh(
assert channel.refreshed == expected
assert bool(channel.info_json) == expected
await test_async_client.sanic_app.dispatch('wrolpi.download.')
await async_client.sanic_app.dispatch('wrolpi.download.')
assert_refreshed(False)
test_session.commit()
@ -259,7 +259,7 @@ def test_channel_by_id(test_session, test_client, simple_channel, simple_video):
@pytest.mark.asyncio
async def test_channel_crud(test_session, test_async_client, test_directory, test_download_manager):
async def test_channel_crud(test_session, async_client, test_directory, test_download_manager):
channel_directory = test_directory / 'channel directory'
channel_directory.mkdir()
@ -270,15 +270,15 @@ async def test_channel_crud(test_session, test_async_client, test_directory, tes
)
# Channel doesn't exist
request, response = await test_async_client.get('/api/videos/channels/examplechannel1')
request, response = await async_client.get('/api/videos/channels/examplechannel1')
assert response.status_code == HTTPStatus.NOT_FOUND, f'Channel exists: {response.json}'
# Create it
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.CREATED, response.content
location = response.headers['Location']
request, response = await test_async_client.get(location)
request, response = await async_client.get(location)
assert response.status_code == HTTPStatus.OK, response.json
created = response.json['channel']
assert created
@ -294,13 +294,13 @@ async def test_channel_crud(test_session, test_async_client, test_directory, tes
f'Channel directory is absolute: {created["directory"]}'
# Can't create it again
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(new_channel))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.BAD_REQUEST
async def put_and_fetch(new_channel_):
request, response = await test_async_client.put(location, content=json.dumps(new_channel_))
request, response = await async_client.put(location, content=json.dumps(new_channel_))
assert response.status_code == HTTPStatus.NO_CONTENT, response.status_code
request, response = await test_async_client.get(location)
request, response = await async_client.get(location)
assert response.status_code == HTTPStatus.OK
channel = response.json['channel']
return channel
@ -315,22 +315,22 @@ async def test_channel_crud(test_session, test_async_client, test_directory, tes
assert channel['url'] == 'https://example.com/channel1'
# Can't update channel that doesn't exist
request, response = await test_async_client.put('/api/videos/channels/doesnt_exist',
request, response = await async_client.put('/api/videos/channels/doesnt_exist',
content=json.dumps(new_channel))
assert response.status_code == HTTPStatus.NOT_FOUND
# Delete the new channel
request, response = await test_async_client.delete(location)
request, response = await async_client.delete(location)
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 0
# Cant delete it again
request, response = await test_async_client.delete(location)
request, response = await async_client.delete(location)
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest.mark.asyncio
async def test_change_channel_url(test_async_client, test_session, test_download_manager, download_channel,
async def test_change_channel_url(async_client, test_session, test_download_manager, download_channel,
test_download_manager_config):
# Change the Channel's URL.
download_channel.update({'url': 'https://example.com/new-url'})
@ -411,12 +411,12 @@ async def test_get_channel_videos_pagination(test_session, simple_channel, video
@pytest.mark.asyncio
async def test_create_channel_download_api(test_async_client, test_session, simple_channel, tag_factory):
async def test_create_channel_download_api(async_client, test_session, simple_channel, tag_factory):
tag = await tag_factory()
# Create Download.
body = {'url': 'https://example.com/1', 'frequency': 42, 'settings': {'title_include': ''}}
request, response = await test_async_client.post(
request, response = await async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download: Download = test_session.query(Download).one()
@ -425,7 +425,7 @@ async def test_create_channel_download_api(test_async_client, test_session, simp
# Create another Download. Previous Downloads is untouched.
body = {'url': 'https://example.com/2', 'frequency': 55, 'settings': {'tag_names': [tag.name]}}
request, response = await test_async_client.post(
request, response = await async_client.post(
f'/api/videos/channels/{simple_channel.id}/download', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download1, download2 = test_session.query(Download).order_by(Download.url).all()
@ -436,7 +436,7 @@ async def test_create_channel_download_api(test_async_client, test_session, simp
# Change frequency of first Download.
body = {'url': 'https://example.com/1', 'frequency': 123, 'settings': {'title_include': 'foo,bar'}}
request, response = await test_async_client.put(
request, response = await async_client.put(
f'/api/videos/channels/{simple_channel.id}/download/{download1.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content
download1, download2 = test_session.query(Download).order_by(Download.url).all()
@ -447,14 +447,14 @@ async def test_create_channel_download_api(test_async_client, test_session, simp
assert len(simple_channel.downloads) == 2
# Deleting Download does not delete Channel.
request, response = await test_async_client.delete(f'/api/download/{download1.id}')
request, response = await async_client.delete(f'/api/download/{download1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 1
assert test_session.query(Channel).count() == 1
@pytest.mark.asyncio
async def test_tag_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory, video_factory,
async def test_tag_channel(async_client, test_session, test_directory, channel_factory, tag_factory, video_factory,
test_channels_config, test_download_manager, test_downloader):
"""A single Tag can be applied to a Channel."""
# Create channel directory in the usual videos directory.
@ -485,7 +485,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
assert [i.path for i in test_session.query(Directory)] == [channel.directory, ]
body = dict(tag_name=tag.name, directory='videos/Tag Name/Channel Name')
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
request, response = await async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT
channel = test_session.query(Channel).one()
@ -518,7 +518,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
# Remove tag, Channel/Videos should be moved back.
body = dict(tag_name=None, directory='videos/Channel Name')
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
request, response = await async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = test_session.query(Channel).one()
assert channel.tag_name is None
@ -535,7 +535,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
# Channel can be Tagged, without moving directories.
body = dict(tag_name=tag.name)
request, response = await test_async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
request, response = await async_client.post(f'/api/videos/channels/{channel.id}/tag', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = test_session.query(Channel).one()
assert channel.tag_name == 'Tag Name'
@ -550,7 +550,7 @@ async def test_tag_channel(test_async_client, test_session, test_directory, chan
@pytest.mark.asyncio
async def test_move_channel(test_async_client, test_session, test_directory, channel_factory, tag_factory,
async def test_move_channel(async_client, test_session, test_directory, channel_factory, tag_factory,
video_factory, test_channels_config, test_download_manager, test_downloader):
"""A Channel can be moved."""
channel = channel_factory(name='Channel Name')
@ -566,7 +566,7 @@ async def test_move_channel(test_async_client, test_session, test_directory, cha
name=channel.name,
directory=str(test_directory / 'videos/New Directory'),
)
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
request, response = await async_client.put(f'/api/videos/channels/{channel.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = Channel.find_by_id(channel.id)
assert str(channel.directory) == str(test_directory / 'videos/New Directory')
@ -576,7 +576,7 @@ async def test_move_channel(test_async_client, test_session, test_directory, cha
@pytest.mark.asyncio
async def test_move_channel_after_terminal_move(test_async_client, test_session, test_directory, channel_factory,
async def test_move_channel_after_terminal_move(async_client, test_session, test_directory, channel_factory,
tag_factory, video_factory, test_channels_config, test_download_manager,
test_downloader):
"""A Channel can be moved in the UI after being moved in the Terminal."""
@ -594,7 +594,7 @@ async def test_move_channel_after_terminal_move(test_async_client, test_session,
name=channel.name,
directory=str(test_directory / 'videos/New Directory'),
)
request, response = await test_async_client.put(f'/api/videos/channels/{channel.id}', json=body)
request, response = await async_client.put(f'/api/videos/channels/{channel.id}', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.content.decode()
channel = Channel.find_by_id(channel.id)
vid1 = test_session.query(Video).one()
@ -603,7 +603,7 @@ async def test_move_channel_after_terminal_move(test_async_client, test_session,
@pytest.mark.asyncio
async def test_search_tagged_channels(test_async_client, test_session, channel_factory, tag_factory,
async def test_search_tagged_channels(async_client, test_session, channel_factory, tag_factory,
video_factory):
"""Tagged Channels can be searched."""
tag = await tag_factory()
@ -611,7 +611,7 @@ async def test_search_tagged_channels(test_async_client, test_session, channel_f
# Can search when empty.
body = dict(tag_names=[tag.name])
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
request, response = await async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 0, 'No Channels should be tagged'
@ -620,7 +620,7 @@ async def test_search_tagged_channels(test_async_client, test_session, channel_f
test_session.commit()
body = dict(tag_names=[tag.name])
request, response = await test_async_client.post('/api/videos/channels/search', json=body)
request, response = await async_client.post('/api/videos/channels/search', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['channels']) == 1, 'Only one Channel is tagged'
assert response.json['channels'][0]['id'] == channel1.id

12
modules/videos/channel/test/test_lib.py
View file

@ -8,6 +8,7 @@ from modules.videos.channel import lib
from modules.videos.errors import UnknownChannel
from modules.videos.lib import save_channels_config, import_channels_config
from modules.videos.models import Channel
from wrolpi.switches import await_switches
@pytest.mark.parametrize('params', [
@ -66,7 +67,7 @@ def test_get_channel(test_session, test_directory, channel_factory):
@pytest.mark.asyncio
async def test_channels_no_url(test_async_client, test_session, test_directory, test_channels_config):
async def test_channels_no_url(async_client, test_session, test_directory, test_channels_config):
"""Test that a Channel's URL is coerced to None if it is empty."""
channel1_directory = test_directory / 'channel1'
channel1_directory.mkdir()
@ -91,7 +92,7 @@ async def test_channels_no_url(test_async_client, test_session, test_directory,
assert len(channels) == 2
assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels]
save_channels_config()
save_channels_config.activate_switch()
import_channels_config()
@ -128,20 +129,21 @@ async def test_search_channels_by_name(test_session, channel_factory, video_fact
@pytest.mark.asyncio
async def test_tag_channel_existing(test_async_client, test_session, test_directory, channel_factory, tag_factory,
async def test_tag_channel_existing(async_client, test_session, test_directory, channel_factory, tag_factory,
video_factory, test_channels_config):
"""A Channel can already exist in a Tag's directory and still be tagged."""
"""A Channel already in a Tag's directory can still be tagged."""
channel_directory = test_directory / 'videos/one/Channel Name'
channel = channel_factory(name='Channel Name', download_frequency=120, directory=channel_directory)
video_factory(channel_id=channel.id)
test_session.commit()
await await_switches()
assert str(channel.directory).startswith(f'{test_directory}/videos/one/')
tag = await tag_factory()
await lib.tag_channel(tag.name, channel_directory, channel.id, test_session)
await await_switches()
# Channel's tag is saved to the config.
save_channels_config(test_session)
with test_channels_config.open() as fh:
config = yaml.load(fh, Loader=yaml.Loader)
assert config['channels'][0]['tag_name'] == tag.name

10
modules/videos/channel/test/test_models.py
View file

@ -20,7 +20,7 @@ def test_delete_channel_no_url(test_session, test_client, channel_factory):
@pytest.mark.asyncio
async def test_delete_channel_with_videos(test_session, test_async_client, channel_factory, video_factory):
async def test_delete_channel_with_videos(test_session, async_client, channel_factory, video_factory):
"""Videos are disowned when their Channel is deleted."""
channel = channel_factory()
video = video_factory(channel_id=channel.id)
@ -28,7 +28,7 @@ async def test_delete_channel_with_videos(test_session, test_async_client, chann
test_session.commit()
# Delete the Channel.
request, response = await test_async_client.delete(f'/api/videos/channels/{channel.id}')
request, response = await async_client.delete(f'/api/videos/channels/{channel.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
# Video still exists, but has no channel.
@ -39,7 +39,7 @@ async def test_delete_channel_with_videos(test_session, test_async_client, chann
@pytest.mark.asyncio
async def test_nested_channel_directories(test_session, test_async_client, test_directory, channel_factory,
async def test_nested_channel_directories(test_session, async_client, test_directory, channel_factory,
video_factory):
"""Channel directories cannot contain another Channel's directory."""
(test_directory / 'foo').mkdir()
@ -54,7 +54,7 @@ async def test_nested_channel_directories(test_session, test_async_client, test_
name='channel 2',
directory=str(channel2_directory),
)
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(content))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['cause'] and \
response.json['cause']['message'] == 'The directory is already used by another channel.'
@ -64,7 +64,7 @@ async def test_nested_channel_directories(test_session, test_async_client, test_
name='channel 3',
directory=str(test_directory),
)
request, response = await test_async_client.post('/api/videos/channels', content=json.dumps(content))
request, response = await async_client.post('/api/videos/channels', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['cause'] and \
response.json['cause']['message'] == 'The directory is already used by another channel.'

4
modules/videos/conftest.py
View file

@ -246,7 +246,7 @@ def video_with_search_factory(test_session, test_directory, video_file_factory):
@pytest.fixture
def assert_video_search(test_async_client):
def assert_video_search(async_client):
"""A fixture which performs a video search request against the API.
If assert_* params are passed, the response is checked."""
@ -276,7 +276,7 @@ def assert_video_search(test_async_client):
if channel_id is not None:
content['channel_id'] = channel_id
request, response = await test_async_client.post('/api/videos/search', content=json.dumps(content))
request, response = await async_client.post('/api/videos/search', content=json.dumps(content))
assert response.json
assert response.status_code == HTTPStatus.OK

19
modules/videos/lib.py
View file

@ -3,12 +3,10 @@ import dataclasses
import html
import pathlib
import re
import traceback
from typing import Generator, Type
from typing import Tuple, Optional
import pytz
from sqlalchemy import or_
from sqlalchemy.orm import Session
from yt_dlp import YoutubeDL
@ -18,10 +16,11 @@ from wrolpi.captions import extract_captions
from wrolpi.common import ConfigFile, limit_concurrent, get_wrolpi_config, extract_domain, \
escape_file_name, get_media_directory, background_task, Base
from wrolpi.dates import Seconds, from_timestamp
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.db import get_db_curs, get_db_session
from wrolpi.downloader import Download
from wrolpi.errors import UnknownDirectory
from wrolpi.files.lib import split_path_stem_and_suffix
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import YTDLP_CACHE_DIR, PYTEST
from .common import is_valid_poster, convert_image, \
generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
@ -364,19 +363,23 @@ def get_channels_config_from_db(session: Session) -> dict:
return dict(channels=channels)
@optional_session()
def save_channels_config(session: Session = None):
@register_switch_handler('save_channels_config')
def save_channels_config():
"""Get the Channel information from the DB, save it to the config."""
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
with get_db_session() as session:
config = get_channels_config_from_db(session)
channels_config = get_channels_config()
if PYTEST and not channels_config:
logger.warning('Refusing to save channels config because test did not initialize a test config!')
return
channels_config.update(config)
logger.info('save_channels_config completed')
save_channels_config: ActivateSwitchMethod
channel_import_logger = logger.getChild('channel_import')
@ -391,7 +394,7 @@ async def fetch_channel_source_id(channel_id: int):
channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
else:
session.commit()
save_channels_config(session)
save_channels_config.activate_switch()
except Exception as e:
logger.error(f'Failed to get Channel source id of id={channel_id}', exc_info=e)

11
modules/videos/models.py
View file

@ -220,7 +220,7 @@ class Video(ModelHelper, Base):
return previous_video, next_video
def validate(self):
def validate(self, session: Session = None):
"""Perform a validation of this video and it's files."""
if not self.file_group.primary_path:
# Can't validate if there is no video file.
@ -245,7 +245,7 @@ class Video(ModelHelper, Base):
self.have_comments = True
# If this Video is in a Channel's directory, then it is part of that Channel.
session: Session = Session.object_session(self)
session: Session = session or Session.object_session(self)
if session and (channel := Channel.get_by_path(self.video_path.parent, session)):
self.channel = channel
logger.debug(f'{self} has Channel {channel}')
@ -371,7 +371,6 @@ class Video(ModelHelper, Base):
if not self.ffprobe_json:
from modules.videos.common import ffprobe_json
self.ffprobe_json = await ffprobe_json(self.video_path)
self.flush()
return self.ffprobe_json
@ -819,9 +818,9 @@ class Channel(ModelHelper, Base):
# Save configs before move, this is because move imports configs.
from modules.videos.lib import save_channels_config
await save_downloads_config(session)
save_channels_config(session)
save_tags_config(session)
save_downloads_config.activate_switch()
save_channels_config.activate_switch()
save_tags_config.activate_switch()
# Move the contents of the Channel directory into the destination directory.
logger.info(f'Moving {self} from {repr(str(old_directory))}')

20
modules/videos/test/test_api.py
View file

@ -14,7 +14,7 @@ from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_refresh_videos_index(test_async_client, test_session, test_directory, video_factory):
async def test_refresh_videos_index(async_client, test_session, test_directory, video_factory):
"""The video modeler indexes video data into the Video's FileGroup."""
video_factory(with_video_file=True, with_caption_file=True, with_poster_ext='jpg', with_info_json=True)
test_session.commit()
@ -162,14 +162,14 @@ def test_api_download(test_session, test_client, test_directory):
@pytest.mark.asyncio
async def test_api_download_video_tags(test_session, test_async_client, tag_factory):
async def test_api_download_video_tags(test_session, async_client, tag_factory):
"""A user can request Tags for a video being downloaded."""
tag1 = await tag_factory()
tag2 = await tag_factory()
content = dict(urls=['https://example.com/video1', ], downloader='video',
settings={'tag_names': [tag1.name, tag2.name]})
request, response = await test_async_client.post('/api/download', content=json.dumps(content))
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
@ -264,7 +264,7 @@ async def test_search_videos(test_session, video_factory, assert_video_search, s
@pytest.mark.asyncio
async def test_delete_video(test_async_client, test_session, video_factory, test_download_manager):
async def test_delete_video(async_client, test_session, video_factory, test_download_manager):
"""When a Video record is deleted, the FileGroup should be deleted. The URL is added to the
global skip list so the video is not downloaded again."""
video = video_factory(with_video_file=True, with_poster_ext='png')
@ -272,7 +272,7 @@ async def test_delete_video(test_async_client, test_session, video_factory, test
files = video.file_group.my_paths()
test_session.commit()
request, response = await test_async_client.delete(f'/api/videos/video/{video.id}')
request, response = await async_client.delete(f'/api/videos/video/{video.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Video).count() == 0, 'Video was not deleted.'
@ -282,28 +282,28 @@ async def test_delete_video(test_async_client, test_session, video_factory, test
@pytest.mark.asyncio
async def test_format_videos_description(test_async_client, test_session, test_directory, channel_factory, tag_factory):
async def test_format_videos_description(async_client, test_session, test_directory, channel_factory, tag_factory):
channel = channel_factory(name='Channel Name')
tag = await tag_factory()
test_session.commit()
body = dict()
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos'
body = dict(channel_id=channel.id)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/Channel Name'
body = dict(channel_id=channel.id, tag_name=tag.name)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/Channel Name'
get_wrolpi_config().videos_destination = 'videos/%(channel_tag)s/%(channel_domain)s/%(channel_name)s'
body = dict(channel_id=channel.id, tag_name=tag.name)
request, response = await test_async_client.post('/api/videos/tag_info', json=body)
request, response = await async_client.post('/api/videos/tag_info', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['videos_destination'] == 'videos/one/example.com/Channel Name'

14
modules/videos/test/test_common.py
View file

@ -11,6 +11,7 @@ from modules.videos.models import Channel, Video
from wrolpi.common import get_absolute_media_path, get_wrolpi_config
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.files import lib as files_lib
from wrolpi.switches import await_switches
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import convert_image, update_view_counts_and_censored, get_video_duration, generate_video_poster, \
@ -135,7 +136,7 @@ def test_generate_video_poster(video_file):
@pytest.mark.asyncio
async def test_import_channel_downloads(test_async_client, test_session, channel_factory, test_channels_config,
async def test_import_channel_downloads(async_client, test_session, channel_factory, test_channels_config,
tag_factory):
"""Importing the Channels' config should create any missing download records"""
channel1 = channel_factory(source_id='foo', url='https://example.com/channel1')
@ -155,6 +156,8 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
# Two Channels, but neither have Downloads.
save_channels_config()
await await_switches()
# Import after config is saved.
import_channels_config()
assert not channel1.downloads
assert not channel2.downloads
@ -167,6 +170,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
{'downloads': [
{'url': 'https://example.com/channel1', 'frequency': DownloadFrequency.weekly}
]})
await await_switches()
# Download record is created on import.
import_channels_config()
@ -189,6 +193,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
save_channels_config()
Download.find_by_url(channel2.url).delete(add_to_skip_list=False)
test_session.commit()
await await_switches()
# Missing Download is re-created on import.
import_channels_config()
@ -204,6 +209,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
channel2.get_or_create_download('https://example.com/channel2', 60, session=test_session)
test_session.commit()
save_channels_config()
await await_switches()
# Check config is written to match new URLs.
config = get_channels_config()
assert len(config.channels) == 2
@ -242,7 +248,7 @@ async def test_import_channel_downloads(test_async_client, test_session, channel
@pytest.mark.asyncio
async def test_import_channel_delete_missing_channels(
test_async_client, test_session, channel_factory, test_channels_config):
async_client, test_session, channel_factory, test_channels_config):
"""The Channel import function deletes any Channels that are not in the config."""
# Create a DB and config with two Channels.
channel1 = channel_factory(source_id='foo')
@ -253,6 +259,7 @@ async def test_import_channel_delete_missing_channels(
test_session.delete(channel1)
test_session.delete(channel2)
test_session.commit()
await await_switches()
# Importing the config creates two Channels.
import_channels_config()
@ -265,6 +272,7 @@ async def test_import_channel_delete_missing_channels(
config_dict = config.dict()
config_dict['channels'] = [i for i in config.channels if i['directory'] != str(channel2.directory)]
config.update(config_dict)
await await_switches()
assert str(channel1.directory) in test_channels_config.read_text()
assert str(channel2.directory) not in test_channels_config.read_text()
@ -282,7 +290,7 @@ async def test_import_channel_delete_missing_channels(
@pytest.mark.asyncio
async def test_ffprobe_json(test_async_client, video_file, corrupted_video_file):
async def test_ffprobe_json(async_client, video_file, corrupted_video_file):
content = await common.ffprobe_json(video_file)
assert not content['chapters']
assert content['format']['duration'] == '5.312000'

4
modules/videos/test/test_lib.py
View file

@ -178,7 +178,7 @@ def test_link_channel_and_downloads(test_session, channel_factory, test_download
@pytest.mark.asyncio
def test_link_channel_and_downloads_migration(test_async_client, test_session, channel_factory, test_download_manager):
def test_link_channel_and_downloads_migration(async_client, test_session, channel_factory, test_download_manager):
"""Test the 8d0d81bc9c34_channel_channel_downloads.py migration."""
# A simple Channel which has a download for its URL.
channel1 = channel_factory(url='https://example.com/channel1')
@ -220,7 +220,7 @@ def test_link_channel_and_downloads_migration(test_async_client, test_session, c
@pytest.mark.asyncio
async def test_format_videos_destination(test_async_client, test_directory):
async def test_format_videos_destination(async_client, test_directory):
"""Videos destination is formatted according to the WROLPiConfig."""
wrolpi_config = get_wrolpi_config()

2
modules/videos/test/test_models.py
View file

@ -19,7 +19,7 @@ async def test_ffprobe_stream_methods(simple_video):
@pytest.mark.asyncio
async def test_channel_move(test_async_client, test_session, test_directory, channel_factory, video_factory):
async def test_channel_move(async_client, test_session, test_directory, channel_factory, video_factory):
"""A Channel can be moved to another directory, any files in the Channel's directory are moved."""
channel = channel_factory(name='Channel Name')
video = video_factory(title='Vid', channel_id=channel.id)

6
modules/videos/test/test_video.py
View file

@ -30,7 +30,7 @@ def test_delete_video_no_channel(test_session, simple_video):
@pytest.mark.asyncio
async def test_delete_video_with_tag(test_async_client, test_session, video_factory, tag_factory):
async def test_delete_video_with_tag(async_client, test_session, video_factory, tag_factory):
"""You cannot delete a video with a tag."""
video = video_factory(with_video_file=True, with_poster_ext='png')
tag = await tag_factory()
@ -47,7 +47,7 @@ async def test_delete_video_with_tag(test_async_client, test_session, video_fact
@pytest.mark.asyncio
async def test_video_channel_refresh(test_async_client, test_session, test_directory, channel_factory, video_factory):
async def test_video_channel_refresh(async_client, test_session, test_directory, channel_factory, video_factory):
"""A Video will be associated with a Channel when it's files are in that Channel's directory."""
# Create a video file in this channel's directory.
channel = channel_factory()
@ -134,7 +134,7 @@ async def test_delete_duplicate_video(test_session, channel_factory, video_facto
@pytest.mark.asyncio
async def test_delete_renamed_video(test_async_client, test_session, channel_factory, video_factory, tag_factory):
async def test_delete_renamed_video(async_client, test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'

3
modules/videos/video/api.py
View file

@ -12,6 +12,7 @@ from wrolpi.events import Events
from wrolpi.schema import JSONErrorResponse
from . import lib
from .. import schema
from ..lib import save_channels_config
video_bp = Blueprint('Video', '/api/videos')
@ -59,7 +60,6 @@ async def search_videos(_: Request, body: schema.VideoSearchRequest):
@openapi.response(HTTPStatus.NO_CONTENT)
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@wrol_mode_check
@run_after(lib.save_channels_config)
async def video_delete(_: Request, video_ids: str):
try:
video_ids = [int(i) for i in str(video_ids).split(',')]
@ -68,5 +68,6 @@ async def video_delete(_: Request, video_ids: str):
lib.delete_videos(*video_ids)
save_channels_config.activate_switch()
Events.send_deleted(f'Deleted {len(video_ids)} videos')
return response.empty()

26
modules/videos/video/test/test_api.py
View file

@ -9,7 +9,7 @@ from modules.videos.models import Video
from wrolpi.dates import now
def test_get_video_prev_next(test_async_client, test_session, channel_factory, video_factory):
def test_get_video_prev_next(async_client, test_session, channel_factory, video_factory):
"""
Test that the previous and next videos will be retrieved when fetching a video.
"""
@ -69,7 +69,7 @@ def test_get_video_prev_next(test_async_client, test_session, channel_factory, v
raise AssertionError(f'Assert failed for {id_=} {prev_title=} {next_title=}') from e
def test_get_video_prev_next_no_upload_date(test_async_client, test_session, video_factory, channel_factory):
def test_get_video_prev_next_no_upload_date(async_client, test_session, video_factory, channel_factory):
"""
Previous and Next Videos should be those next to a Video file when videos do not have upload dates.
"""
@ -108,7 +108,7 @@ def test_get_video_prev_next_no_upload_date(test_async_client, test_session, vid
@pytest.mark.asyncio
async def test_video_delete(test_async_client, test_session, channel_factory, video_factory, test_download_manager):
async def test_video_delete(async_client, test_session, channel_factory, video_factory, test_download_manager):
"""Video.delete() removes the video's files, but leave the DB record."""
channel1, channel2 = channel_factory(), channel_factory()
vid1 = video_factory(channel_id=channel1.id, with_video_file=True, with_caption_file=True,
@ -125,7 +125,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert vid1_video_path.is_file() and vid1_caption_path.is_file()
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
request, response = await test_async_client.delete(f'/api/videos/video/{vid1.id}')
request, response = await async_client.delete(f'/api/videos/video/{vid1.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
# Video was added to skip list.
@ -135,7 +135,7 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert vid1_video_path.is_file() is False and vid1_caption_path.is_file() is False
assert vid2_video_path.is_file() and vid2_info_json_path.is_file()
request, response = await test_async_client.delete(f'/api/videos/video/{vid2.id}')
request, response = await async_client.delete(f'/api/videos/video/{vid2.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_download_manager.is_skipped(vid1.file_group.url, vid2.file_group.url)
@ -144,12 +144,12 @@ async def test_video_delete(test_async_client, test_session, channel_factory, vi
assert vid2_video_path.is_file() is False and vid2_info_json_path.is_file() is False
# 3 does not exist.
request, response = await test_async_client.delete(f'/api/videos/video/3')
request, response = await async_client.delete(f'/api/videos/video/3')
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest.mark.asyncio
async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_mode_fixture,
async def test_wrol_mode(async_client, simple_channel, simple_video, wrol_mode_fixture,
test_download_manager, tag_factory):
"""Many methods are blocked when WROL Mode is enabled."""
channel = dumps(dict(name=simple_channel.name, directory=str(simple_channel.directory)))
@ -158,29 +158,29 @@ async def test_wrol_mode(test_async_client, simple_channel, simple_video, wrol_m
await wrol_mode_fixture(True)
# Can't create, update, or delete a channel.
_, resp = await test_async_client.post('/api/videos/channels', content=channel)
_, resp = await async_client.post('/api/videos/channels', content=channel)
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
_, resp = await test_async_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
_, resp = await async_client.put(f'/api/videos/channels/{simple_channel.id}', content=channel)
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
_, resp = await test_async_client.delete(f'/api/videos/channels/{simple_channel.id}')
_, resp = await async_client.delete(f'/api/videos/channels/{simple_channel.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
# Can't delete a video
_, resp = await test_async_client.delete(f'/api/videos/video/{simple_video.id}')
_, resp = await async_client.delete(f'/api/videos/video/{simple_video.id}')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
# Can't refresh
_, resp = await test_async_client.post('/api/files/refresh')
_, resp = await async_client.post('/api/files/refresh')
assert resp.status_code == HTTPStatus.FORBIDDEN
assert resp.json['code'] == 'WROL_MODE_ENABLED'
# THE REST OF THESE METHODS ARE ALLOWED
content = dict(file_group_id=simple_video.file_group_id, tag_name=tag.name)
_, resp = await test_async_client.post('/api/files/tag', content=json.dumps(content))
_, resp = await async_client.post('/api/files/tag', content=json.dumps(content))
assert resp.status_code == HTTPStatus.CREATED
assert test_download_manager.stopped.is_set()

4
modules/zim/conftest.py
View file

@ -100,14 +100,14 @@ def zim_factory(test_session, zim_path_factory, test_directory):
@pytest.fixture
def assert_zim_search(test_async_client, test_directory):
def assert_zim_search(async_client, test_directory):
from http import HTTPStatus
async def _(search_str: str, zim_id: int, expected: Dict, tag_names: List[str] = [],
expected_status_code: int = HTTPStatus.OK):
content = dict(search_str=search_str, tag_names=tag_names)
assert isinstance(zim_id, int), 'This fixture only supports Zim ids of type integer'
request, response = await test_async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
request, response = await async_client.post(f'api/zim/search/{zim_id}', content=json.dumps(content))
assert response.status_code == expected_status_code
result = response.json['zim']

4
modules/zim/models.py
View file

@ -192,7 +192,7 @@ class Zim(Base):
else Tag.get_by_id(tag_id_or_name, session)
tag_zim_entry = TagZimEntry(tag=tag, zim=self, zim_entry=zim_entry)
session.add(tag_zim_entry)
tags.save_tags_config()
tags.save_tags_config.activate_switch()
return tag_zim_entry
def untag_entry(self, tag_id_or_name: int | str, zim_entry: str):
@ -207,7 +207,7 @@ class Zim(Base):
session.delete(tag_zim_entry)
else:
raise UnknownZimTagEntry(f'Could not find at {repr(str(zim_entry))}')
tags.save_tags_config()
tags.save_tags_config.activate_switch()
@staticmethod
def _entries_with_tags_process(limit: int, offset: int, params: dict, session: Session, tags_sub_select: str):

28
modules/zim/test/test_api.py
View file

@ -9,7 +9,7 @@ from wrolpi.downloader import Download
@pytest.mark.asyncio
async def test_zim_search(test_async_client, test_session, test_zim, tag_factory, assert_zim_search):
async def test_zim_search(async_client, test_session, test_zim, tag_factory, assert_zim_search):
"""Entries can be tagged and retrieved using their containing Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
@ -80,7 +80,7 @@ async def test_zim_search(test_async_client, test_session, test_zim, tag_factory
@pytest.mark.asyncio
async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_factory):
async def test_entries_by_tag(async_client, test_session, test_zim, tag_factory):
"""Entries can be tagged and retrieved using their containing Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_zim.tag_entry(tag1.name, 'one')
@ -99,37 +99,37 @@ async def test_entries_by_tag(test_async_client, test_session, test_zim, tag_fac
@pytest.mark.asyncio
async def test_zim_crud(test_async_client, test_session, test_directory, test_zim):
request, response = await test_async_client.get('/api/zim')
async def test_zim_crud(async_client, test_session, test_directory, test_zim):
request, response = await async_client.get('/api/zim')
assert response.status_code == HTTPStatus.OK
assert len(response.json['zims']) == 1
assert response.json['zims'][0]['path'] == test_zim.path.name
request, response = await test_async_client.delete('/api/zim/1')
request, response = await async_client.delete('/api/zim/1')
assert response.status_code == HTTPStatus.NO_CONTENT
assert not test_zim.path.exists()
assert test_session.query(Zim).count() == 0
@pytest.mark.asyncio
async def test_zim_tag_and_untag(test_async_client, test_session, test_zim, tag_factory):
async def test_zim_tag_and_untag(async_client, test_session, test_zim, tag_factory):
"""Zim entries can be Tagged and Untagged."""
tag1, tag2 = await tag_factory(), await tag_factory()
test_session.commit()
# Entries can be tagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = await test_async_client.post('/api/zim/tag', content=json.dumps(content))
request, response = await async_client.post('/api/zim/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
# Entries can be untagged in the API.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
request, response = await async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
# Cannot untag twice.
content = dict(tag_name=tag1.name, zim_id=test_zim.id, zim_entry='home')
request, response = await test_async_client.post('/api/zim/untag', content=json.dumps(content))
request, response = await async_client.post('/api/zim/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NOT_FOUND
@ -199,7 +199,7 @@ def test_get_zim_entry(test_session, test_client, test_zim):
@pytest.mark.asyncio
async def test_find_outdated_zim_files(test_session, test_directory, test_zim_bytes, test_async_client):
async def test_find_outdated_zim_files(test_session, test_directory, test_zim_bytes, async_client):
"""Outdated Zim files are found and reported. They can automatically be deleted."""
# Can search empty directory.
outdated, current = lib.find_outdated_zim_files()
@ -217,11 +217,11 @@ async def test_find_outdated_zim_files(test_session, test_directory, test_zim_by
asdf.write_bytes(test_zim_bytes)
empty = test_directory / 'zims/wikipedia_en_all_maxi_2023-03.zim' # empty files should be ignored.
empty.touch()
request, response = await test_async_client.get('/api/zim/outdated')
request, response = await async_client.get('/api/zim/outdated')
assert response.status_code == HTTPStatus.OK
assert response.json['outdated'] == [str(wikipedia_one.relative_to(test_directory)), ]
request, response = await test_async_client.delete('/api/zim/outdated')
request, response = await async_client.delete('/api/zim/outdated')
assert response.status_code == HTTPStatus.NO_CONTENT
# Only the outdated was deleted.
assert wikibooks_one.is_file()
@ -232,12 +232,12 @@ async def test_find_outdated_zim_files(test_session, test_directory, test_zim_by
@pytest.mark.asyncio
async def test_search_estimates(test_session, test_async_client, zim_factory):
async def test_search_estimates(test_session, async_client, zim_factory):
zim_factory('test zim')
test_session.commit()
body = dict(search_str='one')
request, response = await test_async_client.post('/api/zim/search_estimates', json=body)
request, response = await async_client.post('/api/zim/search_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert len(response.json['zims_estimates']) == 1
assert response.json['zims_estimates'][0]['estimate'] == 2

23
modules/zim/test/test_lib.py
View file

@ -12,10 +12,11 @@ from wrolpi.common import DownloadFileInfo, get_wrolpi_config
from wrolpi.downloader import Download, import_downloads_config, save_downloads_config
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.switches import await_switches
@pytest.mark.asyncio
async def test_get_zim(test_async_client, test_session, zim_path_factory):
async def test_get_zim(async_client, test_session, zim_path_factory):
zim_path_factory()
await files_lib.refresh_files()
@ -26,7 +27,7 @@ async def test_get_zim(test_async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
async def test_zim_get_entry(async_client, test_session, zim_path_factory):
zim_path_factory()
await files_lib.refresh_files()
@ -50,14 +51,14 @@ async def test_zim_get_entry(test_async_client, test_session, zim_path_factory):
@pytest.mark.asyncio
async def test_zim_bad_entry(test_async_client, test_session, test_zim, tag_factory):
async def test_zim_bad_entry(async_client, test_session, test_zim, tag_factory):
tag1 = await tag_factory()
with pytest.raises(UnknownZimEntry):
test_zim.tag_entry(tag1.name, 'path does not exist')
@pytest.mark.asyncio
async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, tag_factory):
async def test_zim_get_entries_tags(async_client, test_session, test_zim, tag_factory):
tag1, tag2 = await tag_factory('tag1'), await tag_factory('tag2')
test_zim.tag_entry(tag1.name, 'one')
test_zim.tag_entry(tag2.name, 'one')
@ -69,7 +70,7 @@ async def test_zim_get_entries_tags(test_async_client, test_session, test_zim, t
@pytest.mark.asyncio
async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory, tag_factory):
async def test_tag_zim_entry_model(async_client, test_session, zim_factory, tag_factory):
tag = await tag_factory()
tze = zim_factory('zim1').tag_entry(tag.name, 'one')
# Test TagZimEntry.__repr__
@ -77,7 +78,7 @@ async def test_tag_zim_entry_model(test_async_client, test_session, zim_factory,
@pytest.mark.asyncio
async def test_zim_entries_with_tags(test_async_client, test_session, zim_factory, tag_factory):
async def test_zim_entries_with_tags(async_client, test_session, zim_factory, tag_factory):
"""Can fetch Zim Entries that have Tags. Can filter only those entries that are tagged on a specific Zim."""
tag1, tag2 = await tag_factory(), await tag_factory()
zim1, zim2 = zim_factory('zim1'), zim_factory('zim2')
@ -126,7 +127,8 @@ def test_get_unique_paths():
@pytest.mark.asyncio
async def test_zim_tags_config(test_session, test_directory, test_zim, tag_factory, test_tags_config, fake_now):
async def test_zim_tags_config(async_client, test_session, test_directory, test_zim, tag_factory, test_tags_config,
fake_now):
fake_now(datetime.datetime(2000, 1, 1, 0, 0, 0, 1))
config = tags.get_tags_config()
assert not config.tag_zims
@ -137,6 +139,7 @@ async def test_zim_tags_config(test_session, test_directory, test_zim, tag_facto
test_zim.tag_entry(tag2.name, 'one')
test_zim.tag_entry(tag1.name, 'two')
test_session.commit()
await await_switches()
# Tags are created in empty database.
assert {i[0] for i in test_session.query(tags.Tag.id)} == {1, 2}
@ -150,6 +153,7 @@ async def test_zim_tags_config(test_session, test_directory, test_zim, tag_facto
# Remove a tag, the config should change.
test_zim.untag_entry(tag1.name, 'one')
test_session.commit()
await await_switches()
config = tags.get_tags_config()
assert config.tag_zims == [
@ -246,7 +250,7 @@ async def test_zim_download(test_session, kiwix_download_zim, test_directory, te
@pytest.mark.asyncio
async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_factory, test_tags_config):
async def test_zim_tag_migration(async_client, test_session, test_directory, zim_factory, tag_factory, test_tags_config):
"""When an outdated Zim file is deleted, the Tags should be moved to the latest Zim."""
zim1 = zim_factory('wikipedia_en_all_maxi_2020-01.zim') # The outdated Zim.
zim2 = zim_factory('wikipedia_en_all_maxi_2020-02.zim')
@ -261,6 +265,7 @@ async def test_zim_tag_migration(test_session, test_directory, zim_factory, tag_
zim3.tag_entry(tag2.name, 'one')
test_session.commit()
assert test_session.query(TagZimEntry).count() == 4
await await_switches()
# All Zims have been tagged.
tags_config = tags.get_tags_config()
@ -352,7 +357,7 @@ async def test_zim_subscription_download_import(test_session, test_downloader_co
assert download.url == 'https://download.kiwix.org/zim/wikisource/wikisource_en_all_maxi_'
# Save config file, delete all entries.
await save_downloads_config()
save_downloads_config()
test_session.query(Download).delete()
test_session.query(ZimSubscription).delete()
test_session.commit()

57
wrolpi/api_utils.py
View file

@ -1,4 +1,7 @@
import asyncio
import json
import logging
from asyncio import CancelledError
from datetime import datetime, timezone, date
from decimal import Decimal
from functools import wraps
@ -106,3 +109,57 @@ def json_error_handler(request: Request, exception: Exception):
# Some unknown error, use internal error code.
return json_response(body, HTTPStatus.INTERNAL_SERVER_ERROR)
PERPETUAL_WORKERS = list()
@api_app.after_server_start
async def start_perpetual_tasks(app: Sanic):
# Only one set of perpetual tasks needs to be started.
if app.shared_ctx.perpetual_tasks_started.is_set():
return
logger.info('start_perpetual_tasks started')
app.shared_ctx.perpetual_tasks_started.set()
try:
for event_ in PERPETUAL_WORKERS:
logger.debug(f'start_perpetual_tasks {event_}')
await app.dispatch(event_)
except Exception as e:
logger.error('Failed to start perpetual tasks', exc_info=e)
raise
logger.debug('start_perpetual_tasks completed')
def perpetual_signal(event: str = None, sleep: int | float = 1):
"""Use Sanic signals to continually call the wrapped function. The wrapped function will continually be called,
even if it has errors. If the function is long-running, it will only be called again after it has finished."""
def wrapper(func: callable):
# Create a Sanic "signal" for the provided function.
event_ = event or f'wrolpi.perpetual.{func.__name__}'
# Wrap the function in a worker that will call it perpetually.
@api_app.signal(event_)
async def worker(*args, **kwargs):
logger.log(logging.NOTSET, f'perpetual_signal {event_}')
cancelled = False
try:
await func(*args, **kwargs)
except CancelledError:
cancelled = True
raise
except Exception as e:
logger.error(f'Perpetual worker {event_} had error', exc_info=e)
finally:
if not cancelled:
await asyncio.sleep(sleep)
await api_app.dispatch(event_)
# Add this new signal to the global list so that a task will be started after server startup.
PERPETUAL_WORKERS.append(event_)
return func
return wrapper

26
wrolpi/common.py
View file

@ -349,14 +349,21 @@ class ConfigFile:
file_name: str = None
default_config: dict = None
width: int = None
background_save: callable = None
def __init__(self):
self.width = self.width or 90
if PYTEST:
# Do not load a global config on import while testing. A global instance should never be used for testing.
self._config = self.default_config.copy()
return
# Do not load a global config on import while testing. A global instance should never be used for testing.
self._config = self.default_config.copy()
from wrolpi.switches import register_switch_handler
@register_switch_handler(f'background_save_{self.file_name}')
def background_save():
self.save()
self.background_save = background_save
def __repr__(self):
return f'<{self.__class__.__name__} file={self.get_file()}>'
@ -385,7 +392,7 @@ class ConfigFile:
path = path.with_name(name)
return path
def save(self, ignore_lock: bool = False):
def save(self):
"""
Write this config to its file.
@ -401,11 +408,8 @@ class ConfigFile:
raise ValueError(f'Refusing to save config file while testing: {config_file}')
# Only one process can write to a config.
acquired = False
lock = None
if ignore_lock is False:
lock = api_app.shared_ctx.config_save_lock
acquired = lock.acquire(block=True, timeout=5.0)
lock = api_app.shared_ctx.config_save_lock
acquired = lock.acquire(block=True, timeout=5.0)
try:
# Config directory may not exist.
@ -457,7 +461,7 @@ class ConfigFile:
"""Update any values of this config. Save the config to its file."""
config = {k: v for k, v in config.items() if k in self._config}
self._config.update(config)
self.save(ignore_lock=ignore_lock)
self.background_save.activate_switch()
def dict(self) -> dict:
"""Get a deepcopy of this config."""

36
wrolpi/conftest.py
View file

@ -45,6 +45,7 @@ from wrolpi.downloader import DownloadManager, DownloadResult, Download, Downloa
downloads_manager_config_context
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.models import Directory, FileGroup
from wrolpi.switches import await_switches
from wrolpi.tags import Tag, upsert_tag
from wrolpi.vars import PROJECT_DIR
@ -164,18 +165,33 @@ def test_client(test_directory) -> ReusableClient:
raise RuntimeError('Test never got unused port')
@pytest.fixture
async def test_async_client(test_directory) -> SanicASGITestClient:
"""Get an Async Sanic Test Client with all default routes attached."""
attach_shared_contexts(api_app)
@api_app.on_response
async def background_task_listener(request, response):
"""Wait for all background tasks to finish before returning API response while testing."""
await await_switches()
@pytest.fixture
async def async_client(test_directory) -> SanicASGITestClient:
"""Get an Async Sanic Test Client with all default routes attached."""
api_app.signalize()
attach_shared_contexts(api_app)
initialize_configs_contexts(api_app)
return SanicASGITestClient(api_app)
# Call API so server actually starts.
client = SanicASGITestClient(api_app)
await client.get('/api/echo')
try:
yield client
# Do not remove, or tests will be significantly slower.
# await switches.await_switches()
finally:
logger.debug('Destroying async_client')
@pytest.fixture
def test_download_manager_config(test_async_client, test_directory) -> pathlib.Path:
def test_download_manager_config(async_client, test_directory) -> pathlib.Path:
with downloads_manager_config_context():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/download_manager.yaml'
@ -184,13 +200,11 @@ def test_download_manager_config(test_async_client, test_directory) -> pathlib.P
@pytest.fixture
async def test_download_manager(
test_async_client,
async_client,
test_session, # session is required because downloads can start without the test DB in place.
test_download_manager_config,
) -> DownloadManager:
# Needed to use signals in test app?
api_app.signalize()
# Needed to use signals in
manager = DownloadManager()
await manager.enable()
@ -486,7 +500,7 @@ def mock_create_subprocess_shell() -> Callable:
@pytest.fixture
def events_history(test_async_client):
def events_history(async_client):
"""Give each test its own Events history."""
yield api_app.shared_ctx.events_history

16
wrolpi/contexts.py
View file

@ -31,7 +31,6 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.status = manager.dict()
app.shared_ctx.map_importing = manager.dict()
# Shared lists.
app.shared_ctx.events_history = manager.list()
# Shared ints
app.shared_ctx.log_level = multiprocessing.Value(ctypes.c_int, default_log_level)
@ -41,16 +40,24 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.download_manager_disabled = multiprocessing.Event()
app.shared_ctx.download_manager_stopped = multiprocessing.Event()
# Events.
app.shared_ctx.single_tasks_started = multiprocessing.Event()
app.shared_ctx.flags_initialized = multiprocessing.Event()
app.shared_ctx.perpetual_tasks_started = multiprocessing.Event()
# Switches
app.shared_ctx.switches = manager.dict()
app.shared_ctx.switches_lock = multiprocessing.Lock()
app.shared_ctx.switches_changed = multiprocessing.Event()
# Warnings
app.shared_ctx.warn_once = manager.dict()
# Locks
app.shared_ctx.config_save_lock = multiprocessing.Lock()
# Events.
app.shared_ctx.events_lock = multiprocessing.Lock()
app.shared_ctx.events_history = manager.list()
reset_shared_contexts(app)
@ -88,11 +95,16 @@ def reset_shared_contexts(app: Sanic):
killed_downloads=[],
))
# Switches
app.shared_ctx.switches.clear()
app.shared_ctx.switches_changed.clear()
# Events.
app.shared_ctx.single_tasks_started.clear()
app.shared_ctx.download_manager_disabled.clear()
app.shared_ctx.download_manager_stopped.clear()
app.shared_ctx.flags_initialized.clear()
app.shared_ctx.perpetual_tasks_started.clear()
def initialize_configs_contexts(app: Sanic):

95
wrolpi/downloader.py
View file

@ -24,44 +24,31 @@ from sqlalchemy.orm import Session, relationship
from sqlalchemy.sql import Delete
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import pid_is_running
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \
limit_concurrent, wrol_mode_enabled, background_task
from wrolpi.dates import TZDateTime, now, Seconds
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError, UnknownDownload
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod, await_switches
from wrolpi.vars import PYTEST, SIMULTANEOUS_DOWNLOAD_DOMAINS
logger = logger.getChild(__name__)
@perpetual_signal(sleep=1)
async def perpetual_download_worker():
logger.debug('perpetual_download waiting for db...')
async with flags.db_up.wait_for():
pass
logger.debug('perpetual_download starting...')
async with flags.have_internet.wait_for(timeout=5):
pass
while True:
try:
await asyncio.sleep(1)
if download_manager.is_stopped:
return
async with flags.have_internet.wait_for(timeout=5):
pass
if download_manager.is_stopped:
logger.warning('DownloadManager stopped, quitting...')
return
await download_manager.do_downloads()
except asyncio.CancelledError:
logger.info('perpetual_download cancelled...')
return
except TimeoutError:
logger.debug('DownloadManager waiting for internet...')
except Exception as e:
logger.error('perpetual_download failed from unexpected error', exc_info=e)
await download_manager.do_downloads()
class DownloadFrequency(int, Enum):
@ -224,9 +211,9 @@ class Download(ModelHelper, Base): # noqa
if self.channel_id:
# Save Channels config if this download was associated with a Channel.
from modules.videos.lib import save_channels_config
save_channels_config()
save_channels_config.activate_switch()
# Save download config again because this download is now removed from the download lists.
background_task(save_downloads_config())
save_downloads_config.activate_switch()
@staticmethod
def get_by_id(id_: int, session: Session = None) -> Optional[Base]:
@ -548,7 +535,7 @@ class DownloadManager:
# Start downloading ASAP.
background_task(self.dispatch_downloads())
# Save the config now that new Downloads exist.
background_task(save_downloads_config())
save_downloads_config.activate_switch()
except RuntimeError:
# Event loop isn't running. Probably testing?
if not PYTEST:
@ -658,6 +645,8 @@ class DownloadManager:
# Wait for processes to start.
await asyncio.sleep(1)
await await_switches()
# Break out of loop only when all downloads have been processed.
with get_db_session() as session:
statuses = {i.status for i in self.get_downloads(session)}
@ -673,7 +662,7 @@ class DownloadManager:
with get_db_curs(commit=True) as curs:
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
background_task(save_downloads_config())
save_downloads_config.activate_switch()
DOWNLOAD_SORT = (
DownloadStatus.pending,
@ -747,7 +736,7 @@ class DownloadManager:
session.commit()
# Save the config now that some Downloads renewed.
background_task(save_downloads_config())
save_downloads_config.activate_switch()
@staticmethod
def get_downloads(session: Session) -> List[Download]:
@ -1030,7 +1019,7 @@ class DownloadManager:
stmt = self._delete_downloads_q(once=True)
deleted_ids = [i for i, in session.execute(stmt).fetchall()]
session.commit()
background_task(save_downloads_config())
save_downloads_config.activate_switch()
return deleted_ids
@staticmethod
@ -1135,7 +1124,7 @@ async def signal_download_download(download_id: int, download_url: str):
# Allow the download to resume.
download_manager.unkill_download(download_id)
# Save the config now that the Download has finished.
background_task(save_downloads_config())
save_downloads_config.activate_switch()
except asyncio.CancelledError as e:
worker_logger.warning('Canceled!', exc_info=e)
return
@ -1193,28 +1182,33 @@ def get_download_manager_config() -> DownloadManagerConfig:
return DOWNLOAD_MANAGER_CONFIG
@optional_session(commit=False)
async def save_downloads_config(session: Session):
@register_switch_handler('save_downloads_config')
def save_downloads_config():
"""Fetch all Downloads from the DB, save them to the Download Manager Config."""
downloads: Iterable[Download] = session.query(Download).order_by(Download.url)
config = []
for download in downloads:
if download.last_successful_download and not download.frequency:
# This once-download has completed, do not save it.
continue
config.append(dict(
downloader=download.downloader,
frequency=download.frequency,
last_successful_download=download.last_successful_download,
next_download=download.next_download,
settings=download.settings,
status=download.status,
sub_downloader=download.sub_downloader,
url=download.url,
))
if config != get_download_manager_config().downloads:
# Only save if there are changes.
get_download_manager_config().downloads = config
with get_db_session() as session:
downloads: Iterable[Download] = session.query(Download).order_by(Download.url)
config = []
for download in downloads:
if download.last_successful_download and not download.frequency:
# This once-download has completed, do not save it.
continue
config.append(dict(
downloader=download.downloader,
frequency=download.frequency,
last_successful_download=download.last_successful_download,
next_download=download.next_download,
settings=download.settings,
status=download.status,
sub_downloader=download.sub_downloader,
url=download.url,
))
if config != get_download_manager_config().downloads:
# Only save if there are changes.
get_download_manager_config().downloads = config
logger.info('save_downloads_config completed')
save_downloads_config: ActivateSwitchMethod
@optional_session
@ -1234,7 +1228,7 @@ async def import_downloads_config(session: Session):
logger.warning('Importing downloads in config')
downloads_by_url = {i['url']: i for i in get_download_manager_config().downloads}
existing_downloads = list(session.query(Download))
existing_downloads: List[Download] = list(session.query(Download))
for existing in existing_downloads:
download = downloads_by_url.pop(existing.url, None)
if download:
@ -1247,6 +1241,7 @@ async def import_downloads_config(session: Session):
existing.status = download['status']
existing.sub_downloader = download['sub_downloader']
existing.settings = download.get('settings') or dict()
existing.flush()
else:
logger.warning(f'Deleting Download {existing.url} because it is no longer in the config')
session.delete(existing)

42
wrolpi/files/lib.py
View file

@ -1415,7 +1415,7 @@ async def move(destination: pathlib.Path, *sources: pathlib.Path, session: Sessi
with flags.refresh_cleanup:
await apply_refresh_cleanup()
# Save tags now that files have been moved.
save_tags_config()
save_tags_config.activate_switch()
return plan
@ -1525,8 +1525,7 @@ def remove_ignored_directory(directory: Union[pathlib.Path, str]):
raise UnknownDirectory('Directory is not ignored')
@optional_session
async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_names: List[str] = None) -> FileGroup:
async def upsert_file(file: pathlib.Path | str, tag_names: List[str] = None) -> FileGroup:
"""Insert/update all files in the provided file's FileGroup."""
# Update/Insert all files in the FileGroup.
paths = glob_shared_stem(pathlib.Path(file))
@ -1537,31 +1536,38 @@ async def upsert_file(file: pathlib.Path | str, session: Session = None, tag_nam
return
for i in range(2):
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.
try:
session.flush([file_group, ])
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
session.rollback()
continue
with get_db_session() as session:
file_group = FileGroup.from_paths(session, *paths)
# Re-index the contents of the file.
try:
session.flush([file_group, ])
file_group_id = file_group.id
session.commit()
break
except IntegrityError:
# Another process inserted this FileGroup.
logger.error(f'upsert_file failed because FileGroup already exists, trying again... {file}')
continue
else:
raise RuntimeError(f'upsert_file failed to create FileGroup every try! {file}')
logger.debug(f'upsert_file: {file_group}')
try:
file_group.do_model(session)
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
file_group.do_model(session)
except Exception as e:
logger.error(f'Failed to model FileGroup: {file_group}', exc_info=e)
if PYTEST:
raise
for tag_name in (tag_names or []):
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
if tag_names:
with get_db_session(commit=True) as session:
file_group = FileGroup.find_by_id(file_group_id, session)
for tag_name in tag_names:
if tag_name not in file_group.tag_names:
tag = Tag.get_by_name(tag_name)
file_group.add_tag(tag.id)
session.commit()
return file_group

80
wrolpi/files/test/test_api.py
View file

@ -151,11 +151,11 @@ def test_delete_invalid_file(test_client, paths):
@pytest.mark.asyncio
async def test_delete_wrol_mode(test_async_client, wrol_mode_fixture):
async def test_delete_wrol_mode(async_client, wrol_mode_fixture):
"""Can't delete a file when WROL Mode is enabled."""
await wrol_mode_fixture(True)
request, response = await test_async_client.post('/api/files/delete', content=json.dumps({'paths': ['foo', ]}))
request, response = await async_client.post('/api/files/delete', content=json.dumps({'paths': ['foo', ]}))
assert response.status_code == HTTPStatus.FORBIDDEN
assert response.json['code'] == 'WROL_MODE_ENABLED'
@ -189,7 +189,7 @@ def test_files_search(test_session, test_client, make_files_structure, assert_fi
@pytest.mark.asyncio
async def test_files_search_any_tag(test_async_client, test_session, make_files_structure, tag_factory):
async def test_files_search_any_tag(async_client, test_session, make_files_structure, tag_factory):
one, two = await tag_factory(), await tag_factory()
files = [
'foo.txt',
@ -208,25 +208,25 @@ async def test_files_search_any_tag(test_async_client, test_session, make_files_
# Only `foo` is tagged with `one`
body = dict(search_str='foo', tag_names=['one'])
request, response = await test_async_client.post('/api/files/search', json=body)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt'}
# Both `foo bar` and `foo` are tagged.
body = dict(search_str='foo', any_tag=True)
request, response = await test_async_client.post('/api/files/search', json=body)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo.txt', 'foo bar.txt'}
# `bar` is not tagged.
body = dict(search_str='bar', any_tag=True)
request, response = await test_async_client.post('/api/files/search', json=body)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.OK
assert {i['primary_path'] for i in response.json['file_groups']} == {'foo bar.txt'}
# Cannot search for any_tag, and tag names.
body = dict(search_str='foo', tag_names=['one'], any_tag=True)
request, response = await test_async_client.post('/api/files/search', json=body)
request, response = await async_client.post('/api/files/search', json=body)
assert response.status_code == HTTPStatus.BAD_REQUEST
@ -300,7 +300,7 @@ def test_file_statistics(test_session, test_client, test_directory, example_pdf,
@pytest.mark.asyncio
async def test_file_group_tag_by_primary_path(test_session, test_async_client, test_directory, example_singlefile,
async def test_file_group_tag_by_primary_path(test_session, async_client, test_directory, example_singlefile,
tag_factory, insert_file_group):
singlefile = FileGroup.from_paths(test_session, example_singlefile)
tag1 = await tag_factory()
@ -309,25 +309,25 @@ async def test_file_group_tag_by_primary_path(test_session, test_async_client, t
# FileGroup can be tagged with its primary_path.
content = dict(file_group_primary_path=str(singlefile.primary_path.relative_to(test_directory)), tag_name=tag1.name)
request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
request, response = await async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 1
# FileGroup can be tagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
request, response = await test_async_client.post('/api/files/tag', content=json.dumps(content))
request, response = await async_client.post('/api/files/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert test_session.query(TagFile).count() == 2
# FileGroup can be untagged with its primary_path.
content = dict(file_group_id=singlefile.id, tag_id=tag1.id)
request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
request, response = await async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 1
# FileGroup can be untagged with its id.
content = dict(file_group_id=singlefile.id, tag_name=tag2.name)
request, response = await test_async_client.post('/api/files/untag', content=json.dumps(content))
request, response = await async_client.post('/api/files/untag', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(TagFile).count() == 0
@ -470,7 +470,7 @@ def test_post_upload_directory(test_session, test_client, test_directory, make_f
@pytest.mark.asyncio
async def test_post_upload(test_session, test_async_client, test_directory, make_files_structure, make_multipart_form,
async def test_post_upload(test_session, async_client, test_directory, make_files_structure, make_multipart_form,
tag_factory,
video_bytes):
"""A file can be uploaded in chunks directly to the destination."""
@ -490,7 +490,7 @@ async def test_post_upload(test_session, test_async_client, test_directory, make
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK, response.content.decode()
output = test_directory / 'uploads/video.mp4'
@ -508,7 +508,7 @@ async def test_post_upload(test_session, test_async_client, test_directory, make
dict(name='chunk', value=part2, filename='chunk'),
]
body = make_multipart_form(forms)
request, response = await test_async_client.post('/api/files/upload', content=body, headers=headers)
request, response = await async_client.post('/api/files/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.CREATED, response.content.decode()
assert output.is_file()
@ -555,7 +555,7 @@ def test_post_upload_text(test_session, test_client, test_directory, make_files_
@pytest.mark.asyncio
async def test_directory_crud(test_session, test_async_client, test_directory, assert_directories, assert_files):
async def test_directory_crud(test_session, async_client, test_directory, assert_directories, assert_files):
"""A directory can be created in a subdirectory. Errors are returned if there are conflicts."""
foo = test_directory / 'foo'
foo.mkdir()
@ -564,14 +564,14 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
# Create a subdirectory.
content = dict(path='foo/bar')
request, response = await test_async_client.post('/api/files/directory', content=json.dumps(content))
request, response = await async_client.post('/api/files/directory', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert (test_directory / 'foo/bar').is_dir()
assert_directories({'foo', 'foo/bar'})
# Cannot create twice.
content = dict(path='foo/bar')
request, response = await test_async_client.post('/api/files/directory', content=json.dumps(content))
request, response = await async_client.post('/api/files/directory', content=json.dumps(content))
assert response.status_code == HTTPStatus.CONFLICT
assert (test_directory / 'foo/bar').is_dir()
assert_directories({'foo', 'foo/bar'})
@ -580,7 +580,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
(test_directory / 'foo/bar/asdf.txt').write_text('asdf')
# Can get information about a directory.
request, response = await test_async_client.post('/api/files/get_directory', content=json.dumps(content))
request, response = await async_client.post('/api/files/get_directory', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['path'] == 'foo/bar'
assert response.json['size'] == 4
@ -588,7 +588,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
# Can rename the directory.
content = dict(path='foo/bar', new_name='baz')
request, response = await test_async_client.post('/api/files/rename', content=json.dumps(content))
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'foo/bar').exists()
assert (test_directory / 'foo/baz').exists()
@ -597,7 +597,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
# Deletion is recursive.
assert (test_directory / 'foo/baz/asdf.txt').is_file()
content = dict(paths=['foo/baz', ])
request, response = await test_async_client.post('/api/files/delete', content=json.dumps(content))
request, response = await async_client.post('/api/files/delete', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'foo/baz').exists()
assert not (test_directory / 'foo/baz/asdf.txt').exists()
@ -606,7 +606,7 @@ async def test_directory_crud(test_session, test_async_client, test_directory, a
@pytest.mark.asyncio
async def test_move(test_session, test_directory, make_files_structure, test_async_client):
async def test_move(test_session, test_directory, make_files_structure, async_client):
"""Files can be moved up and down the media directory. Destination directories should already exist."""
baz, bar = make_files_structure({
'foo/bar.txt': 'bar',
@ -615,7 +615,7 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy
# qux directory does not exist
content = dict(paths=['foo', 'baz.txt'], destination='qux')
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NOT_FOUND
(test_directory / 'qux').mkdir()
@ -623,14 +623,14 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy
# mv foo baz.txt qux
content = dict(paths=['foo', 'baz.txt'], destination='qux')
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not bar.exists()
assert (test_directory / 'qux/foo/bar.txt').is_file()
# mv foo/bar.txt qux
content = dict(paths=['qux/foo/bar.txt', ], destination='qux')
request, response = await test_async_client.post('/api/files/move', content=json.dumps(content))
request, response = await async_client.post('/api/files/move', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not bar.exists()
assert (test_directory / 'qux/bar.txt').is_file()
@ -638,19 +638,19 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy
# mv qux/bar.txt ./
content = dict(paths=['qux/bar.txt', ], destination='')
request, response = await test_async_client.post('/api/files/move', json=content)
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
assert (test_directory / 'bar.txt').is_file()
# mv bar.txt quux/quuz/bar.txt
content = dict(paths=['bar.txt', ], destination='quux/quuz')
request, response = await test_async_client.post('/api/files/move', json=content)
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
assert (test_directory / 'quux/quuz/bar.txt').is_file()
# mv quux/quuz/bar.txt quux/bar.txt
content = dict(paths=['quux/quuz/bar.txt', ], destination='quux')
request, response = await test_async_client.post('/api/files/move', json=content)
request, response = await async_client.post('/api/files/move', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
assert (test_directory / 'quux/bar.txt').is_file()
assert (test_directory / 'qux/foo').is_dir()
@ -658,14 +658,14 @@ async def test_move(test_session, test_directory, make_files_structure, test_asy
@pytest.mark.asyncio
async def test_rename_file(test_session, test_directory, make_files_structure, test_async_client):
async def test_rename_file(test_session, test_directory, make_files_structure, async_client):
foo, = make_files_structure({
'foo/bar/baz.txt': 'asdf',
})
# mv foo/bar/baz.txt foo/bar/qux.txt
content = dict(path='foo/bar/baz.txt', new_name='qux.txt')
request, response = await test_async_client.post('/api/files/rename', content=json.dumps(content))
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not foo.exists()
assert (test_directory / 'foo/bar/qux.txt').is_file()
@ -673,14 +673,14 @@ async def test_rename_file(test_session, test_directory, make_files_structure, t
@pytest.mark.asyncio
async def test_rename_directory(test_session, test_directory, make_files_structure, test_async_client):
async def test_rename_directory(test_session, test_directory, make_files_structure, async_client):
make_files_structure({
'foo/bar/baz.txt': 'asdf',
})
# mv foo/bar/baz.txt foo/bar/qux.txt
content = dict(path='foo/bar', new_name='qux')
request, response = await test_async_client.post('/api/files/rename', content=json.dumps(content))
request, response = await async_client.post('/api/files/rename', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert not (test_directory / 'foo/bar').exists()
assert (test_directory / 'foo/qux/baz.txt').is_file()
@ -688,14 +688,14 @@ async def test_rename_directory(test_session, test_directory, make_files_structu
@pytest.mark.asyncio
async def test_delete_directory_recursive(test_session, test_directory, make_files_structure, test_async_client,
async def test_delete_directory_recursive(test_session, test_directory, make_files_structure, async_client,
assert_files):
make_files_structure(['dir/foo', 'dir/bar', 'empty'])
await lib.refresh_files()
assert test_session.query(FileGroup).count() == 3
content = {'paths': ['dir/', ]}
request, response = await test_async_client.post('/api/files/delete', content=json.dumps(content))
request, response = await async_client.post('/api/files/delete', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(FileGroup).count() == 1
@ -704,20 +704,20 @@ async def test_delete_directory_recursive(test_session, test_directory, make_fil
@pytest.mark.asyncio
async def test_get_file(test_session, test_async_client, test_directory, make_files_structure):
async def test_get_file(test_session, async_client, test_directory, make_files_structure):
"""Can get info about a single file."""
make_files_structure({'foo/bar.txt': 'foo contents'})
await lib.refresh_files()
content = dict(file='foo/bar.txt')
request, response = await test_async_client.post('/api/files/file', content=json.dumps(content))
request, response = await async_client.post('/api/files/file', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['file']
assert response.json['file']['path'] == 'foo/bar.txt'
@pytest.mark.asyncio
async def test_ignore_directory(test_session, test_async_client, test_directory, make_files_structure, test_config,
async def test_ignore_directory(test_session, async_client, test_directory, make_files_structure, test_config,
skip_config_backups):
"""A maintainer can ignore/un-ignore directories. The files in the directory should not be refreshed."""
# Remove any default ignored directories.
@ -728,7 +728,7 @@ async def test_ignore_directory(test_session, test_async_client, test_directory,
# Ignore baz/
content = dict(path=str(baz.parent))
request, response = await test_async_client.post('/api/files/ignore_directory', json=content)
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert len(get_wrolpi_config().ignored_directories) == 1
@ -738,7 +738,7 @@ async def test_ignore_directory(test_session, test_async_client, test_directory,
assert {i.primary_path.name for i in files} == {'foo.txt', 'bar.txt', 'wrolpi.yaml'}
# Un-ignore baz/
request, response = await test_async_client.post('/api/files/unignore_directory', json=content)
request, response = await async_client.post('/api/files/unignore_directory', json=content)
assert response.status_code == HTTPStatus.OK
assert len(get_wrolpi_config().ignored_directories) == 0
@ -751,6 +751,6 @@ async def test_ignore_directory(test_session, test_async_client, test_directory,
# Cannot ignore special directories.
content = dict(path=str(test_directory / 'videos'))
request, response = await test_async_client.post('/api/files/ignore_directory', json=content)
request, response = await async_client.post('/api/files/ignore_directory', json=content)
assert response.status_code == HTTPStatus.BAD_REQUEST
assert len(get_wrolpi_config().ignored_directories) == 0

4
wrolpi/files/test/test_ebooks.py
View file

@ -11,7 +11,7 @@ from wrolpi.test.common import assert_dict_contains
@pytest.mark.asyncio
async def test_index(test_async_client, test_session, test_directory, example_epub, example_mobi):
async def test_index(async_client, test_session, test_directory, example_epub, example_mobi):
"""Ebooks can be indexed.
Covers can be discovered."""
@ -135,7 +135,7 @@ async def test_discover_calibre_cover(test_session, test_directory, example_epub
@pytest.mark.asyncio
async def test_move_ebook(test_async_client, test_session, test_directory, example_epub, image_file, tag_factory):
async def test_move_ebook(async_client, test_session, test_directory, example_epub, image_file, tag_factory):
"""An ebook is re-indexed when moved."""
tag = await tag_factory()
shutil.move(image_file, example_epub.with_suffix('.jpg'))

67
wrolpi/files/test/test_lib.py
View file

@ -1,5 +1,6 @@
import asyncio
import json
import pathlib
import shutil
import zipfile
from datetime import datetime
@ -19,12 +20,13 @@ from wrolpi.dates import now
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup
from wrolpi.switches import await_switches
from wrolpi.tags import TagFile
from wrolpi.vars import PROJECT_DIR
@pytest.mark.asyncio
async def test_delete_file(test_async_client, test_session, make_files_structure, test_directory):
async def test_delete_file(async_client, test_session, make_files_structure, test_directory):
"""
File in the media directory can be deleted.
"""
@ -56,7 +58,7 @@ async def test_delete_file(test_async_client, test_session, make_files_structure
@pytest.mark.asyncio
async def test_delete_file_multiple(test_async_client, test_session, make_files_structure, test_directory):
async def test_delete_file_multiple(async_client, test_session, make_files_structure, test_directory):
"""Multiple files can be deleted at once."""
foo, bar, baz = make_files_structure([
'archives/foo.txt',
@ -74,7 +76,7 @@ async def test_delete_file_multiple(test_async_client, test_session, make_files_
@pytest.mark.asyncio
async def test_delete_file_names(test_async_client, test_session, make_files_structure, test_directory, tag_factory):
async def test_delete_file_names(async_client, test_session, make_files_structure, test_directory, tag_factory):
"""Will not refuse to delete a file that shares the name of a nearby file when they are in different FileGroups."""
foo, foo1 = make_files_structure({
'archives/foo': 'text',
@ -94,7 +96,7 @@ async def test_delete_file_names(test_async_client, test_session, make_files_str
@pytest.mark.asyncio
async def test_delete_file_link(test_async_client, test_session, test_directory):
async def test_delete_file_link(async_client, test_session, test_directory):
"""Links can be deleted."""
foo, bar = test_directory / 'foo', test_directory / 'bar'
foo.touch()
@ -104,7 +106,7 @@ async def test_delete_file_link(test_async_client, test_session, test_directory)
@pytest.mark.asyncio
async def test_delete_tagged(test_async_client, test_session, make_files_structure, tag_factory, video_bytes):
async def test_delete_tagged(async_client, test_session, make_files_structure, tag_factory, video_bytes):
"""Cannot delete a file that has been tagged."""
tag = await tag_factory()
make_files_structure({'foo/bar.txt': 'asdf', 'foo/bar.mp4': video_bytes})
@ -112,6 +114,7 @@ async def test_delete_tagged(test_async_client, test_session, make_files_structu
# Both files end up in a group.
bar = test_session.query(FileGroup).one()
bar.add_tag(tag.id)
await await_switches()
test_session.commit()
# Neither file can be deleted.
@ -160,7 +163,7 @@ def test_split_path_stem_and_suffix(path, full, expected):
@pytest.mark.asyncio
async def test_refresh_files(test_async_client, test_session, make_files_structure, assert_file_groups):
async def test_refresh_files(async_client, test_session, make_files_structure, assert_file_groups):
"""All files in the media directory should be found when calling `refresh_files`"""
files = make_files_structure([
'foo.txt',
@ -187,7 +190,7 @@ async def test_refresh_files(test_async_client, test_session, make_files_structu
@pytest.mark.asyncio
async def test_file_group_location(test_async_client, test_session, make_files_structure):
async def test_file_group_location(async_client, test_session, make_files_structure):
"""FileGroup can return the URL necessary to preview the FileGroup."""
make_files_structure({
'foo/one.txt': 'one',
@ -201,7 +204,7 @@ async def test_file_group_location(test_async_client, test_session, make_files_s
@pytest.mark.asyncio
async def test_refresh_bogus_files(test_async_client, test_session, make_files_structure, test_directory,
async def test_refresh_bogus_files(async_client, test_session, make_files_structure, test_directory,
assert_file_groups, insert_file_group):
"""Bogus files are removed during a refresh."""
make_files_structure(['does exist.txt'])
@ -222,7 +225,7 @@ async def test_refresh_bogus_files(test_async_client, test_session, make_files_s
@pytest.mark.asyncio
async def test_refresh_empty_media_directory(test_async_client, test_session, test_directory):
async def test_refresh_empty_media_directory(async_client, test_session, test_directory):
"""refresh_paths will refuse to refresh with an empty media directory."""
with pytest.raises(UnknownDirectory):
await lib.refresh_files()
@ -379,7 +382,7 @@ async def test_file_group_tag(test_session, make_files_structure, test_directory
@pytest.mark.asyncio
async def test_refresh_a_text_no_indexer(test_async_client, test_session, make_files_structure):
async def test_refresh_a_text_no_indexer(async_client, test_session, make_files_structure):
"""File.a_text is filled even if the file does not match an Indexer."""
make_files_structure(['foo', 'bar-bar'])
@ -390,7 +393,7 @@ async def test_refresh_a_text_no_indexer(test_async_client, test_session, make_f
@pytest.mark.asyncio
async def test_refresh_many_files(test_async_client, test_session, make_files_structure):
async def test_refresh_many_files(async_client, test_session, make_files_structure):
"""Used to profile file refreshing"""
file_count = 10_000
make_files_structure([f'{uuid4()}.txt' for _ in range(file_count)])
@ -404,7 +407,7 @@ async def test_refresh_many_files(test_async_client, test_session, make_files_st
@pytest.mark.asyncio
async def test_refresh_cancel(test_async_client, test_session, make_files_structure, test_directory):
async def test_refresh_cancel(async_client, test_session, make_files_structure, test_directory):
"""Refresh tasks can be canceled."""
# Creat a lot of files so the refresh will take too long.
make_files_structure([f'{uuid4()}.txt' for _ in range(1_000)])
@ -413,20 +416,20 @@ async def test_refresh_cancel(test_async_client, test_session, make_files_struct
# Time the time it takes to cancel.
before = datetime.now()
# Sleep so the refresh task has time to run.
await asyncio.sleep(0.1)
await asyncio.sleep(0)
# Cancel the refresh (it will be sleeping soon).
task_.cancel()
with pytest.raises(asyncio.CancelledError):
await task_
assert (datetime.now() - before).total_seconds() < 0.8, 'Task took too long. Was the refresh canceled?'
assert (datetime.now() - before).total_seconds() < 1.2, 'Task took too long. Was the refresh canceled?'
task = asyncio.create_task(lib.refresh_files())
await assert_cancel(task)
@pytest.mark.asyncio
async def test_mime_type(test_async_client, test_session, make_files_structure, test_directory, assert_files):
async def test_mime_type(async_client, test_session, make_files_structure, test_directory, assert_files):
"""Files module uses the `file` command to get the mimetype of each file."""
from PIL import Image
@ -460,7 +463,7 @@ async def test_mime_type(test_async_client, test_session, make_files_structure,
@pytest.mark.asyncio
async def test_files_indexer(test_async_client, test_session, make_files_structure, test_directory):
async def test_files_indexer(async_client, test_session, make_files_structure, test_directory):
"""An Indexer is provided for each file based on it's mimetype or contents."""
source_files: List[str] = [
'a text file.txt',
@ -534,7 +537,7 @@ def test_split_file_name_words(name, expected):
@pytest.mark.asyncio
async def test_large_text_indexer(test_async_client, test_session, make_files_structure):
async def test_large_text_indexer(async_client, test_session, make_files_structure):
"""
Large files have their indexes truncated.
"""
@ -727,7 +730,7 @@ async def test_refresh_directories(test_session, test_directory, assert_director
@pytest.mark.asyncio
async def test_file_group_merge(test_async_client, test_session, test_directory, make_files_structure, tag_factory,
async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory,
video_bytes, srt_file3):
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
migrated."""
@ -767,7 +770,7 @@ async def test_file_group_merge(test_async_client, test_session, test_directory,
@pytest.mark.asyncio
async def test_move(test_async_client, test_session, test_directory, make_files_structure, video_bytes,
async def test_move(async_client, test_session, test_directory, make_files_structure, video_bytes,
singlefile_contents_factory):
"""files.lib.move behaves likes posix mv"""
make_files_structure({
@ -802,7 +805,7 @@ async def test_move(test_async_client, test_session, test_directory, make_files_
@pytest.mark.asyncio
async def test_move_files(test_async_client, test_session, test_directory, make_files_structure):
async def test_move_files(async_client, test_session, test_directory, make_files_structure):
"""Files can be moved using files.lib.move."""
one, two = make_files_structure({
'foo/one.txt': 'one',
@ -823,7 +826,7 @@ async def test_move_files(test_async_client, test_session, test_directory, make_
@pytest.mark.asyncio
async def test_move_deep_directory(test_async_client, test_session, test_directory, make_files_structure):
async def test_move_deep_directory(async_client, test_session, test_directory, make_files_structure):
"""Moving files/directories from one deep directory to another is supported."""
baz, foo, quuz_mp4, quuz_txt, quux = make_files_structure({
'foo/foo.txt': 'foo',
@ -868,7 +871,7 @@ async def test_move_deep_directory(test_async_client, test_session, test_directo
@pytest.mark.asyncio
async def test_move_directory(test_async_client, test_session, test_directory, make_files_structure,
async def test_move_directory(async_client, test_session, test_directory, make_files_structure,
assert_directories):
"""A Directory record is deleted when it's directory is deleted."""
make_files_structure({
@ -920,7 +923,7 @@ async def test_move_error(test_session, test_directory, make_files_structure, vi
@pytest.mark.asyncio
async def test_file_group_move(test_async_client, test_session, make_files_structure, test_directory, video_bytes,
async def test_file_group_move(async_client, test_session, make_files_structure, test_directory, video_bytes,
srt_text):
"""Test FileGroup's move method"""
video, srt = make_files_structure({
@ -943,7 +946,7 @@ async def test_file_group_move(test_async_client, test_session, make_files_struc
@pytest.mark.asyncio
async def test_move_tagged(test_async_client, test_session, test_directory, make_files_structure, tag_factory):
async def test_move_tagged(async_client, test_session, test_directory, make_files_structure, tag_factory):
"""A FileGroup's tag is preserved when moved or renamed."""
tag = await tag_factory()
foo, bar, = make_files_structure({
@ -987,7 +990,7 @@ async def test_move_tagged(test_async_client, test_session, test_directory, make
@pytest.mark.asyncio
async def test_html_index(test_async_client, test_session, test_directory, make_files_structure):
async def test_html_index(async_client, test_session, test_directory, make_files_structure):
make_files_structure({
'archive.html': '''<html>
<title>The Title</title>
@ -1033,7 +1036,7 @@ Outside element.'''
@pytest.mark.asyncio
async def test_doc_indexer(test_async_client, test_session, example_doc):
async def test_doc_indexer(async_client, test_session, example_doc):
"""The contents of a Microsoft doc files can be indexed."""
await lib.refresh_files()
assert test_session.query(FileGroup).count() == 1
@ -1044,7 +1047,7 @@ async def test_doc_indexer(test_async_client, test_session, example_doc):
@pytest.mark.asyncio
async def test_docx_indexer(test_async_client, test_session, example_docx):
async def test_docx_indexer(async_client, test_session, example_docx):
"""The contents of a Microsoft docx files can be indexed."""
await lib.refresh_files()
assert test_session.query(FileGroup).count() == 1
@ -1055,7 +1058,7 @@ async def test_docx_indexer(test_async_client, test_session, example_docx):
@pytest.mark.asyncio
async def test_file_search_date_range(test_async_client, test_session, example_pdf, example_doc):
async def test_file_search_date_range(async_client, test_session, example_pdf, example_doc):
"""Test searching FileGroups by their published datetime."""
await lib.refresh_files()
doc, pdf = test_session.query(FileGroup).order_by(FileGroup.primary_path).all()
@ -1104,19 +1107,19 @@ def test_replace_file(test_directory):
@pytest.mark.asyncio
async def test_upsert_file_video_with_channel(test_async_client, test_session, test_directory, video_bytes,
async def test_upsert_file_video_with_channel(async_client, test_session, test_directory, video_bytes,
channel_factory, video_factory):
"""A video file that is uploaded should be assigned to a Channel if it is in the Channel's directory."""
channel = channel_factory()
# Put the video file in the Channel's directory.
video_file = channel.directory / 'video.mp4'
video_file: pathlib.Path = channel.directory / 'video.mp4'
video_file.write_bytes(video_bytes)
# Put an info json file next to the video file.
info_json_file = video_file.with_suffix('.info.json')
info_json_file.write_text(json.dumps({'duration': 5}))
# Upsert the file, it should be modeled.
fg: FileGroup = await lib.upsert_file(video_file, test_session)
fg: FileGroup = await lib.upsert_file(video_file)
assert fg.model == 'video', 'Upserted file should have been modeled.'
assert fg.my_video_files(), 'Video file was upserted.'
assert fg.my_json_files(), 'Info json file should have been found near video file.'
@ -1129,7 +1132,7 @@ async def test_upsert_file_video_with_channel(test_async_client, test_session, t
@pytest.mark.asyncio
async def test_move_many_files(test_async_client, test_session, test_directory, make_files_structure):
async def test_move_many_files(async_client, test_session, test_directory, make_files_structure):
make_files_structure([f'foo/{i}.txt' for i in range(10_000)])
await lib.refresh_files()

2
wrolpi/files/test/test_models.py
View file

@ -2,7 +2,7 @@ import pytest
@pytest.mark.asyncio
async def test_file_group_move(test_async_client, test_session, test_directory, video_factory):
async def test_file_group_move(async_client, test_session, test_directory, video_factory):
"""Any new or renamed files are adopted during FileGroup.move."""
video = video_factory(title='video', with_poster_ext='.webp')
test_session.commit()

2
wrolpi/files/test/test_pdfs.py
View file

@ -9,7 +9,7 @@ from wrolpi.files.models import FileGroup
@pytest.mark.asyncio
async def test_pdf_title(test_async_client, test_session, example_pdf):
async def test_pdf_title(async_client, test_session, example_pdf):
"""PDF modeler extracts a PDFs title and contents."""
await files_lib.refresh_files()

8
wrolpi/status.py
View file

@ -13,7 +13,7 @@ from typing import List, Optional, Dict
import jc
from wrolpi.api_utils import api_app
from wrolpi.api_utils import api_app, perpetual_signal
from wrolpi.cmd import which
from wrolpi.common import logger, get_warn_once, unique_by_predicate
from wrolpi.dates import now
@ -475,7 +475,7 @@ async def get_disk_counters(shared_status):
return disk_bandwidth_stats
@api_app.signal('wrolpi.periodic.status')
@perpetual_signal(sleep=5)
async def status_worker(count: int = None, sleep_time: int = 5):
"""A background process which will gather historical data about system statistics."""
shared_status = api_app.shared_ctx.status
@ -528,14 +528,10 @@ async def status_worker(count: int = None, sleep_time: int = 5):
if load_stats and load_stats.minute_1 and load_stats.minute_1 > multiprocessing.cpu_count():
# System is stressed, slow down status updates.
await asyncio.sleep((sleep_time * load_stats.minute_1) / multiprocessing.cpu_count())
else:
await asyncio.sleep(sleep_time)
if count:
# Running in testing, or __main__.
return await status_worker(count - 1, sleep_time)
else:
await api_app.dispatch('wrolpi.periodic.status', context={'sleep_time': sleep_time})
if __name__ == '__main__':

137
wrolpi/switches.py Normal file
View file

@ -0,0 +1,137 @@
import asyncio
import inspect
import multiprocessing
from functools import partial
from typing import Dict, Mapping, Protocol
from wrolpi.api_utils import api_app, logger, perpetual_signal
from wrolpi.vars import PYTEST
SWITCH_HANDLERS: Dict[str, callable] = dict()
def activate_switch(switch_name: str, context: dict = None):
"""
Activate a named switch. Replaces the context of the previous call if it has not yet been handled.
"""
if switch_name not in SWITCH_HANDLERS:
raise RuntimeError(f'Cannot activate switch {switch_name}, it has not been registered.')
switches_lock: multiprocessing.Lock = api_app.shared_ctx.switches_lock
if switches_lock.acquire(timeout=5):
try:
try:
switches = api_app.shared_ctx.switches
except AttributeError:
raise RuntimeError(
f'Sanic shared_ctx has not been initialized. If testing, use `async_client` fixture.')
switches_changed: multiprocessing.Event = api_app.shared_ctx.switches_changed
context = context or dict()
if not isinstance(context, Mapping):
raise RuntimeError('Switch context must be a dict (for kwargs)')
switches_changed.set()
switches.update({**switches.copy(), switch_name: context})
logger.debug(f'activate_switch: {switch_name}')
finally:
switches_lock.release()
else:
raise RuntimeError(f'Failed to acquire switches_lock lock to set switch {switch_name}')
class ActivateSwitchMethod(Protocol):
"""Adds type hinting for @register_switch_handler use."""
def activate_switch(self, context: dict = None) -> None:
...
def register_switch_handler(switch_name: str):
"""Register a handler for a switch. The switch can be activated by name using `active_switch`, or by the
`activate_switch` method attached to the wrapped function.
>>> def func():
>>> pass
>>> @register_switch_handler('unique_name')
>>> async def func_handler(**context):
>>> # handle the switch
>>>
>>> func_handler.activate_switch()
"""
def wrapper(handler: callable):
if switch_name in SWITCH_HANDLERS and not PYTEST:
raise RuntimeError(f'register_switch_handler: switch name already taken {switch_name}')
# Add `handler` function to global dict, this will be called by `switch_worker`.
SWITCH_HANDLERS[switch_name] = handler
setattr(handler, 'activate_switch', partial(activate_switch, switch_name))
return handler
return wrapper
DEBUG_LOGGED = False
@perpetual_signal(sleep=0.1)
async def switch_worker():
"""Watches `api_app.shared_ctx.switches` for new activated switches, handles each one at a time."""
global DEBUG_LOGGED
if not DEBUG_LOGGED:
logger.debug('switch_worker started')
DEBUG_LOGGED = True
# Wait for the switches to be changed.
switches_changed: multiprocessing.Event = api_app.shared_ctx.switches_changed
switch_name = None
try:
# Get new switch and its context. Handle each switch one at a time.
switches_changed.wait(timeout=1)
switches: dict = api_app.shared_ctx.switches
with api_app.shared_ctx.switches_lock:
switches_keys = switches.keys()
switch_name, context = switches.popitem()
# Call handler with the stored context, await coroutine, if any.
handler = SWITCH_HANDLERS[switch_name]
logger.debug(f'switch_worker handling {switch_name} of {len(switches_keys)}')
coro = handler(**context)
if inspect.iscoroutine(coro):
# Handler is async.
await coro
except TimeoutError:
# `switches_changed` is not set.
pass
except StopIteration:
# No switches to handle.
pass
except KeyError:
# `switches` is empty, or `switch_name` not in `SWITCH_HANDLER`
if switch_name:
logger.critical(f'No switch handler defined for: {switch_name}')
finally:
# Clear only after task is complete to avoid
switches_changed.clear()
async def await_switches(timeout: int = 10):
if not PYTEST:
raise RuntimeError('This function is only for testing purposes')
count = 0
while count < (timeout * 10):
count += 1
if api_app.shared_ctx.switches:
await asyncio.sleep(0.1)
continue
if api_app.shared_ctx.switches_changed.is_set():
await asyncio.sleep(0.1)
continue
# All switches handled.
break
else:
raise RuntimeError('Timed out waiting for switches')

64
wrolpi/tags.py
View file

@ -8,11 +8,12 @@ from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import relationship, Session
from wrolpi import dates, flags
from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, run_after, \
from wrolpi.common import ModelHelper, Base, logger, ConfigFile, get_media_directory, background_task, \
get_relative_to_media_directory, is_valid_hex_color, walk, INVALID_FILE_CHARS
from wrolpi.dates import TZDateTime
from wrolpi.db import optional_session, get_db_curs
from wrolpi.db import optional_session, get_db_curs, get_db_session
from wrolpi.errors import UnknownTag, UsedTag, InvalidTag, FileGroupAlreadyTagged, RefreshConflict, NoPrimaryFile
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from wrolpi.vars import PYTEST
logger = logger.getChild(__name__)
@ -98,8 +99,8 @@ class Tag(ModelHelper, Base):
logger.info(f'Tagged FileGroup {file_group_id} with Tag {tag_id_or_name}')
# Save changes to config.
save_tags_config(session)
sync_tags_directory(session)
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
return tag_file
@staticmethod
@ -115,8 +116,8 @@ class Tag(ModelHelper, Base):
logger.debug(f'Deleted TagFile: {tag_file}')
# Save changes to config.
save_tags_config(session)
sync_tags_directory(session)
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
else:
logger.warning(f'Could not find TagFile for FileGroup.id={file_group_id}/Tag.id={tag_id_or_name}')
@ -213,8 +214,8 @@ class Tag(ModelHelper, Base):
session.delete(self)
session.commit()
save_tags_config(session)
sync_tags_directory(session)
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
class TagsConfig(ConfigFile):
@ -253,7 +254,7 @@ class TagsConfig(ConfigFile):
def tags(self, value: dict):
self.update({'tags': value})
def save_tags(self, session: Session, ignore_lock: bool = False):
def save_tags(self, session: Session):
media_directory = get_media_directory()
tags = dict()
@ -301,7 +302,7 @@ class TagsConfig(ConfigFile):
'tag_files': tag_files,
'tag_zims': tag_zims,
'tags': tags,
}, ignore_lock=ignore_lock)
})
TAGS_CONFIG: TagsConfig = TagsConfig()
@ -325,16 +326,15 @@ def test_tags_config():
TEST_TAGS_CONFIG = None
@optional_session
def save_tags_config(session: Session = None):
@register_switch_handler('save_tags_config')
def save_tags_config():
"""Schedule a background task to save all TagFiles to the config file. If testing, save synchronously."""
if PYTEST:
with get_db_session() as session:
get_tags_config().save_tags(session)
else:
async def _():
get_tags_config().save_tags(session)
logger.info('save_tags_config complete')
background_task(_())
save_tags_config: ActivateSwitchMethod
def _sync_tags_directory_tag_files(tags_directory: pathlib.Path, session: Session) -> List[pathlib.Path]:
@ -408,20 +408,21 @@ be AUTOMATICALLY DELETED!
''')
@optional_session
def sync_tags_directory(session: Session = None):
@register_switch_handler('sync_tags_directory')
def sync_tags_directory():
"""Synchronizes database Tags with the Tags directory (typically /media/wrolpi/tags). Removes any
files that do not belong in the Tags Directory."""
try:
tags_directory = get_tags_directory()
create_tags_directory(tags_directory)
with get_db_session() as session:
try:
tags_directory = get_tags_directory()
create_tags_directory(tags_directory)
links = _sync_tags_directory_tag_files(tags_directory, session)
logger.debug(f'Tags Directory should contain {len(links)} links')
_delete_extra_tags_directory_paths(tags_directory, links)
except Exception as e:
logger.error('Failed to sync DB with tags directory in media directory!', exc_info=e)
raise
links = _sync_tags_directory_tag_files(tags_directory, session)
logger.debug(f'Tags Directory should contain {len(links)} links')
_delete_extra_tags_directory_paths(tags_directory, links)
except Exception as e:
logger.error('Failed to sync DB with tags directory in media directory!', exc_info=e)
raise
def get_tags() -> List[dict]:
@ -439,7 +440,6 @@ def get_tags() -> List[dict]:
@optional_session
@run_after(save_tags_config)
async def upsert_tag(name: str, color: str, tag_id: int = None, session: Session = None) -> Tag:
if tag_id:
tag = Tag.find_by_id(tag_id, session)
@ -458,8 +458,8 @@ async def upsert_tag(name: str, color: str, tag_id: int = None, session: Session
session.rollback()
raise InvalidTag(f'Name already taken') from e
save_tags_config(session)
sync_tags_directory(session)
save_tags_config.activate_switch()
sync_tags_directory.activate_switch()
return tag
@ -614,7 +614,7 @@ def import_tags_config(session: Session = None):
if PYTEST:
raise
sync_tags_directory()
sync_tags_directory.activate_switch()
def tag_names_to_file_group_sub_select(tag_names: List[str], params: dict) -> Tuple[str, dict]:

4
wrolpi/test/test_admin.py
View file

@ -69,7 +69,7 @@ def test_enable_hotspot_connected():
)
def test_change_hotspot(test_config, test_async_client):
def test_change_hotspot(test_config, async_client):
"""The hotspot can be configured."""
with mock.patch('wrolpi.admin.hotspot_status') as mock_hotspot_status, \
mock.patch('wrolpi.admin.subprocess') as mock_subprocess, \
@ -113,7 +113,7 @@ def test_throttle_off():
@skip_circleci
def test_hotspot_device(test_async_client):
def test_hotspot_device(async_client):
"""Changing the hotspot device changes what device is turned on."""
from wrolpi.common import WROLPI_CONFIG
WROLPI_CONFIG.hotspot_device = 'wlp2s0'

10
wrolpi/test/test_common.py
View file

@ -814,7 +814,7 @@ def test_find_file(test_directory, make_files_structure):
assert common.find_file(test_directory, 'does not exist', 100) is None
def test_config_backup(test_async_client, test_config, test_directory):
def test_config_backup(async_client, test_config, test_directory):
"""Configs have a backup each day."""
config = common.get_wrolpi_config()
@ -860,7 +860,7 @@ def test_can_connect_to_server(simple_web_server):
@pytest.mark.asyncio
async def test_log_level(test_async_client, test_config):
async def test_log_level(async_client, test_config):
"""User can change API log level."""
from wrolpi.api_utils import api_app
try:
@ -869,19 +869,19 @@ async def test_log_level(test_async_client, test_config):
# Change log level to WARNING.
body = dict(log_level=30)
request, response = await test_async_client.patch('/api/settings', json=body)
request, response = await async_client.patch('/api/settings', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
assert api_app.shared_ctx.log_level.value == 30
# Change log level to NOTSET.
body = dict(log_level=0)
request, response = await test_async_client.patch('/api/settings', json=body)
request, response = await async_client.patch('/api/settings', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
assert api_app.shared_ctx.log_level.value == 0
finally:
# Reset log level to debug.
body = dict(log_level=10)
request, response = await test_async_client.patch('/api/settings', json=body)
request, response = await async_client.patch('/api/settings', json=body)
assert response.status_code == HTTPStatus.NO_CONTENT, response.json
assert api_app.shared_ctx.log_level.value == 10

29
wrolpi/test/test_downloader.py
View file

@ -17,6 +17,7 @@ from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \
RSSDownloader, get_download_manager_config
from wrolpi.errors import InvalidDownload, WROLModeEnabled
from wrolpi.switches import await_switches
from wrolpi.test.common import assert_dict_contains
@ -61,18 +62,6 @@ async def test_delete_old_once_downloads(test_session, test_download_manager, te
assert_dict_contains(download.dict(), expected)
@pytest.mark.asyncio
async def test_create_downloads(test_async_client, test_session, test_download_manager, test_downloader,
assert_download_urls):
"""Multiple downloads can be scheduled using DownloadManager.create_downloads."""
mock_save_downloads_config = AsyncMock()
# Both URLs are scheduled.
with mock.patch('wrolpi.downloader.save_downloads_config', mock_save_downloads_config):
test_download_manager.create_downloads(['https://example.com/1', 'https://example.com/2'], test_downloader.name)
assert_download_urls({'https://example.com/1', 'https://example.com/2'})
mock_save_downloads_config.assert_called_once()
@pytest.mark.asyncio
async def test_recreate_download(test_session, test_download_manager, test_downloader, tag_factory):
"""Settings are preserved when a Download is restarted."""
@ -284,7 +273,7 @@ async def test_skip_urls(test_session, test_download_manager, assert_download_ur
@pytest.mark.asyncio
async def test_process_runner_timeout(test_async_client, test_session, test_directory):
async def test_process_runner_timeout(async_client, test_session, test_directory):
"""A Downloader can cancel its download using a timeout."""
# Default timeout of 3 seconds.
downloader = Downloader('downloader', timeout=3)
@ -310,7 +299,7 @@ async def test_process_runner_timeout(test_async_client, test_session, test_dire
@pytest.mark.asyncio
async def test_crud_download(test_async_client, test_session, test_download_manager, test_downloader):
async def test_crud_download(async_client, test_session, test_download_manager, test_downloader):
"""Test the ways that Downloads can be created."""
body = dict(
urls=['https://example.com', ],
@ -318,7 +307,7 @@ async def test_crud_download(test_async_client, test_session, test_download_mana
frequency=DownloadFrequency.weekly,
settings=dict(excluded_urls='example.org,something'),
)
request, response = await test_async_client.post('/api/download', content=json.dumps(body))
request, response = await async_client.post('/api/download', content=json.dumps(body))
assert response.status_code == HTTPStatus.NO_CONTENT, response.body
download: Download = test_session.query(Download).one()
@ -328,20 +317,20 @@ async def test_crud_download(test_async_client, test_session, test_download_mana
assert download.frequency == DownloadFrequency.weekly
assert download.settings['excluded_urls'] == ['example.org', 'something']
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
assert response.status_code == HTTPStatus.OK
assert 'recurring_downloads' in response.json
assert [i['url'] for i in response.json['recurring_downloads']] == ['https://example.com']
request, response = await test_async_client.delete('/api/download/123')
request, response = await async_client.delete('/api/download/123')
assert response.status_code == HTTPStatus.NOT_FOUND
assert test_session.query(Download).count() == 1
request, response = await test_async_client.delete(f'/api/download/{download.id}')
request, response = await async_client.delete(f'/api/download/{download.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 0
request, response = await test_async_client.delete(f'/api/download/{download.id}')
request, response = await async_client.delete(f'/api/download/{download.id}')
assert response.status_code == HTTPStatus.NOT_FOUND
assert test_session.query(Download).count() == 0
@ -371,7 +360,7 @@ async def test_downloads_config(test_session, test_download_manager, test_downlo
test_session.commit()
# Allow background tasks to run.
await asyncio.sleep(0)
await await_switches()
# Downloads were saved to config.
assert get_download_manager_config().downloads, 'No downloads were saved'

10
wrolpi/test/test_events.py
View file

@ -18,13 +18,13 @@ def assert_events(expected: List[Dict], after=None):
@pytest.mark.asyncio
async def test_events_api_feed(test_session, test_async_client, example_pdf):
async def test_events_api_feed(test_session, async_client, example_pdf):
"""Events can be gotten from the API."""
# refresh_files() creates a few events.
request, response = await test_async_client.post('/api/files/refresh')
request, response = await async_client.post('/api/files/refresh')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await test_async_client.get('/api/events/feed')
request, response = await async_client.get('/api/events/feed')
assert response.status_code == HTTPStatus.OK
assert (result := response.json.get('events'))
@ -41,7 +41,7 @@ async def test_events_api_feed(test_session, test_async_client, example_pdf):
@pytest.mark.asyncio
async def test_events_history_limit(test_async_client):
async def test_events_history_limit(async_client):
"""EVENTS_HISTORY has a limited size."""
for i in range(HISTORY_SIZE + 5):
Events.send_ready()
@ -50,7 +50,7 @@ async def test_events_history_limit(test_async_client):
@pytest.mark.asyncio
async def test_get_events(test_async_client):
async def test_get_events(async_client):
after = now()
Events.send_global_refresh_started()

6
wrolpi/test/test_flags.py
View file

@ -14,7 +14,7 @@ def assert_db(session: Session, name: str, expected: bool):
assert getattr(f, name) is expected
def test_flags(test_async_client, test_session, flags_lock):
def test_flags(async_client, test_session, flags_lock):
"""DB flags can be saved."""
assert flags.refresh_complete.is_set() is False
assert not any(v for k, v in flags.get_flags().items())
@ -28,7 +28,7 @@ def test_flags(test_async_client, test_session, flags_lock):
assert not any(v for k, v in flags.get_flags().items())
def test_flags_with(test_async_client, flags_lock):
def test_flags_with(async_client, flags_lock):
"""A Flag can be used with `with` context."""
assert flags.refreshing.is_set() is False, 'Flag should not be set by default'
assert not any(v for k, v in flags.get_flags().items())
@ -48,7 +48,7 @@ def test_flags_with(test_async_client, flags_lock):
@pytest.mark.asyncio
async def test_flag_wait_for(test_async_client, flags_lock):
async def test_flag_wait_for(async_client, flags_lock):
"""Wait for throws an error when waiting exceeds timeout."""
with pytest.raises(TimeoutError):
async with flags.refreshing.wait_for(timeout=1):

149
wrolpi/test/test_root_api.py
View file

@ -5,91 +5,93 @@ from itertools import zip_longest
import pytest
from mock import mock
from sanic import Request
from selenium.webdriver.common.devtools.v112.runtime import await_promise
from wrolpi.admin import HotspotStatus
from wrolpi.api_utils import json_error_handler
from wrolpi.common import get_wrolpi_config
from wrolpi.downloader import Download, get_download_manager_config
from wrolpi.errors import ValidationError, SearchEmpty
from wrolpi.switches import await_switches
from wrolpi.test.common import skip_circleci, assert_dict_contains
@pytest.mark.asyncio
async def test_index(test_session, test_async_client):
async def test_index(test_session, async_client):
"""
Index should have some details in an HTML response
"""
request, response = await test_async_client.get('/')
request, response = await async_client.get('/')
assert response.status_code == HTTPStatus.OK
assert b'html' in response.body
request, response = await test_async_client.get('/api')
request, response = await async_client.get('/api')
assert response.status_code == HTTPStatus.OK
assert b'html' in response.body
@pytest.mark.asyncio
async def test_valid_regex(test_session, test_async_client):
async def test_valid_regex(test_session, async_client):
"""
The endpoint should only return valid if the regex is valid.
"""
data = {'regex': 'foo'}
request, response = await test_async_client.post('/api/valid_regex', content=json.dumps(data))
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert json.loads(response.body) == {'valid': True, 'regex': 'foo'}
data = {'regex': '.*(title match).*'}
request, response = await test_async_client.post('/api/valid_regex', content=json.dumps(data))
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert json.loads(response.body) == {'valid': True, 'regex': '.*(title match).*'}
data = {'regex': '.*(missing parenthesis.*'}
request, response = await test_async_client.post('/api/valid_regex', content=json.dumps(data))
request, response = await async_client.post('/api/valid_regex', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert json.loads(response.body) == {'valid': False, 'regex': '.*(missing parenthesis.*'}
@pytest.mark.asyncio
async def test_get_settings(test_session, test_async_client):
async def test_get_settings(test_session, async_client):
get_wrolpi_config().ignored_directories = list()
with mock.patch('wrolpi.admin.CPUFREQ_INFO_BIN', '/usr/bin/cpufreq-info'), \
mock.patch('wrolpi.admin.NMCLI_BIN', '/usr/bin/nmcli'):
request, response = await test_async_client.get('/api/settings')
request, response = await async_client.get('/api/settings')
assert response.status_code == HTTPStatus.OK
@pytest.mark.asyncio
async def test_log_level(test_async_client):
async def test_log_level(async_client):
"""Log level must be between 0 and 40."""
data = {'log_level': 0}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
data = {'log_level': -1}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'log_level': 40}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
data = {'log_level': 41}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'log_level': None}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_get_downloads(test_session, test_async_client):
request, response = await test_async_client.get('/api/download')
async def test_get_downloads(test_session, async_client):
request, response = await async_client.get('/api/download')
assert_dict_contains(response.json, {'once_downloads': [], 'recurring_downloads': []})
d1 = Download(url='https://example.com/1', status='complete')
@ -102,7 +104,7 @@ async def test_get_downloads(test_session, test_async_client):
test_session.add_all([d1, d2, d3, d4, d5, d6, d7])
test_session.commit()
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
expected = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/5', 'status': 'new'},
@ -121,7 +123,7 @@ async def test_get_downloads(test_session, test_async_client):
@pytest.mark.asyncio
async def test_downloads_sorter_recurring(test_async_client, test_session):
async def test_downloads_sorter_recurring(async_client, test_session):
downloads = [
dict(status='complete', frequency=1),
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:01+00:00'),
@ -144,14 +146,14 @@ async def test_downloads_sorter_recurring(test_async_client, test_session):
dict(status='complete', frequency=2, next_download='2020-01-01T00:00:02+00:00'),
dict(status='complete', frequency=1),
]
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
recurring_downloads = response.json['recurring_downloads']
for d1, d2 in zip_longest(expected, recurring_downloads):
assert_dict_contains(d2, d1)
@pytest.mark.asyncio
async def test_downloads_sorter(test_async_client, test_session):
async def test_downloads_sorter(async_client, test_session):
downloads = [
dict(status='complete', last_successful_download='2020-01-01T00:00:01+00:00'),
dict(status='complete', last_successful_download='2020-01-01T00:00:03+00:00'),
@ -178,18 +180,18 @@ async def test_downloads_sorter(test_async_client, test_session):
dict(status='complete', last_successful_download='2020-01-01T00:00:02+00:00'),
dict(status='complete', last_successful_download='2020-01-01T00:00:01+00:00'),
]
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
once_downloads = response.json['once_downloads']
for d1, d2 in zip_longest(expected, once_downloads):
assert_dict_contains(d2, d1)
@pytest.mark.asyncio
async def test_echo(test_async_client):
async def test_echo(async_client):
"""
Echo should send back a JSON object with our request details.
"""
request, response = await test_async_client.get('/api/echo?hello=world')
request, response = await async_client.get('/api/echo?hello=world')
assert response.status_code == HTTPStatus.OK
assert_dict_contains(response.json['form'], {})
assert response.json['method'] == 'GET', 'Method was not GET'
@ -197,7 +199,7 @@ async def test_echo(test_async_client):
assert response.json['args'] == {'hello': ['world']}
data = {'foo': 'bar'}
request, response = await test_async_client.post('/api/echo', content=json.dumps(data))
request, response = await async_client.post('/api/echo', content=json.dumps(data))
assert response.status_code == HTTPStatus.OK
assert_dict_contains(response.json['form'], {})
assert response.json['method'] == 'POST', 'Method was not POST'
@ -285,7 +287,7 @@ def test_throttle_toggle(test_session, test_client, test_config):
@pytest.mark.asyncio
async def test_clear_downloads(test_session, test_async_client, test_config, test_download_manager_config):
async def test_clear_downloads(test_session, async_client, test_config, test_download_manager_config):
d1 = Download(url='https://example.com/1', status='complete')
d2 = Download(url='https://example.com/2', status='pending')
d3 = Download(url='https://example.com/3', status='deferred')
@ -312,7 +314,7 @@ async def test_clear_downloads(test_session, test_async_client, test_config, tes
assert download['status'] == recurring_download['status']
# All created downloads are present.
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
once_downloads = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/5', 'status': 'failed'},
@ -327,9 +329,9 @@ async def test_clear_downloads(test_session, test_async_client, test_config, tes
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
# Once "complete" download is removed.
request, response = await test_async_client.post('/api/download/clear_completed')
request, response = await async_client.post('/api/download/clear_completed')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
once_downloads = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/5', 'status': 'failed'},
@ -339,9 +341,9 @@ async def test_clear_downloads(test_session, test_async_client, test_config, tes
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
# Failed "once" download is removed, recurring failed is not removed.
request, response = await test_async_client.post('/api/download/clear_failed')
request, response = await async_client.post('/api/download/clear_failed')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
once_downloads = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/4', 'status': 'new'},
@ -353,9 +355,9 @@ async def test_clear_downloads(test_session, test_async_client, test_config, tes
assert get_download_manager_config().skip_urls == ['https://example.com/5', ]
# Downloads can be retried.
request, response = await test_async_client.post('/api/download/retry_once')
request, response = await async_client.post('/api/download/retry_once')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
once_downloads = [
{'url': 'https://example.com/2', 'status': 'new'},
{'url': 'https://example.com/3', 'status': 'new'},
@ -364,9 +366,9 @@ async def test_clear_downloads(test_session, test_async_client, test_config, tes
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
# "Delete All" button deletes all once-downloads.
request, response = await test_async_client.post('/api/download/delete_once')
request, response = await async_client.post('/api/download/delete_once')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = await test_async_client.get('/api/download')
request, response = await async_client.get('/api/download')
check_downloads(response, [], recurring_downloads, status_code=HTTPStatus.OK)
# Failed once-downloads will not be downloaded again.
@ -374,9 +376,9 @@ async def test_clear_downloads(test_session, test_async_client, test_config, tes
@pytest.mark.asyncio
async def test_get_status(test_async_client, test_session):
async def test_get_status(async_client, test_session):
"""Get the server status information."""
request, response = await test_async_client.get('/api/status')
request, response = await async_client.get('/api/status')
assert response.status_code == HTTPStatus.OK
assert isinstance(response.json.get('cpu_stats'), dict), 'cpu_stats should be a dict'
assert isinstance(response.json.get('load_stats'), dict), 'load_stats should be a dict'
@ -390,7 +392,7 @@ async def test_get_status(test_async_client, test_session):
@pytest.mark.asyncio
async def test_download_crud(test_session, test_async_client, test_download_manager_config, tag_factory):
async def test_download_crud(test_session, async_client, test_download_manager_config, tag_factory):
"""Test creating once-downloads and recurring downloads."""
tag1, tag2 = await tag_factory(), await tag_factory()
@ -404,7 +406,7 @@ async def test_download_crud(test_session, test_async_client, test_download_mana
frequency=1_000,
downloader='archive',
)
request, response = await test_async_client.post('/api/download', content=json.dumps(content))
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.NO_CONTENT
assert {i.url for i in test_session.query(Download)} == {'https://example.com/1', }
@ -414,7 +416,7 @@ async def test_download_crud(test_session, test_async_client, test_download_mana
urls=['https://example.com/2', 'https://example.com/3'],
downloader='archive',
)
request, response = await test_async_client.post('/api/download', json=content)
request, response = await async_client.post('/api/download', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
assert {i.url for i in test_session.query(Download)} == {f'https://example.com/{i}' for i in range(1, 4)}
@ -430,7 +432,7 @@ async def test_download_crud(test_session, test_async_client, test_download_mana
settings={'tag_names': [tag1.name, tag2.name]},
)
expected_download1.update({'url': 'https://example.com/1', 'settings': {'tag_names': [tag1.name, tag2.name]}})
request, response = await test_async_client.put(f'/api/download/{download1.id}', json=content)
request, response = await async_client.put(f'/api/download/{download1.id}', json=content)
assert response.status_code == HTTPStatus.NO_CONTENT
test_session.flush([download2])
download2 = test_session.query(Download).filter_by(id=expected_download1['id']).first()
@ -447,39 +449,40 @@ def test_get_downloaders(test_client):
@pytest.mark.asyncio
async def test_empty_post_download(test_async_client):
async def test_empty_post_download(async_client):
"""Cannot have empty download URLs"""
content = dict(
urls=[''], # Cannot have empty URLs
frequency=1_000,
downloader='archive',
)
request, response = await test_async_client.post('/api/download', content=json.dumps(content))
request, response = await async_client.post('/api/download', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json['code'] == 'INVALID_DOWNLOAD'
@pytest.mark.asyncio
async def test_restart_download(test_session, test_async_client, test_download_manager, test_downloader):
async def test_restart_download(test_session, async_client, test_download_manager, test_downloader):
"""A Download can be restarted."""
# Create a download, fail it, it should be restarted.
download = test_download_manager.create_download('https://example.com', test_downloader.name)
download.fail()
test_session.commit()
await await_switches()
assert test_session.query(Download).one().is_failed
test_downloader.set_test_failure()
# Download is now "new" again.
request, response = await test_async_client.post(f'/api/download/{download.id}/restart')
request, response = await async_client.post(f'/api/download/{download.id}/restart')
assert response.status_code == HTTPStatus.NO_CONTENT
download = test_session.query(Download).one()
assert download.is_new, download.status_code
assert download.is_new, download.status
# Wait for the background download to fail. It should be deferred.
await test_download_manager.wait_for_all_downloads()
download = test_session.query(Download).one()
assert download.is_deferred, download.status_code
assert download.is_deferred, download.status
def test_get_global_statistics(test_session, test_client):
@ -488,17 +491,17 @@ def test_get_global_statistics(test_session, test_client):
@pytest.mark.asyncio
async def test_post_vin_number_decoder(test_async_client):
async def test_post_vin_number_decoder(async_client):
"""A VIN number can be decoded."""
# Invalid VIN number.
content = dict(vin_number='5N1NJ01CXST00000')
request, response = await test_async_client.post('/api/vin_number_decoder', content=json.dumps(content))
request, response = await async_client.post('/api/vin_number_decoder', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert response.json == dict()
# Test VIN from vininfo.
content = dict(vin_number='5N1NJ01CXST000001')
request, response = await test_async_client.post('/api/vin_number_decoder', content=json.dumps(content))
request, response = await async_client.post('/api/vin_number_decoder', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert response.json['vin'] == {'body': 'Sedan, 4-Door, Standard Body Truck',
'country': 'United States',
@ -513,7 +516,7 @@ async def test_post_vin_number_decoder(test_async_client):
@pytest.mark.asyncio
async def test_search_suggestions(test_session, test_async_client, channel_factory, archive_factory, video_factory):
async def test_search_suggestions(test_session, async_client, channel_factory, archive_factory, video_factory):
# WARNING results are cached, this test uses unique queries to avoid conflicts.
channel_factory(name='Foo')
channel_factory(name='Fool')
@ -527,7 +530,7 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
expected_channels = expected_channels or []
expected_domains = expected_domains or []
request, response = await test_async_client.post('/api/search_suggestions', json=body)
request, response = await async_client.post('/api/search_suggestions', json=body)
assert response.status_code == HTTPStatus.OK
if expected_channels:
for channel, expected_channel in zip_longest(response.json['channels'], expected_channels):
@ -561,10 +564,10 @@ async def test_search_suggestions(test_session, test_async_client, channel_facto
@pytest.mark.asyncio
async def test_search_file_estimates(test_session, test_async_client, archive_factory, tag_factory, video_factory):
async def test_search_file_estimates(test_session, async_client, archive_factory, tag_factory, video_factory):
"""Can quickly get count of possible files when searching."""
body_ = dict(search_str='can search with no files')
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
request_, response_ = await async_client.post('/api/search_file_estimates', json=body_)
assert response_.status_code == HTTPStatus.OK
tag1, tag2, tag3 = await tag_factory(), await tag_factory(), await tag_factory()
@ -576,7 +579,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
async def assert_results(body: dict, expected_file_groups=None):
expected_file_groups = expected_file_groups or 0
request, response = await test_async_client.post('/api/search_file_estimates', json=body)
request, response = await async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == expected_file_groups or 0
@ -626,7 +629,7 @@ async def test_search_file_estimates(test_session, test_async_client, archive_fa
@pytest.mark.asyncio
async def test_search_file_estimates_any_tag(test_session, test_async_client, archive_factory, tag_factory,
async def test_search_file_estimates_any_tag(test_session, async_client, archive_factory, tag_factory,
video_factory):
"""File estimates can be filtered by any_tag."""
tag1 = await tag_factory()
@ -637,7 +640,7 @@ async def test_search_file_estimates_any_tag(test_session, test_async_client, ar
async def assert_results(body: dict, expected_file_groups=None):
expected_file_groups = expected_file_groups or 0
request, response = await test_async_client.post('/api/search_file_estimates', json=body)
request, response = await async_client.post('/api/search_file_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['file_groups'] == expected_file_groups or 0
@ -647,19 +650,19 @@ async def test_search_file_estimates_any_tag(test_session, test_async_client, ar
await assert_results(dict(search_str='bar', any_tag=True), 0)
body_ = dict(search_str='bunny', tag_names=[tag1.name, ], any_tag=True)
request_, response_ = await test_async_client.post('/api/search_file_estimates', json=body_)
request_, response_ = await async_client.post('/api/search_file_estimates', json=body_)
assert response_.status_code == HTTPStatus.BAD_REQUEST
@pytest.mark.asyncio
async def test_external_error(test_async_client):
async def test_external_error(async_client):
"""A non-SanicException error is handled as an Internal Server Error."""
@test_async_client.sanic_app.get('/error')
@async_client.sanic_app.get('/error')
async def error(_: Request):
raise RuntimeError('Oh no!')
request, response = await test_async_client.get('/error')
request, response = await async_client.get('/error')
assert response.status_code == HTTPStatus.INTERNAL_SERVER_ERROR
assert response.json == {'code': 'RuntimeError', 'error': 'Oh no!', 'message': None}
@ -704,9 +707,9 @@ def test_recursive_errors():
@pytest.mark.asyncio
async def test_settings_special_directories(test_async_client, test_config):
async def test_settings_special_directories(async_client, test_config):
"""Maintainer can change special directories."""
request, response = await test_async_client.get('/api/settings')
request, response = await async_client.get('/api/settings')
assert response.status_code == HTTPStatus.OK
assert response.json['archive_destination'] == 'archive/%(domain)s'
assert response.json['map_destination'] == 'map'
@ -714,36 +717,36 @@ async def test_settings_special_directories(test_async_client, test_config):
assert response.json['zims_destination'] == 'zims'
data = {'archive_destination': 'archives'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
assert get_wrolpi_config().archive_destination == 'archives'
data = {'archive_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'videos_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'map_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
data = {'zims_destination': '/absolute/not/allowed'}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.BAD_REQUEST
# Empty directory restores default.
data = {'archive_destination': ''}
request, response = await test_async_client.patch('/api/settings', content=json.dumps(data))
request, response = await async_client.patch('/api/settings', content=json.dumps(data))
assert response.status_code == HTTPStatus.NO_CONTENT
assert get_wrolpi_config().archive_destination == 'archive/%(domain)s'
@pytest.mark.asyncio
async def test_search_other_estimates(test_async_client, test_session, channel_factory, tag_factory):
async def test_search_other_estimates(async_client, test_session, channel_factory, tag_factory):
# Can search without Tags or Channels.
body = dict(tag_names=[])
request, response = await test_async_client.post('/api/search_other_estimates', json=body)
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == 0, 'Only one Channel is tagged.'
@ -754,6 +757,6 @@ async def test_search_other_estimates(test_async_client, test_session, channel_f
# One Channel is tagged.
body = dict(tag_names=[tag.name, ])
request, response = await test_async_client.post('/api/search_other_estimates', json=body)
request, response = await async_client.post('/api/search_other_estimates', json=body)
assert response.status_code == HTTPStatus.OK
assert response.json['others']['channel_count'] == 1, 'Only one Channel is tagged.'

10
wrolpi/test/test_status.py
View file

@ -7,7 +7,7 @@ from wrolpi import status
@pytest.mark.asyncio
async def test_get_load(test_async_client):
async def test_get_load(async_client):
"""Test reading load information from /proc/loadavg."""
load = await status.get_load_stats()
assert isinstance(load, status.SystemLoad)
@ -18,7 +18,7 @@ async def test_get_load(test_async_client):
@pytest.mark.asyncio
async def test_get_cpu_stats(test_async_client):
async def test_get_cpu_stats(async_client):
"""Minimum CPU info testing because this will fail in docker, etc."""
info = await status.get_cpu_stats()
assert isinstance(info, status.CPUInfo)
@ -27,7 +27,7 @@ async def test_get_cpu_stats(test_async_client):
@pytest.mark.asyncio
async def test_get_drives_stats(test_async_client):
async def test_get_drives_stats(async_client):
"""Minimum drives info testing because this will fail in docker, etc."""
info = await status.get_drives_stats()
assert isinstance(info, list)
@ -44,10 +44,10 @@ async def test_get_drives_stats(test_async_client):
@pytest.mark.asyncio
async def test_status_worker(test_async_client):
async def test_status_worker(async_client):
"""Status worker calls itself perpetually, but can be limited during testing."""
from wrolpi.api_utils import api_app
assert test_async_client.sanic_app.shared_ctx.status['cpu_stats'] == dict()
assert async_client.sanic_app.shared_ctx.status['cpu_stats'] == dict()
# Status worker fills out stats as it goes along.
await status.status_worker(count=2)

77
wrolpi/test/test_switches.py Normal file
View file

@ -0,0 +1,77 @@
import asyncio
import multiprocessing
import pytest
from wrolpi import switches
from wrolpi.switches import register_switch_handler
count1 = multiprocessing.Value('i', 0)
count2 = multiprocessing.Value('i', 0)
expected_context = None
@register_switch_handler('test1') # wrolpi.switch.test signal
async def switch_handler_test1(**context):
assert context == expected_context
count1.value += 1
# Sleep so that multiple dispatches are ignored.
await asyncio.sleep(0.5)
@pytest.mark.asyncio
async def test_switches1(async_client):
global expected_context
# Dispatch "test" switch, the switch handler should call the handler above, once.
expected_context = {'foo': 'bar'}
switches.activate_switch('test1', expected_context)
await switches.await_switches(timeout=2)
assert count1.value == 1
# Dispatching many times only causes switch handler to be called once because it is already pending.
expected_context = {'foo': 'bar6'}
# These contexts are overwritten.
switch_handler_test1.activate_switch(context={'foo': 'bar1'})
switch_handler_test1.activate_switch(context={'foo': 'bar2'})
switch_handler_test1.activate_switch(context={'foo': 'bar3'})
switch_handler_test1.activate_switch(context={'foo': 'bar4'})
switch_handler_test1.activate_switch(context={'foo': 'bar5'})
switch_handler_test1.activate_switch(context=expected_context)
await switches.await_switches(timeout=2)
assert count1.value == 2
@register_switch_handler('test2')
async def switch_handler_test2(**context):
count2.value += 1
if count2.value == 2:
raise Exception('second call error')
@pytest.mark.asyncio
async def test_perpetual_signal(async_client):
"""Test that perpetual_signal survives errors."""
# Signal event is dispatched again.
switch_handler_test2.activate_switch()
await asyncio.sleep(1)
assert count2.value == 1
# Error is logged.
switch_handler_test2.activate_switch()
await asyncio.sleep(1)
assert count2.value == 2
# Switch is dispatched again.
switch_handler_test2.activate_switch()
await asyncio.sleep(1)
assert count2.value == 3
@pytest.mark.asyncio
async def test_invalid_activate_switch(async_client):
with pytest.raises(RuntimeError):
switches.activate_switch('does not exist')
with pytest.raises(RuntimeError):
switches.activate_switch('test1', 'bad context')

65
wrolpi/test/test_tags.py
View file

@ -9,11 +9,12 @@ from wrolpi.common import is_hardlinked, walk
from wrolpi.errors import FileGroupIsTagged, InvalidTag, UnknownTag, FileGroupAlreadyTagged, UsedTag
from wrolpi.files import lib as files_lib
from wrolpi.files.models import FileGroup
from wrolpi.switches import await_switches
from wrolpi.tags import TagFile, Tag
@pytest.mark.asyncio
async def test_tags_file_group_json(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
async def test_tags_file_group_json(async_client, test_session, make_files_structure, tag_factory, example_pdf):
"""The tags of a file are returned in its JSON."""
tag_one = await tag_factory()
tag_two = await tag_factory()
@ -35,7 +36,7 @@ async def test_tags_file_group_json(test_async_client, test_session, make_files_
@pytest.mark.asyncio
async def test_tags_model(test_async_client, test_session, make_files_structure, tag_factory, example_pdf):
async def test_tags_model(async_client, test_session, make_files_structure, tag_factory, example_pdf):
"""Can get Tag using class methods."""
tag1 = await tag_factory()
assert tag1.__json__()
@ -54,7 +55,7 @@ async def test_tags_model(test_async_client, test_session, make_files_structure,
@pytest.mark.asyncio
async def test_tags_file_group(test_async_client, test_session, make_files_structure, tag_factory, video_bytes,
async def test_tags_file_group(async_client, test_session, make_files_structure, tag_factory, video_bytes,
image_bytes_factory):
"""A FileGroup can be tagged with multiple Tags."""
make_files_structure({
@ -89,7 +90,7 @@ async def test_tags_file_group(test_async_client, test_session, make_files_struc
@pytest.mark.asyncio
async def test_tags_config_file(test_async_client, test_session, test_directory, tag_factory, example_pdf, video_file,
async def test_tags_config_file(async_client, test_session, test_directory, tag_factory, example_pdf, video_file,
test_tags_config):
"""Test that the config is updated when a FileGroup is tagged."""
await files_lib.refresh_files()
@ -99,31 +100,37 @@ async def test_tags_config_file(test_async_client, test_session, test_directory,
tag2 = await tag_factory()
test_session.commit()
tags.save_tags_config(test_session)
tags.save_tags_config.activate_switch()
await await_switches()
assert tag1.name in test_tags_config.read_text()
assert tag2.name in test_tags_config.read_text()
video.add_tag(tag1.id)
await await_switches()
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
# Tag PDF twice.
pdf.add_tag(tag1.id)
pdf.add_tag(tag2.id)
await await_switches()
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) in test_tags_config.read_text()
# One TagFile still exists.
pdf.untag(tag1.id)
await await_switches()
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) in test_tags_config.read_text()
# PDF is no longer tagged.
pdf.untag(tag2.id)
await await_switches()
assert str(video_file.relative_to(test_directory)) in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
video.untag(tag1.id)
await await_switches()
assert str(video_file.relative_to(test_directory)) not in test_tags_config.read_text()
assert str(example_pdf.relative_to(test_directory)) not in test_tags_config.read_text()
@ -136,25 +143,25 @@ async def test_tags_config_file(test_async_client, test_session, test_directory,
@pytest.mark.asyncio
async def test_tags_crud(test_async_client, test_session, example_pdf, assert_tags_config):
async def test_tags_crud(async_client, test_session, example_pdf, assert_tags_config):
"""Test API Create/Retrieve/Update/Delete of Tags."""
pdf = FileGroup.from_paths(test_session, example_pdf)
test_session.add(pdf)
test_session.commit()
# Can get empty tags.
request, response = await test_async_client.get('/api/tag')
request, response = await async_client.get('/api/tag')
assert response.status_code == HTTPStatus.OK
assert response.json['tags'] == list()
# Tags can be created.
content = dict(name='jardín', color='#123456')
request, response = await test_async_client.post('/api/tag', content=json.dumps(content))
request, response = await async_client.post('/api/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.CREATED
assert_tags_config(tags={'jardín': {'color': '#123456'}})
# The tag can be retrieved.
request, response = await test_async_client.get('/api/tag')
request, response = await async_client.get('/api/tag')
assert response.status_code == HTTPStatus.OK
assert response.json['tags'] == [dict(name='jardín', color='#123456', id=1, file_group_count=0, zim_entry_count=0)]
@ -165,34 +172,34 @@ async def test_tags_crud(test_async_client, test_session, example_pdf, assert_ta
# The tag can be updated.
content = dict(name='ガーデン', color='#000000')
request, response = await test_async_client.post('/api/tag/1', content=json.dumps(content))
request, response = await async_client.post('/api/tag/1', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
request, response = await test_async_client.get('/api/tag')
request, response = await async_client.get('/api/tag')
assert response.json['tags'] == [
dict(name='ガーデン', color='#000000', id=1, file_group_count=1, zim_entry_count=0)]
assert_tags_config(tags={'ガーデン': {'color': '#000000'}})
# Tag color can be changed
content = dict(name='ガーデン', color='#ffffff')
request, response = await test_async_client.post('/api/tag/1', content=json.dumps(content))
request, response = await async_client.post('/api/tag/1', content=json.dumps(content))
assert response.status_code == HTTPStatus.OK
assert Tag.find_by_name('ガーデン').color == '#ffffff'
# Conflicting names return an error.
content = dict(name='ガーデン', color='#111111')
request, response = await test_async_client.post('/api/tag', content=json.dumps(content))
request, response = await async_client.post('/api/tag', content=json.dumps(content))
assert response.status_code == HTTPStatus.BAD_REQUEST
assert_tags_config(tags={'ガーデン': {'color': '#ffffff'}})
# Cannot delete Tag that is used.
request, response = await test_async_client.delete('/api/tag/1')
request, response = await async_client.delete('/api/tag/1')
assert response.status_code == HTTPStatus.BAD_REQUEST
pdf.untag(tag.id, test_session)
test_session.commit()
# Can delete unused Tag.
request, response = await test_async_client.delete('/api/tag/1')
request, response = await async_client.delete('/api/tag/1')
assert response.status_code == HTTPStatus.NO_CONTENT
# Config is empty.
@ -222,7 +229,7 @@ def test_import_empty_tags_config(test_session, test_tags_config):
@pytest.mark.asyncio
async def test_import_tags_config(test_async_client, test_session, test_directory, test_tags_config,
async def test_import_tags_config(async_client, test_session, test_directory, test_tags_config,
example_singlefile):
with test_tags_config.open('wt') as fh:
data = dict(
@ -278,7 +285,7 @@ async def test_import_tags_config_missing_file(test_session, test_directory, tes
@pytest.mark.asyncio
async def test_import_tags_delete_missing(test_session, test_directory, test_tags_config, make_files_structure,
tag_factory, test_zim):
tag_factory, test_zim, async_client):
"""Tags not in the config are deleted on import. Tags that are used will not be deleted."""
from modules.zim import lib as zim_lib
@ -291,7 +298,8 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
foo.add_tag(tag1.id)
# use tag4 for Zim entry.
await zim_lib.add_tag(tag4.name, test_zim.id, 'home')
tags.save_tags_config()
tags.save_tags_config.activate_switch()
await await_switches()
# All tags were saved.
assert tag1.name in test_tags_config.read_text()
@ -304,6 +312,7 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
config_dict = config.dict()
config_dict['tags'] = {k: v for k, v in config.tags.items() if k == tag3.name}
config.update(config_dict)
await await_switches()
config.initialize()
# Tag Files and Tag Zims are untouched. Only tag3 is left.
assert len(config.tag_files) == 1
@ -320,7 +329,8 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
}
# DB is written to config with only those Tags left.
tags.save_tags_config()
tags.save_tags_config.activate_switch()
await await_switches()
# Tags that are used are restored.
assert len(config.tag_files) == 1
assert len(config.tag_zims) == 1
@ -328,7 +338,7 @@ async def test_import_tags_delete_missing(test_session, test_directory, test_tag
@pytest.mark.asyncio
async def test_invalid_tag(test_async_client, test_session, test_directory):
async def test_invalid_tag(async_client, test_session, test_directory):
with pytest.raises(InvalidTag):
await tags.upsert_tag('cannot use comma: ,', '#fff')
@ -340,7 +350,7 @@ async def test_invalid_tag(test_async_client, test_session, test_directory):
@pytest.mark.asyncio
async def test_tags_directory(test_async_client, test_session, test_directory, tag_factory, video_factory,
async def test_tags_directory(async_client, test_session, test_directory, tag_factory, video_factory,
example_pdf):
"""Test that Tag Directory is synchronized with database."""
readme = test_directory / 'tags/README.txt'
@ -355,6 +365,8 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
with_poster_ext='jpg')
vid2 = video_factory(with_video_file=test_directory / 'vid2.mp4', with_info_json=True)
pdf = FileGroup.from_paths(test_session, example_pdf)
test_session.commit()
await await_switches()
assert 'vid1' in str(vid1.file_group.primary_path)
assert 'vid2' in str(vid2.file_group.primary_path)
assert 'pdf' in str(pdf.primary_path)
@ -378,6 +390,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
vid2.add_tag(tag1.id)
# PDF tagged with Special/name.
pdf.add_tag(tag2.id)
await await_switches()
assert_tag_links([
'First Aid, Special⧸name/vid1.en.vtt',
'First Aid, Special⧸name/vid1.en.vtt',
@ -397,6 +410,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
# Video1 tagged with First Aid only.
vid1.untag(tag2.id)
await await_switches()
assert_tag_links([
'First Aid/vid1.en.vtt',
'First Aid/vid1.en.vtt',
@ -413,6 +427,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
# PDF no longer tagged.
pdf.untag(tag2.id)
await await_switches()
assert_tag_links([
'First Aid/vid1.en.vtt',
'First Aid/vid1.en.vtt',
@ -430,6 +445,7 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
# No more tagged files.
vid1.untag(tag1.id)
vid2.untag(tag1.id)
await await_switches()
assert test_session.query(TagFile).count() == 0
# Readme, and un-deletable file and directory exist.
assert len(list(walk(test_directory / 'tags'))) == 3
@ -443,12 +459,13 @@ async def test_tags_directory(test_async_client, test_session, test_directory, t
@pytest.mark.asyncio
async def test_update_tag(test_async_client, test_session, test_directory, video_factory, tag_factory):
async def test_update_tag(async_client, test_session, test_directory, video_factory, tag_factory):
"""Linked files in the Tag Directory are moved when the tag is renamed."""
# Create tagged Video.
tag = await tag_factory()
video_factory(title='video', tag_names=[tag.name])
test_session.commit()
await await_switches()
# Video file is linked in tag directory.
assert (test_directory / 'tags').is_dir()
@ -457,6 +474,7 @@ async def test_update_tag(test_async_client, test_session, test_directory, video
# Invalid characters are replaced during rename.
await tags.upsert_tag('new/name%', tag.color, tag.id)
await await_switches()
# Video file was moved.
assert (test_directory / 'tags').is_dir()
@ -466,12 +484,13 @@ async def test_update_tag(test_async_client, test_session, test_directory, video
@pytest.mark.asyncio
async def test_tag_rename_with_channel(test_async_client, test_session, test_directory, video_factory, tag_factory,
async def test_tag_rename_with_channel(async_client, test_session, test_directory, video_factory, tag_factory,
channel_factory):
tag = await tag_factory()
channel = channel_factory(name='Channel Name', tag_name=tag.name)
video_factory(title='video', channel_id=channel.id)
test_session.commit()
await await_switches()
assert (test_directory / 'tags').is_dir()
assert (test_directory / 'videos/one/Channel Name').is_dir()