mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
A RequestSession no longer expires loaded objects at commit, so a handler serializes what it committed without a reload query. An AsyncSession cannot run that reload at all, so this is groundwork for moving request handling to async. Background sessions (get_db_session) keep SQLAlchemy's default: they change data outside the ORM (raw SQL, triggers) and rely on the reload. Collection, CollectionItem and Channel dict() now run inside the serializer guard. Fixes it found: - Channel.dict() no longer loads the deferred yt-dlp catalog (info_json), which the channel handler loaded only to discard. - get_channel eager-loads what Channel.dict() reads (populate_existing replaces a refresh query). - create_channel / get_or_create_download link through relationships, so a loaded Collection.downloads stays current. - Setting a Repo to Never removes the deleted Download from the loaded collection; the response reported the old frequency. - The downloader no longer calls channel.dict() to preload a Channel; its callers read it inside the session. Tests run each request with expiry off, as production does, and turn it back on for the background work that shares the test Session. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
1320 lines
46 KiB
Python
1320 lines
46 KiB
Python
"""
|
|
Fixtures for Pytest tests.
|
|
"""
|
|
import asyncio
|
|
import contextlib
|
|
import copy
|
|
import http.server
|
|
import json
|
|
import logging
|
|
import multiprocessing
|
|
import pathlib
|
|
import shutil
|
|
import sqlite3
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
from abc import ABC
|
|
from datetime import datetime
|
|
from itertools import zip_longest
|
|
from pathlib import Path
|
|
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional, Any, Generator, AsyncGenerator
|
|
from typing import Tuple, Set
|
|
from unittest import mock
|
|
from unittest.mock import MagicMock, AsyncMock
|
|
from uuid import uuid1, uuid4
|
|
|
|
import pytest
|
|
import sqlalchemy
|
|
import yaml
|
|
from PIL import Image
|
|
from sanic_testing.testing import SanicASGITestClient
|
|
from sqlalchemy.engine import Engine, create_engine
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
from sqlalchemy.pool import NullPool
|
|
|
|
# Import root api so blueprints are attached.
|
|
import wrolpi.root_api # noqa
|
|
from wrolpi import flags
|
|
from wrolpi.api_utils import api_app
|
|
from wrolpi.cmd import CommandResult, run_command
|
|
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL, \
|
|
get_wrolpi_config
|
|
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
|
|
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
|
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
|
|
from wrolpi.dates import set_test_now
|
|
from wrolpi.db import create_wrolpi_engine, RequestDB, RequestSession, REQUEST_SESSION_OPTIONS
|
|
from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \
|
|
downloads_manager_config_context, download_cache_config_context
|
|
from wrolpi.errors import UnrecoverableDownloadError
|
|
from wrolpi.files.models import Directory, FileGroup
|
|
from wrolpi.switches import await_switches as await_switches_
|
|
from wrolpi.tags import Tag, upsert_tag
|
|
from wrolpi.vars import PROJECT_DIR
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
|
|
def test_db() -> Tuple[Engine, Session]:
|
|
"""Create a SQLAlchemy engine/session on a SQLite DB inside the (temporary) media directory.
|
|
|
|
The DB file is at the production location (`<media>/config/wrolpi.db`), so each test's
|
|
TemporaryDirectory provides isolation (also under pytest-xdist) and removes the DB (and its
|
|
WAL sidecars) on teardown."""
|
|
from wrolpi.common import get_media_directory, is_tempfile
|
|
from wrolpi.schema_ddl import install_raw_ddl
|
|
|
|
db_file = get_media_directory() / 'config' / 'wrolpi.db'
|
|
if not is_tempfile(db_file):
|
|
raise ValueError(f'Refusing to test on a non-temporary database! {db_file}')
|
|
db_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
test_engine = create_wrolpi_engine(db_file)
|
|
# Create all tables. No need to check if they exist because this is a test DB.
|
|
Base.metadata.create_all(test_engine, checkfirst=False)
|
|
# Triggers and FTS5 tables, exactly as the alembic baseline installs them.
|
|
with test_engine.begin() as conn:
|
|
install_raw_ddl(conn)
|
|
session = sessionmaker(bind=test_engine)()
|
|
return test_engine, session
|
|
|
|
|
|
@pytest.fixture()
|
|
def test_session(test_directory) -> Generator[Session, Any, None]:
|
|
"""Pytest Fixture to get a test database session."""
|
|
test_engine, session = test_db()
|
|
|
|
def fake_get_db_session():
|
|
"""Get the testing db"""
|
|
return test_engine, session
|
|
|
|
def fake_get_request_session():
|
|
"""A request shares the testing db, but commits like a `RequestSession`: without expiring. The default
|
|
returns once the handler is done (`background_task_listener`, `finish`), for the background code which
|
|
shares the session."""
|
|
session.expire_on_commit = REQUEST_SESSION_OPTIONS['expire_on_commit']
|
|
return test_engine, session
|
|
|
|
finish = RequestDB.finish
|
|
|
|
def fake_finish(self, ok: bool):
|
|
try:
|
|
finish(self, ok)
|
|
finally:
|
|
session.expire_on_commit = True
|
|
|
|
try:
|
|
with mock.patch('wrolpi.db._get_db_session', fake_get_db_session), \
|
|
mock.patch('wrolpi.db._get_request_session', fake_get_request_session), \
|
|
mock.patch.object(RequestDB, 'finish', fake_finish):
|
|
yield session
|
|
finally:
|
|
session.rollback()
|
|
session.close()
|
|
test_engine.dispose()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def no_unexpected_queries(request):
|
|
"""Fail any test during which a serializer ran SQL, even if the code under test caught the error."""
|
|
from wrolpi import db
|
|
db.UNEXPECTED_QUERIES.clear()
|
|
yield
|
|
if request.node.get_closest_marker('allow_unexpected_queries'):
|
|
return
|
|
queries = list(db.UNEXPECTED_QUERIES)
|
|
db.UNEXPECTED_QUERIES.clear()
|
|
assert not queries, 'A serializer ran SQL (a relationship or deferred column was not loaded):\n' + \
|
|
'\n'.join(f'{reason}: {" ".join(statement.split())[:300]}' for reason, statement in queries)
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def test_debug_logger(request):
|
|
"""Tests default to DEBUG logging. Use pytest -vvv for TRACE level."""
|
|
level = logging.DEBUG
|
|
if request.config.option.verbose >= 3:
|
|
# -vvv
|
|
level = TRACE_LEVEL
|
|
|
|
with log_level_context(level):
|
|
yield
|
|
|
|
|
|
@pytest.fixture
|
|
def clear_background_tasks(request):
|
|
"""Clear BACKGROUND_TASKS before each test and fail if orphaned tasks remain after.
|
|
|
|
Tests must properly await all background tasks they create. Orphaned tasks indicate
|
|
a test is not properly cleaning up, which can cause cascading failures in other tests.
|
|
"""
|
|
# Clear before test - cancel any orphaned tasks from previous tests
|
|
if BACKGROUND_TASKS:
|
|
logger.warning(f'clear_background_tasks: clearing {len(BACKGROUND_TASKS)} orphaned tasks BEFORE test')
|
|
for task in list(BACKGROUND_TASKS):
|
|
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
|
|
logger.warning(f' orphaned task from previous test: {task_name}')
|
|
if not task.done():
|
|
task.cancel()
|
|
BACKGROUND_TASKS.clear()
|
|
|
|
yield
|
|
|
|
# After test - fail if orphaned tasks remain
|
|
if BACKGROUND_TASKS:
|
|
orphaned_tasks = []
|
|
for task in list(BACKGROUND_TASKS):
|
|
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
|
|
orphaned_tasks.append(f'{task_name} (done={task.done()})')
|
|
if not task.done():
|
|
task.cancel()
|
|
BACKGROUND_TASKS.clear()
|
|
|
|
# Fail the test
|
|
pytest.fail(
|
|
f'Test left {len(orphaned_tasks)} orphaned background task(s): {", ".join(orphaned_tasks)}. '
|
|
f'Tests must await all background tasks they create using await_background_tasks or '
|
|
f'the async_client fixture.'
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def test_directory() -> Generator[Path, Any, None]:
|
|
"""
|
|
Overwrite the media directory with a temporary directory.
|
|
"""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
tmp_path = pathlib.Path(tmp_dir).resolve()
|
|
tmp_path.chmod(0o40755)
|
|
assert tmp_path.is_dir()
|
|
set_test_media_directory(tmp_path)
|
|
yield tmp_path
|
|
|
|
|
|
@pytest.fixture
|
|
def test_wrolpi_config(test_directory) -> Generator[Path, Any, None]:
|
|
"""
|
|
Create a test config based off the example config.
|
|
"""
|
|
config_path = test_directory / 'config/wrolpi.yaml'
|
|
set_test_config(True)
|
|
yield config_path
|
|
set_test_config(False)
|
|
|
|
|
|
ROUTES_ATTACHED = False
|
|
|
|
|
|
@api_app.on_response
|
|
async def background_task_listener(request, response):
|
|
"""Wait for all background tasks to finish before returning API response while testing."""
|
|
if session := getattr(request.ctx, 'session', None):
|
|
# The handler is done. Background work shares the test's Session, and expires on commit like production.
|
|
session.expire_on_commit = True
|
|
await await_switches_()
|
|
|
|
|
|
class AutoAwaitTestClient:
|
|
"""
|
|
A wrapper around SanicASGITestClient that automatically awaits background
|
|
tasks and switches after modifying HTTP requests (POST, PUT, PATCH, DELETE).
|
|
|
|
This eliminates the need to manually call await_switches() and
|
|
await_background_tasks() after each modifying request in tests.
|
|
"""
|
|
|
|
def __init__(self, client: SanicASGITestClient, session: Session):
|
|
self._client = client
|
|
self._session = session
|
|
|
|
async def _await_and_expire(self):
|
|
"""Await all background tasks and switches, then expire session objects."""
|
|
await await_switches_()
|
|
await await_background_tasks_()
|
|
self._session.expire_all()
|
|
|
|
async def post(self, *args, **kwargs):
|
|
"""POST request with auto-await."""
|
|
result = await self._client.post(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def put(self, *args, **kwargs):
|
|
"""PUT request with auto-await."""
|
|
result = await self._client.put(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def patch(self, *args, **kwargs):
|
|
"""PATCH request with auto-await."""
|
|
result = await self._client.patch(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def delete(self, *args, **kwargs):
|
|
"""DELETE request with auto-await."""
|
|
result = await self._client.delete(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def get(self, *args, **kwargs):
|
|
"""GET request (no auto-await)."""
|
|
return await self._client.get(*args, **kwargs)
|
|
|
|
async def head(self, *args, **kwargs):
|
|
"""HEAD request (no auto-await)."""
|
|
return await self._client.head(*args, **kwargs)
|
|
|
|
async def options(self, *args, **kwargs):
|
|
"""OPTIONS request (no auto-await)."""
|
|
return await self._client.options(*args, **kwargs)
|
|
|
|
async def websocket(self, *args, **kwargs):
|
|
"""WebSocket connection (pass-through)."""
|
|
return await self._client.websocket(*args, **kwargs)
|
|
|
|
@property
|
|
def unawaited(self) -> SanicASGITestClient:
|
|
"""The underlying client: requests return without awaiting switches or background tasks."""
|
|
return self._client
|
|
|
|
@property
|
|
def sanic_app(self):
|
|
"""Access to the underlying Sanic app."""
|
|
return self._client.sanic_app
|
|
|
|
|
|
@pytest.fixture
|
|
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[
|
|
AutoAwaitTestClient, Any]:
|
|
"""Get an Async Sanic Test Client with all default routes attached.
|
|
|
|
Depends on test_session to ensure the database mock is in place for session middleware.
|
|
Depends on clear_background_tasks to detect orphaned background tasks.
|
|
|
|
This client automatically awaits switches and background tasks after
|
|
POST, PUT, PATCH, and DELETE requests, and expires session objects.
|
|
"""
|
|
api_app.signalize()
|
|
attach_shared_contexts(api_app)
|
|
initialize_configs_contexts(api_app)
|
|
|
|
client = SanicASGITestClient(api_app)
|
|
wrapped_client = AutoAwaitTestClient(client, test_session)
|
|
|
|
try:
|
|
yield wrapped_client
|
|
finally:
|
|
api_app.stop()
|
|
|
|
|
|
@pytest.fixture
|
|
def test_download_manager_config(async_client, test_directory) -> Generator[Path, Any, None]:
|
|
with downloads_manager_config_context():
|
|
(test_directory / 'config').mkdir(exist_ok=True)
|
|
config_path = test_directory / 'config/download_manager.yaml'
|
|
yield config_path
|
|
|
|
|
|
@pytest.fixture
|
|
def test_download_cache_config(async_client, test_directory) -> Generator[Path, Any, None]:
|
|
with download_cache_config_context():
|
|
(test_directory / 'config').mkdir(exist_ok=True)
|
|
config_path = test_directory / 'config/download_cache.yaml'
|
|
yield config_path
|
|
|
|
|
|
@pytest.fixture
|
|
async def await_switches(async_client):
|
|
"""Returns the `switches.await_switches` function, but primes the Sanic App to actually handle signals."""
|
|
await async_client.get('/api')
|
|
return await_switches_
|
|
|
|
|
|
@pytest.fixture
|
|
async def await_background_tasks(async_client, test_session):
|
|
"""Returns a function that awaits background tasks and expires session objects.
|
|
|
|
After background tasks complete, session objects may be stale, so we automatically
|
|
expire them to force a refresh on next access.
|
|
|
|
Args:
|
|
timeout: Optional timeout in seconds (default 15s from await_background_tasks_)
|
|
"""
|
|
await async_client.get('/api')
|
|
|
|
async def _await_and_expire(timeout: int = 10):
|
|
await await_background_tasks_(timeout=timeout)
|
|
test_session.expire_all()
|
|
|
|
return _await_and_expire
|
|
|
|
|
|
@pytest.fixture
|
|
async def test_download_manager(
|
|
async_client,
|
|
test_session, # session is required because downloads can start without the test DB in place.
|
|
test_download_manager_config,
|
|
) -> AsyncGenerator[DownloadManager, Any]:
|
|
with timer('test_download_manager'):
|
|
# Needed to use signals in
|
|
manager = DownloadManager()
|
|
await manager.enable()
|
|
|
|
# Disable per-domain rate limiting during tests to avoid timeouts.
|
|
get_wrolpi_config().download_wait = 0
|
|
|
|
yield manager
|
|
|
|
manager.stop()
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_now() -> Generator[Callable[..., Any], Any, None]:
|
|
try:
|
|
set_test_now(datetime(2000, 1, 1))
|
|
yield set_test_now
|
|
finally:
|
|
# reset now() to its original functionality.
|
|
set_test_now(None) # noqa
|
|
|
|
|
|
@pytest.fixture
|
|
def successful_download() -> DownloadResult:
|
|
return DownloadResult(success=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def failed_download() -> DownloadResult:
|
|
return DownloadResult(error='wrolpi.conftest failed_download error', success=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_download_urls(test_session) -> Callable[[Set[str]], None]:
|
|
def asserter(expected_urls: Set[str]):
|
|
downloads = test_session.query(Download).all()
|
|
urls = {i.url for i in downloads}
|
|
if urls != set(expected_urls):
|
|
raise AssertionError(f'Download URLs do not match: {urls} != {expected_urls}')
|
|
|
|
return asserter
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
def asserter(expected: List[Dict]):
|
|
downloads = test_session.query(Download).order_by(Download.url)
|
|
for download_, expected_ in zip_longest(downloads, expected):
|
|
assert_dict_contains(download_.__json__(), expected_)
|
|
if (count := test_session.query(Download).count()) != len(expected):
|
|
raise AssertionError(f'Download count does not match: {count} != {len(expected)}')
|
|
|
|
return asserter
|
|
|
|
|
|
def make_test_ctx(*, is_cancelled=None, can_download=None,
|
|
outside_download_window=None, run_command_=None,
|
|
download_timeout=None,
|
|
report_progress=None, clear_progress=None) -> DownloadContext:
|
|
"""Build a DownloadContext suitable for downloader unit tests.
|
|
|
|
Defaults: never-cancelled, can_download True, no window restriction, real run_command,
|
|
no global timeout, progress reporting is a no-op. Override any callable to simulate
|
|
cancellation, throttling, fake subprocess output, or to record progress events.
|
|
|
|
Lives in wrolpi/conftest.py so any test (including primitive tests of process_runner /
|
|
cancel_wrapper) can construct a ctx without crossing module boundaries.
|
|
"""
|
|
return DownloadContext(
|
|
is_cancelled=is_cancelled or (lambda: False),
|
|
can_download=can_download or (lambda: True),
|
|
outside_download_window=outside_download_window or (lambda: False),
|
|
run_command=run_command_ or run_command,
|
|
download_timeout=download_timeout,
|
|
report_progress=report_progress or (lambda p: None),
|
|
clear_progress=clear_progress or (lambda: None),
|
|
)
|
|
|
|
|
|
def probe_write_lock_is_held(db_file: str) -> bool:
|
|
"""Return True if a competing connection cannot immediately take the write lock.
|
|
|
|
Uses a zero busy-timeout so the probe fails instantly if another connection holds a
|
|
RESERVED (write) lock, rather than waiting. In WAL mode a plain reader holds no such
|
|
lock, so this only returns True when a writer transaction is actually in progress.
|
|
"""
|
|
probe = sqlite3.connect(db_file, timeout=0)
|
|
try:
|
|
probe.execute('PRAGMA busy_timeout=0')
|
|
try:
|
|
probe.execute('BEGIN IMMEDIATE')
|
|
except sqlite3.OperationalError as e:
|
|
if 'database is locked' in str(e):
|
|
return True
|
|
raise
|
|
probe.execute('ROLLBACK')
|
|
return False
|
|
finally:
|
|
probe.close()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def write_lock_held_briefly(db_file: str, seconds: float = 1.0):
|
|
"""Hold the SQLite write lock on `db_file` for `seconds`, from another connection.
|
|
|
|
Simulates the concurrent writer (refresh, download, config save) that a real WROLPi runs
|
|
alongside whatever the test is doing. The lock is released while the test's work is still in
|
|
flight, so a correctly-written transaction waits (busy_timeout) and then succeeds; a deferred
|
|
one that upgrades its lock mid-transaction fails instantly instead.
|
|
|
|
Pair with `production_like_sessions` — the shared `test_session` uses one connection, which
|
|
cannot contend with itself.
|
|
"""
|
|
holder = sqlite3.connect(db_file, timeout=30, check_same_thread=False)
|
|
holder.isolation_level = None
|
|
holder.execute('PRAGMA busy_timeout=30000')
|
|
holder.execute('BEGIN IMMEDIATE')
|
|
|
|
def release():
|
|
time.sleep(seconds)
|
|
holder.execute('ROLLBACK')
|
|
holder.close()
|
|
|
|
thread = threading.Thread(target=release, daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield
|
|
finally:
|
|
thread.join(timeout=30)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None]:
|
|
"""Patch get_db_session to hand out a *fresh* Session per call, like production.
|
|
|
|
The `test_session` fixture reuses one shared Session for every `get_db_session` call, so an object loaded in
|
|
one `with` block stays attached for the next and exceptions never roll back. That convenience masks
|
|
detached-object and transaction-rollback bugs that only bite production's per-call sessions. Use this to
|
|
exercise that behavior: the yielded sessionmaker creates independent Sessions so a test can verify committed
|
|
DB state the way a separate worker would see it. API requests get a fresh `RequestSession` each, as in
|
|
production.
|
|
|
|
A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB
|
|
is dropped during fixture teardown.
|
|
"""
|
|
engine = create_wrolpi_engine(test_session.get_bind().url)
|
|
maker = sessionmaker(bind=engine)
|
|
sessions = []
|
|
|
|
def factory():
|
|
session = maker()
|
|
sessions.append(session)
|
|
return engine, session
|
|
|
|
def request_factory():
|
|
session = RequestSession(engine)
|
|
sessions.append(session)
|
|
return engine, session
|
|
|
|
try:
|
|
with mock.patch('wrolpi.db._get_db_session', factory), \
|
|
mock.patch('wrolpi.db._get_request_session', request_factory):
|
|
yield maker
|
|
finally:
|
|
for session in sessions:
|
|
try:
|
|
session.close()
|
|
except Exception as e:
|
|
# Best-effort: engine.dispose() below releases the connections regardless, so a failed
|
|
# close() is inconsequential. Log at debug so it surfaces under -vvv without noising CI.
|
|
logger.debug(f'production_like_sessions: failed to close a session: {e}')
|
|
engine.dispose()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def count_connections(maker: sessionmaker) -> Generator[List[int], Any, None]:
|
|
"""Record every SQLite connection opened on a `production_like_sessions` engine during the block.
|
|
|
|
Yields a list that gains one entry per connection, so `len()` is the count. Each connection
|
|
costs a connect plus WROLPi's PRAGMAs (about 3 ms on a Raspberry Pi 4), so an API request should
|
|
open at most one."""
|
|
engine = maker.kw['bind']
|
|
opened = list()
|
|
|
|
def on_connect(*_):
|
|
opened.append(1)
|
|
|
|
sqlalchemy.event.listen(engine, 'connect', on_connect)
|
|
try:
|
|
yield opened
|
|
finally:
|
|
sqlalchemy.event.remove(engine, 'connect', on_connect)
|
|
|
|
|
|
async def count_request_connections(async_client: 'AutoAwaitTestClient', test_session: Session, method: str,
|
|
path: str, **kwargs) -> Tuple[Any, int]:
|
|
"""Send one request with production's per-call sessions; return its response and how many SQLite
|
|
connections the request opened.
|
|
|
|
The test's data must already be in `test_session` (it is committed here so other connections see
|
|
it). Switches and background tasks are awaited only after counting, so the count is the request's
|
|
own work: in tests, `background_task_listener` would otherwise run switches inside the request,
|
|
where production runs them in the switch worker."""
|
|
test_session.commit()
|
|
with production_like_sessions(test_session) as maker:
|
|
with count_connections(maker) as opened, \
|
|
mock.patch(f'{__name__}.await_switches_', mock.AsyncMock()):
|
|
_, response = await getattr(async_client.unawaited, method.lower())(path, **kwargs)
|
|
count = len(opened)
|
|
await await_switches_()
|
|
await await_background_tasks_()
|
|
test_session.expire_all()
|
|
return response, count
|
|
|
|
|
|
@pytest.fixture
|
|
def test_downloader(test_download_manager) -> Downloader:
|
|
class TestDownloader(Downloader, ABC):
|
|
"""A testing Downloader.
|
|
|
|
prepare_download returns a non-None sentinel so the manager's phase-split dispatch
|
|
routes through this subclass. execute_download is an AsyncMock — tests wire its
|
|
side_effect via the set_test_* methods. Returning a DownloadResult from
|
|
execute_download tells the dispatch to skip finalize (see signal_download_download).
|
|
"""
|
|
name = 'test_downloader'
|
|
|
|
def __repr__(self):
|
|
return '<TESTING Downloader>'
|
|
|
|
execute_download = AsyncMock()
|
|
|
|
def prepare_download(self, session, download):
|
|
return object() # sentinel; activates the phase-split path
|
|
|
|
def finalize_download(self, session, download, executed):
|
|
# execute_download returns a DownloadResult directly, so the dispatch
|
|
# short-circuits before reaching finalize. Provided here only to satisfy
|
|
# the abstract contract.
|
|
return DownloadResult(success=True)
|
|
|
|
def set_test_success(self):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
return DownloadResult(success=True)
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
def set_test_failure(self):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
return DownloadResult(success=False)
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
raise exception
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
def set_test_unrecoverable_exception(self):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
raise UnrecoverableDownloadError()
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
test_downloader = TestDownloader()
|
|
# Default to successful download.
|
|
test_downloader.set_test_success()
|
|
test_download_manager.register_downloader(test_downloader)
|
|
|
|
return test_downloader
|
|
|
|
|
|
@pytest.fixture
|
|
def video_file(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
|
destination = test_directory / f'video-{uuid4()}.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
|
|
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def corrupted_video_file(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of a corrupted video file in the `test_directory`."""
|
|
destination = test_directory / f'corrupted-{uuid4()}.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/corrupted.mp4', destination)
|
|
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]:
|
|
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
|
|
|
def _(path: pathlib.Path = None) -> pathlib.Path:
|
|
destination = path or test_directory / f'{uuid4()}.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
|
|
|
|
return destination
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def video_bytes() -> bytes:
|
|
return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()
|
|
|
|
|
|
@pytest.fixture
|
|
def image_file(test_directory) -> Generator[Path, Any, None]:
|
|
"""Create a small image file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.jpeg'
|
|
Image.new('RGB', (25, 25), color='grey').save(destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def vtt_file1(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example1 VTT file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.en.vtt'
|
|
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def vtt_file2(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example2 VTT file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.en.vtt'
|
|
shutil.copy(PROJECT_DIR / 'test/example2.en.vtt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def srt_text() -> str:
|
|
return (PROJECT_DIR / 'test/example3.en.srt').read_text()
|
|
|
|
|
|
@pytest.fixture
|
|
def srt_file3(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example3 SRT file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.en.srt'
|
|
shutil.copy(PROJECT_DIR / 'test/example3.en.srt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_pdf(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'pdf example.pdf'
|
|
shutil.copy(PROJECT_DIR / 'test/pdf example.pdf', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_epub(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'ebook example.epub'
|
|
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_mobi(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'ebook example.mobi'
|
|
shutil.copy(PROJECT_DIR / 'test/ebook example.mobi', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_doc(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'example word.doc'
|
|
shutil.copy(PROJECT_DIR / 'test/example word.doc', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_docx(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'example word.docx'
|
|
shutil.copy(PROJECT_DIR / 'test/example word.docx', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_odt(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'test_book_odt.odt'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/test_book_odt.odt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbz(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_sample.cbz'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_sample.cbz', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbz_metadata(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_metadata.cbz'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_metadata.cbz', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbt(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_cbt.cbt'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbt.cbt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbr(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_cbr.cbr'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbr.cbr', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cb7(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_cb7.cb7'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cb7.cb7', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbz_dir(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_dir_cbz.cbz'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbz.cbz', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbt_dir(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_dir_cbt.cbt'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbt.cbt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cb7_dir(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_dir_cb7.cb7'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cb7.cb7', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
|
|
"""
|
|
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
|
|
locations. If a dict is provided, files will be created containing the value of the dict item.
|
|
"""
|
|
|
|
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) \
|
|
-> List[pathlib.Path]:
|
|
# Note: session is optional here for backwards compatibility - file_groups=True requires session
|
|
files = []
|
|
|
|
@iterify(list)
|
|
def touch_paths(paths_):
|
|
for name in paths_:
|
|
path = test_directory / name
|
|
if name.endswith('/'):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.touch()
|
|
yield path
|
|
|
|
if isinstance(paths, list):
|
|
files = touch_paths(paths)
|
|
elif isinstance(paths, Dict):
|
|
files = touch_paths(paths.keys())
|
|
for path, content in paths.items():
|
|
path = test_directory / path
|
|
if isinstance(content, str):
|
|
path.write_text(content)
|
|
elif isinstance(content, bytes):
|
|
path.write_bytes(content)
|
|
|
|
if file_groups:
|
|
for idx, path in enumerate(files):
|
|
fg = FileGroup.from_paths(session, path)
|
|
fg.do_index()
|
|
files[idx] = fg
|
|
return sorted(files) if isinstance(paths, dict) else files
|
|
|
|
return create_files
|
|
|
|
|
|
async def set_wrol_mode(enabled: bool):
|
|
if enabled:
|
|
enable_wrol_mode()
|
|
else:
|
|
await disable_wrol_mode()
|
|
|
|
|
|
@pytest.fixture
|
|
def wrol_mode_fixture(test_wrolpi_config, test_download_manager) -> Callable[[bool], Coroutine]:
|
|
return set_wrol_mode
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_create_subprocess_shell() -> Callable:
|
|
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
|
|
async def communicate(*a, **kw):
|
|
if communicate_side_effect:
|
|
return communicate_side_effect()
|
|
return communicate_return or (None, None)
|
|
|
|
async def create_subprocess_shell(*a, **kw):
|
|
proc = MagicMock()
|
|
proc.communicate = communicate
|
|
proc.returncode = return_code if return_code is not None else 0
|
|
return proc
|
|
|
|
return create_subprocess_shell
|
|
|
|
return mocker
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_run_command():
|
|
"""Mock wrolpi.cmd's `run_command` using `TESTING_RUN_COMMAND_RESULT`."""
|
|
default = CommandResult(
|
|
return_code=0,
|
|
cancelled=False,
|
|
stdout=b'mock_run_command fixture stdout',
|
|
stderr=b'mock_run_command fixture stderr',
|
|
elapsed=0,
|
|
pid=123,
|
|
)
|
|
|
|
with mock.patch('wrolpi.cmd.TESTING_RUN_COMMAND_RESULT') as mock_TESTING_RUN_COMMAND_RESULT:
|
|
mock_TESTING_RUN_COMMAND_RESULT.return_value = default
|
|
|
|
def do_mock(new_result):
|
|
mock_TESTING_RUN_COMMAND_RESULT.return_value = new_result
|
|
|
|
yield do_mock
|
|
|
|
|
|
@pytest.fixture
|
|
def run_command_bad_result() -> CommandResult:
|
|
return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123)
|
|
|
|
|
|
class EventsFixture:
|
|
"""Fixture for testing events with rich assertions.
|
|
|
|
Only returns events created after this fixture was instantiated,
|
|
making it safe for parallel test execution.
|
|
"""
|
|
|
|
def __init__(self, events_history: list):
|
|
self._history = events_history
|
|
# Track the starting index to only see new events during this test
|
|
self._start_index = len(events_history)
|
|
|
|
@property
|
|
def events(self) -> List[dict]:
|
|
"""Return events captured during this test (most recent first)."""
|
|
# Only return events added after fixture creation
|
|
new_events = list(self._history[self._start_index:])
|
|
return list(reversed(new_events))
|
|
|
|
def get_by_name(self, event_name: str) -> List[dict]:
|
|
"""Get all events with the given name."""
|
|
return [e for e in self.events if e['event'] == event_name]
|
|
|
|
def get_by_subject(self, subject: str) -> List[dict]:
|
|
"""Get all events with the given subject."""
|
|
return [e for e in self.events if e['subject'] == subject]
|
|
|
|
def assert_has_event(self, event_name: str, message: str = None) -> None:
|
|
"""Assert that an event with the given name exists."""
|
|
matching = self.get_by_name(event_name)
|
|
assert matching, f"No event found with name '{event_name}'. Events: {[e['event'] for e in self.events]}"
|
|
if message:
|
|
assert any(message in (e.get('message') or '') for e in matching), \
|
|
f"No '{event_name}' event contains message '{message}'"
|
|
|
|
def assert_no_event(self, event_name: str) -> None:
|
|
"""Assert that no event with the given name exists."""
|
|
matching = self.get_by_name(event_name)
|
|
assert not matching, f"Found unexpected event '{event_name}': {matching}"
|
|
|
|
def assert_events_contain(self, expected: List[dict]) -> None:
|
|
"""Assert that events contain all expected events (order matters, most recent first)."""
|
|
from wrolpi.test.common import assert_dict_contains
|
|
for exp, actual in zip_longest(expected, self.events):
|
|
assert actual is not None, f"Expected event {exp} but no more events found"
|
|
assert_dict_contains(actual, exp)
|
|
|
|
def assert_event_count(self, count: int) -> None:
|
|
"""Assert the total number of events."""
|
|
assert len(self.events) == count, f"Expected {count} events, got {len(self.events)}"
|
|
|
|
|
|
@pytest.fixture
|
|
def events_fixture(async_client) -> EventsFixture:
|
|
"""Fixture for testing events with rich assertions.
|
|
|
|
Only sees events created after this fixture is instantiated,
|
|
making it safe for parallel test execution.
|
|
"""
|
|
return EventsFixture(api_app.shared_ctx.events_history)
|
|
|
|
|
|
FLAGS_LOCK = multiprocessing.Lock()
|
|
|
|
|
|
@pytest.fixture()
|
|
def flags_lock():
|
|
"""Wait for exclusive access to flags before testing."""
|
|
try:
|
|
FLAGS_LOCK.acquire(timeout=60)
|
|
flags.TESTING_LOCK.set()
|
|
yield
|
|
finally:
|
|
flags.TESTING_LOCK.clear()
|
|
FLAGS_LOCK.release()
|
|
|
|
|
|
@pytest.fixture()
|
|
def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]:
|
|
names = ['one', 'two', 'three', 'four', 'five', 'six']
|
|
count = 1
|
|
|
|
async def factory(name: str = None) -> Tag:
|
|
if not name:
|
|
name = names.pop(0)
|
|
tag = await upsert_tag(test_session, name, f'#{str(count) * 6}')
|
|
return tag
|
|
|
|
return factory
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_files(test_session, test_directory):
|
|
def _(files: List[Dict]):
|
|
from wrolpi.files.models import FileGroup
|
|
for expected in files:
|
|
primary_path = test_directory / expected.pop('path')
|
|
try:
|
|
file: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
|
|
except sqlalchemy.orm.exc.NoResultFound as e:
|
|
raise ValueError(f'No FileGroup found with {primary_path=}') from e
|
|
|
|
for key, value in expected.items():
|
|
attr = getattr(file, key)
|
|
if attr != value:
|
|
raise AssertionError(f'{primary_path} {attr} != {value}')
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_file_groups(test_session, test_directory):
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
def _(file_groups: List[Dict], assert_count: bool = True):
|
|
from wrolpi.files.models import FileGroup
|
|
for expected in file_groups:
|
|
if 'primary_path' not in expected:
|
|
raise Exception('You must specify the primary path for this fixture!')
|
|
|
|
primary_path = str(test_directory / expected.pop('primary_path'))
|
|
try:
|
|
file_group: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
|
|
except Exception as e:
|
|
raise ValueError(f'No FileGroup found with {primary_path=}') from e
|
|
|
|
# Compare file dictionaries.
|
|
files = expected.pop('files', None)
|
|
if files:
|
|
for exp, actual in zip_longest(files, file_group.files):
|
|
assert_dict_contains(actual, exp)
|
|
|
|
for key, value in expected.items():
|
|
attr = getattr(file_group, key)
|
|
if isinstance(value, dict):
|
|
assert_dict_contains(attr, value)
|
|
elif attr != value:
|
|
raise AssertionError(f'{file_group}.{key} {attr=} != {value}')
|
|
|
|
if assert_count is True:
|
|
if (count := test_session.query(FileGroup).count()) != len(file_groups):
|
|
raise AssertionError(f'FileGroup count does not match {count} != {len(file_groups)}')
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_files_search(async_client):
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
async def _(search_str: str, expected: List[dict]):
|
|
content = json.dumps({'search_str': search_str})
|
|
request, response = await async_client.post('/api/files/search', content=content)
|
|
for file_group, exp in zip_longest(response.json['file_groups'], expected):
|
|
assert_dict_contains(file_group, exp)
|
|
|
|
assert len(response.json['file_groups']) == len(expected)
|
|
|
|
return _
|
|
|
|
|
|
SINGLEFILE_CONTENTS = '''<html><!--
|
|
Page saved with SingleFile
|
|
url: {url}
|
|
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
|
|
--><head><meta charset="utf-8">
|
|
|
|
<meta name="msapplication-TileColor" content="#2d89ef">
|
|
<meta name="theme-color" content="#000000">
|
|
|
|
{title}
|
|
|
|
<body>
|
|
some body contents
|
|
</body>
|
|
|
|
</html>
|
|
'''
|
|
|
|
|
|
@pytest.fixture
|
|
def singlefile_contents_factory():
|
|
def _(title: str = '', url: str = 'https://example.com') -> str:
|
|
contents = copy.deepcopy(SINGLEFILE_CONTENTS)
|
|
contents = contents.format(
|
|
title=f'<title>{title}</title>',
|
|
url=url,
|
|
)
|
|
return contents
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.Path:
|
|
archive = test_directory / 'archive'
|
|
archive.mkdir(exist_ok=True)
|
|
singlefile = archive / 'singlefile.html'
|
|
singlefile.write_text(singlefile_contents_factory())
|
|
return singlefile
|
|
|
|
|
|
@pytest.fixture
|
|
def image_bytes_factory():
|
|
def _(suffix: str = '.png') -> bytes:
|
|
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
|
|
Image.new('RGB', (1, 1), color='grey').save(fh)
|
|
fh.seek(0)
|
|
image = fh.read()
|
|
return image
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def insert_file_group(test_session, test_directory):
|
|
"""Inserts a FileGroup based on the provided paths. Does not verify if the files exist."""
|
|
|
|
def _(paths: List[pathlib.Path]):
|
|
# Store relative filenames in files (new schema)
|
|
files = [dict(path=i.name, mimetype='fake') for i in paths]
|
|
primary_path = paths[0]
|
|
from wrolpi.files.lib import split_path_stem_and_suffix
|
|
stem, _ = split_path_stem_and_suffix(primary_path)
|
|
params = dict(
|
|
directory=str(primary_path.parent),
|
|
primary_path=str(primary_path),
|
|
files=json.dumps(files),
|
|
stem=stem,
|
|
)
|
|
test_session.execute(sqlalchemy.text('INSERT INTO file_group '
|
|
'(indexed, directory, primary_path, files, stem) VALUES '
|
|
'(true, :directory, :primary_path, :files, :stem)'), params)
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def test_tags_config(test_directory):
|
|
from wrolpi.tags import test_tags_config
|
|
with test_tags_config():
|
|
(test_directory / 'config').mkdir(exist_ok=True)
|
|
config_path = test_directory / 'config/tags.yaml'
|
|
yield config_path
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_tags_config(test_tags_config):
|
|
def _(tags: dict = None):
|
|
from wrolpi.test.common import assert_dict_contains
|
|
with test_tags_config.open('rt') as fh:
|
|
contents = yaml.load(fh, Loader=yaml.Loader)
|
|
|
|
if tags:
|
|
for key, item in tags.items():
|
|
assert key in contents['tags']
|
|
assert_dict_contains(item, contents['tags'][key])
|
|
else:
|
|
# Expect no tags to be saved.
|
|
assert not contents['tags']
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def zip_file_factory(test_directory):
|
|
def _() -> bytes:
|
|
with tempfile.NamedTemporaryFile() as fh:
|
|
with zipfile.ZipFile(fh, 'w') as zip_file:
|
|
zip_file.write(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4')
|
|
|
|
fh.seek(0)
|
|
return fh.read()
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_directories(test_session, test_directory):
|
|
def _(expected: Sequence[str]):
|
|
directories = test_session.query(Directory).all()
|
|
names = {str(i.path.relative_to(test_directory)) for i in directories}
|
|
assert names == set(expected)
|
|
|
|
return _
|
|
|
|
|
|
FORM_PART = b'''-----------------------------sanic\r
|
|
Content-Disposition: form-data; name="{name}"\r
|
|
\r
|
|
{value}\r
|
|
\r
|
|
'''
|
|
|
|
FORM_FILENAME_PART = b'''-----------------------------sanic\r
|
|
Content-Disposition: form-data; name="{name}"; filename="{filename}"\r
|
|
Content-Type: application/octet-stream\r
|
|
\r
|
|
{value}\r
|
|
\r
|
|
'''
|
|
|
|
|
|
@pytest.fixture
|
|
def make_multipart_form():
|
|
def _(forms: List[dict]):
|
|
boundary = b'-----------------------------sanic\r\n'
|
|
new_forms = []
|
|
for form in forms:
|
|
name, value, filename = form['name'].encode(), form['value'], form.get('filename', '').encode()
|
|
value = value if isinstance(value, bytes) else str(value).encode()
|
|
part = FORM_PART
|
|
if filename:
|
|
part = FORM_FILENAME_PART
|
|
part.replace(b'{filename}', filename)
|
|
part = part.replace(b'{name}', name)
|
|
part = part.replace(b'{value}', value)
|
|
new_forms.append(part)
|
|
body = boundary + boundary.join(new_forms) + boundary
|
|
return body
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def simple_file_server(test_directory):
|
|
# Use a random port for testing
|
|
server_address = ('', 0) # 0 means pick any available port
|
|
|
|
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
|
|
def __init__(self, *args, **kwargs):
|
|
# Pass the directory to the handler
|
|
self.directory = kwargs['directory']
|
|
super().__init__(*args, **kwargs)
|
|
|
|
# Create the server
|
|
server = http.server.HTTPServer(
|
|
server_address,
|
|
# Hack to pass `test_directory` as `self.directory` to `SimpleHTTPRequestHandler`
|
|
lambda *args, **kwargs:
|
|
CustomHTTPRequestHandler(*args, directory=str(test_directory), **kwargs))
|
|
|
|
# Start the server in a background thread
|
|
server_thread = threading.Thread(target=server.serve_forever)
|
|
server_thread.daemon = True # So the server dies when the main thread dies
|
|
server_thread.start()
|
|
|
|
try:
|
|
yield server
|
|
finally:
|
|
# Cleanup
|
|
server.shutdown()
|
|
server.server_close()
|
|
server_thread.join()
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_downloader_download_file():
|
|
"""Mock the Downloader.download_file method."""
|
|
contents = None
|
|
|
|
async def _download_file(self, id_, url, destination, **kwargs):
|
|
# **kwargs absorbs ctx= and any future optional args added to Downloader.download_file.
|
|
name = url.split('/')[-1]
|
|
output_path = destination / name
|
|
output_path.write_bytes(contents or b'')
|
|
return output_path
|
|
|
|
with mock.patch('wrolpi.downloader.Downloader.download_file', _download_file):
|
|
def set_contents(contents_):
|
|
nonlocal contents
|
|
contents = contents_
|
|
|
|
yield set_contents
|
|
|
|
|
|
async def await_file_worker(timeout: int = 30):
|
|
"""Wait for FileWorker to complete all pending tasks. Testing only."""
|
|
from wrolpi.files.worker import file_worker
|
|
|
|
start = time.time()
|
|
while time.time() - start < timeout:
|
|
file_worker.transfer_queue()
|
|
if file_worker.private_queue.qsize() > 0:
|
|
await file_worker.process_queue()
|
|
elif file_worker.public_queue.qsize() == 0:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
else:
|
|
raise RuntimeError('Timed out waiting for file_worker')
|
|
|
|
|
|
@pytest.fixture
|
|
def refresh_files(async_client):
|
|
"""Test helper that queues and waits for file refresh."""
|
|
|
|
async def _refresh(paths=None):
|
|
from wrolpi.common import get_media_directory
|
|
from wrolpi.files.worker import file_worker
|
|
|
|
paths = paths or [get_media_directory()]
|
|
file_worker.queue_refresh(paths)
|
|
await await_file_worker()
|
|
|
|
return _refresh
|