mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
All three bugs confirmed; the two suggestions taken. [bug] `video_modeler` awaited ffprobe -- a subprocess per video, up to 20 per batch -- inside its `commit=True` session. With IMMEDIATE as the default that holds the write lock for the whole batch, which is the same starvation this branch is curing in `video_cleanup`. (It was already partly true before: the batch's first `session.flush([video])` took the lock ahead of the remaining ffprobes.) The batch is now read, ffprobed with no transaction open, then written. `get_or_create_ffprobe_json` in modules/videos/common.py is the path-only entry point that makes that possible; `Video.get_ffprobe_json` delegates to it and keeps appending the cache file to its FileGroup. `validate()` -- and the poster generation it can trigger -- is still inside the write transaction, and now says so. [bug] `test_download_dispatched_outside_immediate_transaction` probed `getattr(_immediate_txn, 'active', False)`, which a ContextVar never has, so it passed unconditionally. It now probes the actual write lock from a competing connection under `production_like_sessions`. Mutation-checked: moving the dispatch back inside the claim transaction fails it. [bug] `asyncio.create_task` snapshots the current Context, so a flag held for a write session's lifetime was inherited by every task spawned inside it -- and the parent's `reset()` cannot reach a child's copy, leaving that task issuing BEGIN IMMEDIATE for its *read* sessions. `create_downloads` spawns `dispatch_downloads` exactly that way, so every user-created download hit it. The flag now lives only across the session's opening `connection()` call, which also means the write lock is taken when the session opens rather than at the caller's first statement. [suggestion] `renew_recurring_downloads` scanned every recurring Download under the write lock, including the usual cycle where nothing is due. It reads first now and takes the lock only for the rows that change, re-checking status since the dispatcher may have claimed one in between. [nit] Parenthesized the directory OR in the claim join. `probe_write_lock_is_held` moved to conftest alongside `write_lock_held_briefly`; three suites use it now. Suite: 1712 passed, 5 skipped. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
1230 lines
42 KiB
Python
1230 lines
42 KiB
Python
"""
|
|
Fixtures for Pytest tests.
|
|
"""
|
|
import asyncio
|
|
import contextlib
|
|
import copy
|
|
import http.server
|
|
import json
|
|
import logging
|
|
import multiprocessing
|
|
import pathlib
|
|
import shutil
|
|
import sqlite3
|
|
import tempfile
|
|
import threading
|
|
import time
|
|
import zipfile
|
|
from abc import ABC
|
|
from datetime import datetime
|
|
from itertools import zip_longest
|
|
from pathlib import Path
|
|
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional, Any, Generator, AsyncGenerator
|
|
from typing import Tuple, Set
|
|
from unittest import mock
|
|
from unittest.mock import MagicMock, AsyncMock
|
|
from uuid import uuid1, uuid4
|
|
|
|
import pytest
|
|
import sqlalchemy
|
|
import yaml
|
|
from PIL import Image
|
|
from sanic_testing.testing import SanicASGITestClient
|
|
from sqlalchemy.engine import Engine, create_engine
|
|
from sqlalchemy.orm import Session, sessionmaker
|
|
from sqlalchemy.pool import NullPool
|
|
|
|
# Import root api so blueprints are attached.
|
|
import wrolpi.root_api # noqa
|
|
from wrolpi import flags
|
|
from wrolpi.api_utils import api_app
|
|
from wrolpi.cmd import CommandResult, run_command
|
|
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL, \
|
|
get_wrolpi_config
|
|
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
|
|
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
|
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
|
|
from wrolpi.dates import set_test_now
|
|
from wrolpi.db import create_wrolpi_engine
|
|
from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \
|
|
downloads_manager_config_context, download_cache_config_context
|
|
from wrolpi.errors import UnrecoverableDownloadError
|
|
from wrolpi.files.models import Directory, FileGroup
|
|
from wrolpi.switches import await_switches as await_switches_
|
|
from wrolpi.tags import Tag, upsert_tag
|
|
from wrolpi.vars import PROJECT_DIR
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
|
|
def test_db() -> Tuple[Engine, Session]:
|
|
"""Create a SQLAlchemy engine/session on a SQLite DB inside the (temporary) media directory.
|
|
|
|
The DB file is at the production location (`<media>/config/wrolpi.db`), so each test's
|
|
TemporaryDirectory provides isolation (also under pytest-xdist) and removes the DB (and its
|
|
WAL sidecars) on teardown."""
|
|
from wrolpi.common import get_media_directory, is_tempfile
|
|
from wrolpi.schema_ddl import install_raw_ddl
|
|
|
|
db_file = get_media_directory() / 'config' / 'wrolpi.db'
|
|
if not is_tempfile(db_file):
|
|
raise ValueError(f'Refusing to test on a non-temporary database! {db_file}')
|
|
db_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
test_engine = create_wrolpi_engine(db_file)
|
|
# Create all tables. No need to check if they exist because this is a test DB.
|
|
Base.metadata.create_all(test_engine, checkfirst=False)
|
|
# Triggers and FTS5 tables, exactly as the alembic baseline installs them.
|
|
with test_engine.connect() as conn:
|
|
install_raw_ddl(conn)
|
|
session = sessionmaker(bind=test_engine)()
|
|
return test_engine, session
|
|
|
|
|
|
@pytest.fixture()
|
|
def test_session(test_directory) -> Generator[Session, Any, None]:
|
|
"""Pytest Fixture to get a test database session."""
|
|
test_engine, session = test_db()
|
|
|
|
def fake_get_db_session():
|
|
"""Get the testing db"""
|
|
return test_engine, session
|
|
|
|
try:
|
|
with mock.patch('wrolpi.db._get_db_session', fake_get_db_session):
|
|
yield session
|
|
finally:
|
|
session.rollback()
|
|
session.close()
|
|
test_engine.dispose()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def test_debug_logger(request):
|
|
"""Tests default to DEBUG logging. Use pytest -vvv for TRACE level."""
|
|
level = logging.DEBUG
|
|
if request.config.option.verbose >= 3:
|
|
# -vvv
|
|
level = TRACE_LEVEL
|
|
|
|
with log_level_context(level):
|
|
yield
|
|
|
|
|
|
@pytest.fixture
|
|
def clear_background_tasks(request):
|
|
"""Clear BACKGROUND_TASKS before each test and fail if orphaned tasks remain after.
|
|
|
|
Tests must properly await all background tasks they create. Orphaned tasks indicate
|
|
a test is not properly cleaning up, which can cause cascading failures in other tests.
|
|
"""
|
|
# Clear before test - cancel any orphaned tasks from previous tests
|
|
if BACKGROUND_TASKS:
|
|
logger.warning(f'clear_background_tasks: clearing {len(BACKGROUND_TASKS)} orphaned tasks BEFORE test')
|
|
for task in list(BACKGROUND_TASKS):
|
|
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
|
|
logger.warning(f' orphaned task from previous test: {task_name}')
|
|
if not task.done():
|
|
task.cancel()
|
|
BACKGROUND_TASKS.clear()
|
|
|
|
yield
|
|
|
|
# After test - fail if orphaned tasks remain
|
|
if BACKGROUND_TASKS:
|
|
orphaned_tasks = []
|
|
for task in list(BACKGROUND_TASKS):
|
|
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
|
|
orphaned_tasks.append(f'{task_name} (done={task.done()})')
|
|
if not task.done():
|
|
task.cancel()
|
|
BACKGROUND_TASKS.clear()
|
|
|
|
# Fail the test
|
|
pytest.fail(
|
|
f'Test left {len(orphaned_tasks)} orphaned background task(s): {", ".join(orphaned_tasks)}. '
|
|
f'Tests must await all background tasks they create using await_background_tasks or '
|
|
f'the async_client fixture.'
|
|
)
|
|
|
|
|
|
@pytest.fixture
|
|
def test_directory() -> Generator[Path, Any, None]:
|
|
"""
|
|
Overwrite the media directory with a temporary directory.
|
|
"""
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
tmp_path = pathlib.Path(tmp_dir).resolve()
|
|
tmp_path.chmod(0o40755)
|
|
assert tmp_path.is_dir()
|
|
set_test_media_directory(tmp_path)
|
|
yield tmp_path
|
|
|
|
|
|
@pytest.fixture
|
|
def test_wrolpi_config(test_directory) -> Generator[Path, Any, None]:
|
|
"""
|
|
Create a test config based off the example config.
|
|
"""
|
|
config_path = test_directory / 'config/wrolpi.yaml'
|
|
set_test_config(True)
|
|
yield config_path
|
|
set_test_config(False)
|
|
|
|
|
|
ROUTES_ATTACHED = False
|
|
|
|
|
|
@api_app.on_response
|
|
async def background_task_listener(request, response):
|
|
"""Wait for all background tasks to finish before returning API response while testing."""
|
|
await await_switches_()
|
|
|
|
|
|
class AutoAwaitTestClient:
|
|
"""
|
|
A wrapper around SanicASGITestClient that automatically awaits background
|
|
tasks and switches after modifying HTTP requests (POST, PUT, PATCH, DELETE).
|
|
|
|
This eliminates the need to manually call await_switches() and
|
|
await_background_tasks() after each modifying request in tests.
|
|
"""
|
|
|
|
def __init__(self, client: SanicASGITestClient, session: Session):
|
|
self._client = client
|
|
self._session = session
|
|
|
|
async def _await_and_expire(self):
|
|
"""Await all background tasks and switches, then expire session objects."""
|
|
await await_switches_()
|
|
await await_background_tasks_()
|
|
self._session.expire_all()
|
|
|
|
async def post(self, *args, **kwargs):
|
|
"""POST request with auto-await."""
|
|
result = await self._client.post(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def put(self, *args, **kwargs):
|
|
"""PUT request with auto-await."""
|
|
result = await self._client.put(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def patch(self, *args, **kwargs):
|
|
"""PATCH request with auto-await."""
|
|
result = await self._client.patch(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def delete(self, *args, **kwargs):
|
|
"""DELETE request with auto-await."""
|
|
result = await self._client.delete(*args, **kwargs)
|
|
await self._await_and_expire()
|
|
return result
|
|
|
|
async def get(self, *args, **kwargs):
|
|
"""GET request (no auto-await)."""
|
|
return await self._client.get(*args, **kwargs)
|
|
|
|
async def head(self, *args, **kwargs):
|
|
"""HEAD request (no auto-await)."""
|
|
return await self._client.head(*args, **kwargs)
|
|
|
|
async def options(self, *args, **kwargs):
|
|
"""OPTIONS request (no auto-await)."""
|
|
return await self._client.options(*args, **kwargs)
|
|
|
|
async def websocket(self, *args, **kwargs):
|
|
"""WebSocket connection (pass-through)."""
|
|
return await self._client.websocket(*args, **kwargs)
|
|
|
|
@property
|
|
def sanic_app(self):
|
|
"""Access to the underlying Sanic app."""
|
|
return self._client.sanic_app
|
|
|
|
|
|
@pytest.fixture
|
|
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[
|
|
AutoAwaitTestClient, Any]:
|
|
"""Get an Async Sanic Test Client with all default routes attached.
|
|
|
|
Depends on test_session to ensure the database mock is in place for session middleware.
|
|
Depends on clear_background_tasks to detect orphaned background tasks.
|
|
|
|
This client automatically awaits switches and background tasks after
|
|
POST, PUT, PATCH, and DELETE requests, and expires session objects.
|
|
"""
|
|
api_app.signalize()
|
|
attach_shared_contexts(api_app)
|
|
initialize_configs_contexts(api_app)
|
|
|
|
client = SanicASGITestClient(api_app)
|
|
wrapped_client = AutoAwaitTestClient(client, test_session)
|
|
|
|
try:
|
|
yield wrapped_client
|
|
finally:
|
|
api_app.stop()
|
|
|
|
|
|
@pytest.fixture
|
|
def test_download_manager_config(async_client, test_directory) -> Generator[Path, Any, None]:
|
|
with downloads_manager_config_context():
|
|
(test_directory / 'config').mkdir(exist_ok=True)
|
|
config_path = test_directory / 'config/download_manager.yaml'
|
|
yield config_path
|
|
|
|
|
|
@pytest.fixture
|
|
def test_download_cache_config(async_client, test_directory) -> Generator[Path, Any, None]:
|
|
with download_cache_config_context():
|
|
(test_directory / 'config').mkdir(exist_ok=True)
|
|
config_path = test_directory / 'config/download_cache.yaml'
|
|
yield config_path
|
|
|
|
|
|
@pytest.fixture
|
|
async def await_switches(async_client):
|
|
"""Returns the `switches.await_switches` function, but primes the Sanic App to actually handle signals."""
|
|
await async_client.get('/api')
|
|
return await_switches_
|
|
|
|
|
|
@pytest.fixture
|
|
async def await_background_tasks(async_client, test_session):
|
|
"""Returns a function that awaits background tasks and expires session objects.
|
|
|
|
After background tasks complete, session objects may be stale, so we automatically
|
|
expire them to force a refresh on next access.
|
|
|
|
Args:
|
|
timeout: Optional timeout in seconds (default 15s from await_background_tasks_)
|
|
"""
|
|
await async_client.get('/api')
|
|
|
|
async def _await_and_expire(timeout: int = 10):
|
|
await await_background_tasks_(timeout=timeout)
|
|
test_session.expire_all()
|
|
|
|
return _await_and_expire
|
|
|
|
|
|
@pytest.fixture
|
|
async def test_download_manager(
|
|
async_client,
|
|
test_session, # session is required because downloads can start without the test DB in place.
|
|
test_download_manager_config,
|
|
) -> AsyncGenerator[DownloadManager, Any]:
|
|
with timer('test_download_manager'):
|
|
# Needed to use signals in
|
|
manager = DownloadManager()
|
|
await manager.enable()
|
|
|
|
# Disable per-domain rate limiting during tests to avoid timeouts.
|
|
get_wrolpi_config().download_wait = 0
|
|
|
|
yield manager
|
|
|
|
manager.stop()
|
|
|
|
|
|
@pytest.fixture
|
|
def fake_now() -> Generator[Callable[..., Any], Any, None]:
|
|
try:
|
|
set_test_now(datetime(2000, 1, 1))
|
|
yield set_test_now
|
|
finally:
|
|
# reset now() to its original functionality.
|
|
set_test_now(None) # noqa
|
|
|
|
|
|
@pytest.fixture
|
|
def successful_download() -> DownloadResult:
|
|
return DownloadResult(success=True)
|
|
|
|
|
|
@pytest.fixture
|
|
def failed_download() -> DownloadResult:
|
|
return DownloadResult(error='wrolpi.conftest failed_download error', success=False)
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_download_urls(test_session) -> Callable[[Set[str]], None]:
|
|
def asserter(expected_urls: Set[str]):
|
|
downloads = test_session.query(Download).all()
|
|
urls = {i.url for i in downloads}
|
|
if urls != set(expected_urls):
|
|
raise AssertionError(f'Download URLs do not match: {urls} != {expected_urls}')
|
|
|
|
return asserter
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
def asserter(expected: List[Dict]):
|
|
downloads = test_session.query(Download).order_by(Download.url)
|
|
for download_, expected_ in zip_longest(downloads, expected):
|
|
assert_dict_contains(download_.__json__(), expected_)
|
|
if (count := test_session.query(Download).count()) != len(expected):
|
|
raise AssertionError(f'Download count does not match: {count} != {len(expected)}')
|
|
|
|
return asserter
|
|
|
|
|
|
def make_test_ctx(*, is_cancelled=None, can_download=None,
|
|
outside_download_window=None, run_command_=None,
|
|
download_timeout=None,
|
|
report_progress=None, clear_progress=None) -> DownloadContext:
|
|
"""Build a DownloadContext suitable for downloader unit tests.
|
|
|
|
Defaults: never-cancelled, can_download True, no window restriction, real run_command,
|
|
no global timeout, progress reporting is a no-op. Override any callable to simulate
|
|
cancellation, throttling, fake subprocess output, or to record progress events.
|
|
|
|
Lives in wrolpi/conftest.py so any test (including primitive tests of process_runner /
|
|
cancel_wrapper) can construct a ctx without crossing module boundaries.
|
|
"""
|
|
return DownloadContext(
|
|
is_cancelled=is_cancelled or (lambda: False),
|
|
can_download=can_download or (lambda: True),
|
|
outside_download_window=outside_download_window or (lambda: False),
|
|
run_command=run_command_ or run_command,
|
|
download_timeout=download_timeout,
|
|
report_progress=report_progress or (lambda p: None),
|
|
clear_progress=clear_progress or (lambda: None),
|
|
)
|
|
|
|
|
|
def probe_write_lock_is_held(db_file: str) -> bool:
|
|
"""Return True if a competing connection cannot immediately take the write lock.
|
|
|
|
Uses a zero busy-timeout so the probe fails instantly if another connection holds a
|
|
RESERVED (write) lock, rather than waiting. In WAL mode a plain reader holds no such
|
|
lock, so this only returns True when a writer transaction is actually in progress.
|
|
"""
|
|
probe = sqlite3.connect(db_file, timeout=0)
|
|
try:
|
|
probe.execute('PRAGMA busy_timeout=0')
|
|
try:
|
|
probe.execute('BEGIN IMMEDIATE')
|
|
except sqlite3.OperationalError as e:
|
|
if 'database is locked' in str(e):
|
|
return True
|
|
raise
|
|
probe.execute('ROLLBACK')
|
|
return False
|
|
finally:
|
|
probe.close()
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def write_lock_held_briefly(db_file: str, seconds: float = 1.0):
|
|
"""Hold the SQLite write lock on `db_file` for `seconds`, from another connection.
|
|
|
|
Simulates the concurrent writer (refresh, download, config save) that a real WROLPi runs
|
|
alongside whatever the test is doing. The lock is released while the test's work is still in
|
|
flight, so a correctly-written transaction waits (busy_timeout) and then succeeds; a deferred
|
|
one that upgrades its lock mid-transaction fails instantly instead.
|
|
|
|
Pair with `production_like_sessions` — the shared `test_session` uses one connection, which
|
|
cannot contend with itself.
|
|
"""
|
|
holder = sqlite3.connect(db_file, timeout=30, check_same_thread=False)
|
|
holder.isolation_level = None
|
|
holder.execute('PRAGMA busy_timeout=30000')
|
|
holder.execute('BEGIN IMMEDIATE')
|
|
|
|
def release():
|
|
time.sleep(seconds)
|
|
holder.execute('ROLLBACK')
|
|
holder.close()
|
|
|
|
thread = threading.Thread(target=release, daemon=True)
|
|
thread.start()
|
|
try:
|
|
yield
|
|
finally:
|
|
thread.join(timeout=30)
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None]:
|
|
"""Patch get_db_session to hand out a *fresh* Session per call, like production.
|
|
|
|
The `test_session` fixture reuses one shared Session for every `get_db_session` call, so an object loaded in
|
|
one `with` block stays attached for the next and exceptions never roll back. That convenience masks
|
|
detached-object and transaction-rollback bugs that only bite production's per-call sessions. Use this to
|
|
exercise that behavior: the yielded sessionmaker creates independent Sessions so a test can verify committed
|
|
DB state the way a separate worker would see it.
|
|
|
|
A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB
|
|
is dropped during fixture teardown.
|
|
"""
|
|
engine = create_wrolpi_engine(test_session.get_bind().url)
|
|
maker = sessionmaker(bind=engine)
|
|
sessions = []
|
|
|
|
def factory():
|
|
session = maker()
|
|
sessions.append(session)
|
|
return engine, session
|
|
|
|
try:
|
|
with mock.patch('wrolpi.db._get_db_session', factory):
|
|
yield maker
|
|
finally:
|
|
for session in sessions:
|
|
try:
|
|
session.close()
|
|
except Exception as e:
|
|
# Best-effort: engine.dispose() below releases the connections regardless, so a failed
|
|
# close() is inconsequential. Log at debug so it surfaces under -vvv without noising CI.
|
|
logger.debug(f'production_like_sessions: failed to close a session: {e}')
|
|
engine.dispose()
|
|
|
|
|
|
@pytest.fixture
|
|
def test_downloader(test_download_manager) -> Downloader:
|
|
class TestDownloader(Downloader, ABC):
|
|
"""A testing Downloader.
|
|
|
|
prepare_download returns a non-None sentinel so the manager's phase-split dispatch
|
|
routes through this subclass. execute_download is an AsyncMock — tests wire its
|
|
side_effect via the set_test_* methods. Returning a DownloadResult from
|
|
execute_download tells the dispatch to skip finalize (see signal_download_download).
|
|
"""
|
|
name = 'test_downloader'
|
|
|
|
def __repr__(self):
|
|
return '<TESTING Downloader>'
|
|
|
|
execute_download = AsyncMock()
|
|
|
|
def prepare_download(self, session, download):
|
|
return object() # sentinel; activates the phase-split path
|
|
|
|
def finalize_download(self, session, download, executed):
|
|
# execute_download returns a DownloadResult directly, so the dispatch
|
|
# short-circuits before reaching finalize. Provided here only to satisfy
|
|
# the abstract contract.
|
|
return DownloadResult(success=True)
|
|
|
|
def set_test_success(self):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
return DownloadResult(success=True)
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
def set_test_failure(self):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
return DownloadResult(success=False)
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
raise exception
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
def set_test_unrecoverable_exception(self):
|
|
async def _(*a, **kwargs):
|
|
# Sleep so download happens after testing is waiting.
|
|
await asyncio.sleep(1)
|
|
raise UnrecoverableDownloadError()
|
|
|
|
self.execute_download.side_effect = _
|
|
|
|
test_downloader = TestDownloader()
|
|
# Default to successful download.
|
|
test_downloader.set_test_success()
|
|
test_download_manager.register_downloader(test_downloader)
|
|
|
|
return test_downloader
|
|
|
|
|
|
@pytest.fixture
|
|
def video_file(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
|
destination = test_directory / f'video-{uuid4()}.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
|
|
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def corrupted_video_file(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of a corrupted video file in the `test_directory`."""
|
|
destination = test_directory / f'corrupted-{uuid4()}.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/corrupted.mp4', destination)
|
|
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]:
|
|
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
|
|
|
|
def _(path: pathlib.Path = None) -> pathlib.Path:
|
|
destination = path or test_directory / f'{uuid4()}.mp4'
|
|
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
|
|
|
|
return destination
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def video_bytes() -> bytes:
|
|
return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()
|
|
|
|
|
|
@pytest.fixture
|
|
def image_file(test_directory) -> Generator[Path, Any, None]:
|
|
"""Create a small image file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.jpeg'
|
|
Image.new('RGB', (25, 25), color='grey').save(destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def vtt_file1(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example1 VTT file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.en.vtt'
|
|
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def vtt_file2(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example2 VTT file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.en.vtt'
|
|
shutil.copy(PROJECT_DIR / 'test/example2.en.vtt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def srt_text() -> str:
|
|
return (PROJECT_DIR / 'test/example3.en.srt').read_text()
|
|
|
|
|
|
@pytest.fixture
|
|
def srt_file3(test_directory) -> Generator[Path, Any, None]:
|
|
"""Return a copy of the example3 SRT file in the `test_directory`."""
|
|
destination = test_directory / f'{uuid4()}.en.srt'
|
|
shutil.copy(PROJECT_DIR / 'test/example3.en.srt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_pdf(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'pdf example.pdf'
|
|
shutil.copy(PROJECT_DIR / 'test/pdf example.pdf', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_epub(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'ebook example.epub'
|
|
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_mobi(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'ebook example.mobi'
|
|
shutil.copy(PROJECT_DIR / 'test/ebook example.mobi', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_doc(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'example word.doc'
|
|
shutil.copy(PROJECT_DIR / 'test/example word.doc', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_docx(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'example word.docx'
|
|
shutil.copy(PROJECT_DIR / 'test/example word.docx', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_odt(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'test_book_odt.odt'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/test_book_odt.odt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbz(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_sample.cbz'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_sample.cbz', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbz_metadata(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_metadata.cbz'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_metadata.cbz', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbt(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_cbt.cbt'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbt.cbt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbr(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_cbr.cbr'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbr.cbr', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cb7(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_cb7.cb7'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cb7.cb7', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbz_dir(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_dir_cbz.cbz'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbz.cbz', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cbt_dir(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_dir_cbt.cbt'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbt.cbt', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def example_cb7_dir(test_directory) -> Generator[Path, Any, None]:
|
|
destination = test_directory / 'bobby_make_believe_dir_cb7.cb7'
|
|
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cb7.cb7', destination)
|
|
yield destination
|
|
|
|
|
|
@pytest.fixture
|
|
def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
|
|
"""
|
|
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
|
|
locations. If a dict is provided, files will be created containing the value of the dict item.
|
|
"""
|
|
|
|
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) \
|
|
-> List[pathlib.Path]:
|
|
# Note: session is optional here for backwards compatibility - file_groups=True requires session
|
|
files = []
|
|
|
|
@iterify(list)
|
|
def touch_paths(paths_):
|
|
for name in paths_:
|
|
path = test_directory / name
|
|
if name.endswith('/'):
|
|
path.mkdir(parents=True, exist_ok=True)
|
|
else:
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
path.touch()
|
|
yield path
|
|
|
|
if isinstance(paths, list):
|
|
files = touch_paths(paths)
|
|
elif isinstance(paths, Dict):
|
|
files = touch_paths(paths.keys())
|
|
for path, content in paths.items():
|
|
path = test_directory / path
|
|
if isinstance(content, str):
|
|
path.write_text(content)
|
|
elif isinstance(content, bytes):
|
|
path.write_bytes(content)
|
|
|
|
if file_groups:
|
|
for idx, path in enumerate(files):
|
|
fg = FileGroup.from_paths(session, path)
|
|
fg.do_index()
|
|
files[idx] = fg
|
|
return sorted(files) if isinstance(paths, dict) else files
|
|
|
|
return create_files
|
|
|
|
|
|
async def set_wrol_mode(enabled: bool):
|
|
if enabled:
|
|
enable_wrol_mode()
|
|
else:
|
|
await disable_wrol_mode()
|
|
|
|
|
|
@pytest.fixture
|
|
def wrol_mode_fixture(test_wrolpi_config, test_download_manager) -> Callable[[bool], Coroutine]:
|
|
return set_wrol_mode
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_create_subprocess_shell() -> Callable:
|
|
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
|
|
async def communicate(*a, **kw):
|
|
if communicate_side_effect:
|
|
return communicate_side_effect()
|
|
return communicate_return or (None, None)
|
|
|
|
async def create_subprocess_shell(*a, **kw):
|
|
proc = MagicMock()
|
|
proc.communicate = communicate
|
|
proc.returncode = return_code if return_code is not None else 0
|
|
return proc
|
|
|
|
return create_subprocess_shell
|
|
|
|
return mocker
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_run_command():
|
|
"""Mock wrolpi.cmd's `run_command` using `TESTING_RUN_COMMAND_RESULT`."""
|
|
default = CommandResult(
|
|
return_code=0,
|
|
cancelled=False,
|
|
stdout=b'mock_run_command fixture stdout',
|
|
stderr=b'mock_run_command fixture stderr',
|
|
elapsed=0,
|
|
pid=123,
|
|
)
|
|
|
|
with mock.patch('wrolpi.cmd.TESTING_RUN_COMMAND_RESULT') as mock_TESTING_RUN_COMMAND_RESULT:
|
|
mock_TESTING_RUN_COMMAND_RESULT.return_value = default
|
|
|
|
def do_mock(new_result):
|
|
mock_TESTING_RUN_COMMAND_RESULT.return_value = new_result
|
|
|
|
yield do_mock
|
|
|
|
|
|
@pytest.fixture
|
|
def run_command_bad_result() -> CommandResult:
|
|
return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123)
|
|
|
|
|
|
class EventsFixture:
|
|
"""Fixture for testing events with rich assertions.
|
|
|
|
Only returns events created after this fixture was instantiated,
|
|
making it safe for parallel test execution.
|
|
"""
|
|
|
|
def __init__(self, events_history: list):
|
|
self._history = events_history
|
|
# Track the starting index to only see new events during this test
|
|
self._start_index = len(events_history)
|
|
|
|
@property
|
|
def events(self) -> List[dict]:
|
|
"""Return events captured during this test (most recent first)."""
|
|
# Only return events added after fixture creation
|
|
new_events = list(self._history[self._start_index:])
|
|
return list(reversed(new_events))
|
|
|
|
def get_by_name(self, event_name: str) -> List[dict]:
|
|
"""Get all events with the given name."""
|
|
return [e for e in self.events if e['event'] == event_name]
|
|
|
|
def get_by_subject(self, subject: str) -> List[dict]:
|
|
"""Get all events with the given subject."""
|
|
return [e for e in self.events if e['subject'] == subject]
|
|
|
|
def assert_has_event(self, event_name: str, message: str = None) -> None:
|
|
"""Assert that an event with the given name exists."""
|
|
matching = self.get_by_name(event_name)
|
|
assert matching, f"No event found with name '{event_name}'. Events: {[e['event'] for e in self.events]}"
|
|
if message:
|
|
assert any(message in (e.get('message') or '') for e in matching), \
|
|
f"No '{event_name}' event contains message '{message}'"
|
|
|
|
def assert_no_event(self, event_name: str) -> None:
|
|
"""Assert that no event with the given name exists."""
|
|
matching = self.get_by_name(event_name)
|
|
assert not matching, f"Found unexpected event '{event_name}': {matching}"
|
|
|
|
def assert_events_contain(self, expected: List[dict]) -> None:
|
|
"""Assert that events contain all expected events (order matters, most recent first)."""
|
|
from wrolpi.test.common import assert_dict_contains
|
|
for exp, actual in zip_longest(expected, self.events):
|
|
assert actual is not None, f"Expected event {exp} but no more events found"
|
|
assert_dict_contains(actual, exp)
|
|
|
|
def assert_event_count(self, count: int) -> None:
|
|
"""Assert the total number of events."""
|
|
assert len(self.events) == count, f"Expected {count} events, got {len(self.events)}"
|
|
|
|
|
|
@pytest.fixture
|
|
def events_fixture(async_client) -> EventsFixture:
|
|
"""Fixture for testing events with rich assertions.
|
|
|
|
Only sees events created after this fixture is instantiated,
|
|
making it safe for parallel test execution.
|
|
"""
|
|
return EventsFixture(api_app.shared_ctx.events_history)
|
|
|
|
|
|
FLAGS_LOCK = multiprocessing.Lock()
|
|
|
|
|
|
@pytest.fixture()
|
|
def flags_lock():
|
|
"""Wait for exclusive access to flags before testing."""
|
|
try:
|
|
FLAGS_LOCK.acquire(timeout=60)
|
|
flags.TESTING_LOCK.set()
|
|
yield
|
|
finally:
|
|
flags.TESTING_LOCK.clear()
|
|
FLAGS_LOCK.release()
|
|
|
|
|
|
@pytest.fixture()
|
|
def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]:
|
|
names = ['one', 'two', 'three', 'four', 'five', 'six']
|
|
count = 1
|
|
|
|
async def factory(name: str = None) -> Tag:
|
|
if not name:
|
|
name = names.pop(0)
|
|
tag = await upsert_tag(test_session, name, f'#{str(count) * 6}')
|
|
return tag
|
|
|
|
return factory
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_files(test_session, test_directory):
|
|
def _(files: List[Dict]):
|
|
from wrolpi.files.models import FileGroup
|
|
for expected in files:
|
|
primary_path = test_directory / expected.pop('path')
|
|
try:
|
|
file: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
|
|
except sqlalchemy.orm.exc.NoResultFound as e:
|
|
raise ValueError(f'No FileGroup found with {primary_path=}') from e
|
|
|
|
for key, value in expected.items():
|
|
attr = getattr(file, key)
|
|
if attr != value:
|
|
raise AssertionError(f'{primary_path} {attr} != {value}')
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_file_groups(test_session, test_directory):
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
def _(file_groups: List[Dict], assert_count: bool = True):
|
|
from wrolpi.files.models import FileGroup
|
|
for expected in file_groups:
|
|
if 'primary_path' not in expected:
|
|
raise Exception('You must specify the primary path for this fixture!')
|
|
|
|
primary_path = str(test_directory / expected.pop('primary_path'))
|
|
try:
|
|
file_group: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
|
|
except Exception as e:
|
|
raise ValueError(f'No FileGroup found with {primary_path=}') from e
|
|
|
|
# Compare file dictionaries.
|
|
files = expected.pop('files', None)
|
|
if files:
|
|
for exp, actual in zip_longest(files, file_group.files):
|
|
assert_dict_contains(actual, exp)
|
|
|
|
for key, value in expected.items():
|
|
attr = getattr(file_group, key)
|
|
if isinstance(value, dict):
|
|
assert_dict_contains(attr, value)
|
|
elif attr != value:
|
|
raise AssertionError(f'{file_group}.{key} {attr=} != {value}')
|
|
|
|
if assert_count is True:
|
|
if (count := test_session.query(FileGroup).count()) != len(file_groups):
|
|
raise AssertionError(f'FileGroup count does not match {count} != {len(file_groups)}')
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_files_search(async_client):
|
|
from wrolpi.test.common import assert_dict_contains
|
|
|
|
async def _(search_str: str, expected: List[dict]):
|
|
content = json.dumps({'search_str': search_str})
|
|
request, response = await async_client.post('/api/files/search', content=content)
|
|
for file_group, exp in zip_longest(response.json['file_groups'], expected):
|
|
assert_dict_contains(file_group, exp)
|
|
|
|
assert len(response.json['file_groups']) == len(expected)
|
|
|
|
return _
|
|
|
|
|
|
SINGLEFILE_CONTENTS = '''<html><!--
|
|
Page saved with SingleFile
|
|
url: {url}
|
|
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
|
|
--><head><meta charset="utf-8">
|
|
|
|
<meta name="msapplication-TileColor" content="#2d89ef">
|
|
<meta name="theme-color" content="#000000">
|
|
|
|
{title}
|
|
|
|
<body>
|
|
some body contents
|
|
</body>
|
|
|
|
</html>
|
|
'''
|
|
|
|
|
|
@pytest.fixture
|
|
def singlefile_contents_factory():
|
|
def _(title: str = '', url: str = 'https://example.com') -> str:
|
|
contents = copy.deepcopy(SINGLEFILE_CONTENTS)
|
|
contents = contents.format(
|
|
title=f'<title>{title}</title>',
|
|
url=url,
|
|
)
|
|
return contents
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.Path:
|
|
archive = test_directory / 'archive'
|
|
archive.mkdir(exist_ok=True)
|
|
singlefile = archive / 'singlefile.html'
|
|
singlefile.write_text(singlefile_contents_factory())
|
|
return singlefile
|
|
|
|
|
|
@pytest.fixture
|
|
def image_bytes_factory():
|
|
def _(suffix: str = '.png') -> bytes:
|
|
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
|
|
Image.new('RGB', (1, 1), color='grey').save(fh)
|
|
fh.seek(0)
|
|
image = fh.read()
|
|
return image
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def insert_file_group(test_session, test_directory):
|
|
"""Inserts a FileGroup based on the provided paths. Does not verify if the files exist."""
|
|
|
|
def _(paths: List[pathlib.Path]):
|
|
# Store relative filenames in files (new schema)
|
|
files = [dict(path=i.name, mimetype='fake') for i in paths]
|
|
primary_path = paths[0]
|
|
params = dict(
|
|
directory=str(primary_path.parent),
|
|
primary_path=str(primary_path),
|
|
files=json.dumps(files),
|
|
)
|
|
test_session.execute('INSERT INTO file_group '
|
|
'(indexed, directory, primary_path, files) VALUES '
|
|
'(true, :directory, :primary_path, :files)', params)
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def test_tags_config(test_directory):
|
|
from wrolpi.tags import test_tags_config
|
|
with test_tags_config():
|
|
(test_directory / 'config').mkdir(exist_ok=True)
|
|
config_path = test_directory / 'config/tags.yaml'
|
|
yield config_path
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_tags_config(test_tags_config):
|
|
def _(tags: dict = None):
|
|
from wrolpi.test.common import assert_dict_contains
|
|
with test_tags_config.open('rt') as fh:
|
|
contents = yaml.load(fh, Loader=yaml.Loader)
|
|
|
|
if tags:
|
|
for key, item in tags.items():
|
|
assert key in contents['tags']
|
|
assert_dict_contains(item, contents['tags'][key])
|
|
else:
|
|
# Expect no tags to be saved.
|
|
assert not contents['tags']
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def zip_file_factory(test_directory):
|
|
def _() -> bytes:
|
|
with tempfile.NamedTemporaryFile() as fh:
|
|
with zipfile.ZipFile(fh, 'w') as zip_file:
|
|
zip_file.write(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4')
|
|
|
|
fh.seek(0)
|
|
return fh.read()
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def assert_directories(test_session, test_directory):
|
|
def _(expected: Sequence[str]):
|
|
directories = test_session.query(Directory).all()
|
|
names = {str(i.path.relative_to(test_directory)) for i in directories}
|
|
assert names == set(expected)
|
|
|
|
return _
|
|
|
|
|
|
FORM_PART = b'''-----------------------------sanic\r
|
|
Content-Disposition: form-data; name="{name}"\r
|
|
\r
|
|
{value}\r
|
|
\r
|
|
'''
|
|
|
|
FORM_FILENAME_PART = b'''-----------------------------sanic\r
|
|
Content-Disposition: form-data; name="{name}"; filename="{filename}"\r
|
|
Content-Type: application/octet-stream\r
|
|
\r
|
|
{value}\r
|
|
\r
|
|
'''
|
|
|
|
|
|
@pytest.fixture
|
|
def make_multipart_form():
|
|
def _(forms: List[dict]):
|
|
boundary = b'-----------------------------sanic\r\n'
|
|
new_forms = []
|
|
for form in forms:
|
|
name, value, filename = form['name'].encode(), form['value'], form.get('filename', '').encode()
|
|
value = value if isinstance(value, bytes) else str(value).encode()
|
|
part = FORM_PART
|
|
if filename:
|
|
part = FORM_FILENAME_PART
|
|
part.replace(b'{filename}', filename)
|
|
part = part.replace(b'{name}', name)
|
|
part = part.replace(b'{value}', value)
|
|
new_forms.append(part)
|
|
body = boundary + boundary.join(new_forms) + boundary
|
|
return body
|
|
|
|
return _
|
|
|
|
|
|
@pytest.fixture
|
|
def simple_file_server(test_directory):
|
|
# Use a random port for testing
|
|
server_address = ('', 0) # 0 means pick any available port
|
|
|
|
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
|
|
def __init__(self, *args, **kwargs):
|
|
# Pass the directory to the handler
|
|
self.directory = kwargs['directory']
|
|
super().__init__(*args, **kwargs)
|
|
|
|
# Create the server
|
|
server = http.server.HTTPServer(
|
|
server_address,
|
|
# Hack to pass `test_directory` as `self.directory` to `SimpleHTTPRequestHandler`
|
|
lambda *args, **kwargs:
|
|
CustomHTTPRequestHandler(*args, directory=str(test_directory), **kwargs))
|
|
|
|
# Start the server in a background thread
|
|
server_thread = threading.Thread(target=server.serve_forever)
|
|
server_thread.daemon = True # So the server dies when the main thread dies
|
|
server_thread.start()
|
|
|
|
try:
|
|
yield server
|
|
finally:
|
|
# Cleanup
|
|
server.shutdown()
|
|
server.server_close()
|
|
server_thread.join()
|
|
|
|
|
|
@pytest.fixture
|
|
def mock_downloader_download_file():
|
|
"""Mock the Downloader.download_file method."""
|
|
contents = None
|
|
|
|
async def _download_file(self, id_, url, destination, **kwargs):
|
|
# **kwargs absorbs ctx= and any future optional args added to Downloader.download_file.
|
|
name = url.split('/')[-1]
|
|
output_path = destination / name
|
|
output_path.write_bytes(contents or b'')
|
|
return output_path
|
|
|
|
with mock.patch('wrolpi.downloader.Downloader.download_file', _download_file):
|
|
def set_contents(contents_):
|
|
nonlocal contents
|
|
contents = contents_
|
|
|
|
yield set_contents
|
|
|
|
|
|
async def await_file_worker(timeout: int = 30):
|
|
"""Wait for FileWorker to complete all pending tasks. Testing only."""
|
|
from wrolpi.files.worker import file_worker
|
|
|
|
start = time.time()
|
|
while time.time() - start < timeout:
|
|
file_worker.transfer_queue()
|
|
if file_worker.private_queue.qsize() > 0:
|
|
await file_worker.process_queue()
|
|
elif file_worker.public_queue.qsize() == 0:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
else:
|
|
raise RuntimeError('Timed out waiting for file_worker')
|
|
|
|
|
|
@pytest.fixture
|
|
def refresh_files(async_client):
|
|
"""Test helper that queues and waits for file refresh."""
|
|
|
|
async def _refresh(paths=None):
|
|
from wrolpi.common import get_media_directory
|
|
from wrolpi.files.worker import file_worker
|
|
|
|
paths = paths or [get_media_directory()]
|
|
file_worker.queue_refresh(paths)
|
|
await await_file_worker()
|
|
|
|
return _refresh
|