wrolpi/wrolpi/conftest.py
Roland 06ac0d2055 review: ffprobe out of the write transaction, and three tests that were watching nothing
All three bugs confirmed; the two suggestions taken.

[bug] `video_modeler` awaited ffprobe -- a subprocess per video, up to 20
per batch -- inside its `commit=True` session.  With IMMEDIATE as the
default that holds the write lock for the whole batch, which is the same
starvation this branch is curing in `video_cleanup`.  (It was already
partly true before: the batch's first `session.flush([video])` took the
lock ahead of the remaining ffprobes.)  The batch is now read, ffprobed
with no transaction open, then written.  `get_or_create_ffprobe_json` in
modules/videos/common.py is the path-only entry point that makes that
possible; `Video.get_ffprobe_json` delegates to it and keeps appending
the cache file to its FileGroup.  `validate()` -- and the poster
generation it can trigger -- is still inside the write transaction, and
now says so.

[bug] `test_download_dispatched_outside_immediate_transaction` probed
`getattr(_immediate_txn, 'active', False)`, which a ContextVar never has,
so it passed unconditionally.  It now probes the actual write lock from a
competing connection under `production_like_sessions`.  Mutation-checked:
moving the dispatch back inside the claim transaction fails it.

[bug] `asyncio.create_task` snapshots the current Context, so a flag held
for a write session's lifetime was inherited by every task spawned inside
it -- and the parent's `reset()` cannot reach a child's copy, leaving
that task issuing BEGIN IMMEDIATE for its *read* sessions.
`create_downloads` spawns `dispatch_downloads` exactly that way, so every
user-created download hit it.  The flag now lives only across the
session's opening `connection()` call, which also means the write lock is
taken when the session opens rather than at the caller's first statement.

[suggestion] `renew_recurring_downloads` scanned every recurring Download
under the write lock, including the usual cycle where nothing is due.  It
reads first now and takes the lock only for the rows that change,
re-checking status since the dispatcher may have claimed one in between.

[nit] Parenthesized the directory OR in the claim join.

`probe_write_lock_is_held` moved to conftest alongside
`write_lock_held_briefly`; three suites use it now.

Suite: 1712 passed, 5 skipped.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
2026-08-07 20:15:50 -06:00

1230 lines
42 KiB
Python

"""
Fixtures for Pytest tests.
"""
import asyncio
import contextlib
import copy
import http.server
import json
import logging
import multiprocessing
import pathlib
import shutil
import sqlite3
import tempfile
import threading
import time
import zipfile
from abc import ABC
from datetime import datetime
from itertools import zip_longest
from pathlib import Path
from typing import List, Callable, Dict, Sequence, Union, Coroutine, Awaitable, Optional, Any, Generator, AsyncGenerator
from typing import Tuple, Set
from unittest import mock
from unittest.mock import MagicMock, AsyncMock
from uuid import uuid1, uuid4
import pytest
import sqlalchemy
import yaml
from PIL import Image
from sanic_testing.testing import SanicASGITestClient
from sqlalchemy.engine import Engine, create_engine
from sqlalchemy.orm import Session, sessionmaker
from sqlalchemy.pool import NullPool
# Import root api so blueprints are attached.
import wrolpi.root_api # noqa
from wrolpi import flags
from wrolpi.api_utils import api_app
from wrolpi.cmd import CommandResult, run_command
from wrolpi.common import iterify, log_level_context, enable_wrol_mode, disable_wrol_mode, timer, TRACE_LEVEL, \
get_wrolpi_config
from wrolpi.common import logger, await_background_tasks as await_background_tasks_, BACKGROUND_TASKS
from wrolpi.common import set_test_media_directory, Base, set_test_config
from wrolpi.contexts import attach_shared_contexts, initialize_configs_contexts
from wrolpi.dates import set_test_now
from wrolpi.db import create_wrolpi_engine
from wrolpi.downloader import DownloadContext, DownloadManager, DownloadResult, Download, Downloader, \
downloads_manager_config_context, download_cache_config_context
from wrolpi.errors import UnrecoverableDownloadError
from wrolpi.files.models import Directory, FileGroup
from wrolpi.switches import await_switches as await_switches_
from wrolpi.tags import Tag, upsert_tag
from wrolpi.vars import PROJECT_DIR
logger = logger.getChild(__name__)
def test_db() -> Tuple[Engine, Session]:
"""Create a SQLAlchemy engine/session on a SQLite DB inside the (temporary) media directory.
The DB file is at the production location (`<media>/config/wrolpi.db`), so each test's
TemporaryDirectory provides isolation (also under pytest-xdist) and removes the DB (and its
WAL sidecars) on teardown."""
from wrolpi.common import get_media_directory, is_tempfile
from wrolpi.schema_ddl import install_raw_ddl
db_file = get_media_directory() / 'config' / 'wrolpi.db'
if not is_tempfile(db_file):
raise ValueError(f'Refusing to test on a non-temporary database! {db_file}')
db_file.parent.mkdir(parents=True, exist_ok=True)
test_engine = create_wrolpi_engine(db_file)
# Create all tables. No need to check if they exist because this is a test DB.
Base.metadata.create_all(test_engine, checkfirst=False)
# Triggers and FTS5 tables, exactly as the alembic baseline installs them.
with test_engine.connect() as conn:
install_raw_ddl(conn)
session = sessionmaker(bind=test_engine)()
return test_engine, session
@pytest.fixture()
def test_session(test_directory) -> Generator[Session, Any, None]:
"""Pytest Fixture to get a test database session."""
test_engine, session = test_db()
def fake_get_db_session():
"""Get the testing db"""
return test_engine, session
try:
with mock.patch('wrolpi.db._get_db_session', fake_get_db_session):
yield session
finally:
session.rollback()
session.close()
test_engine.dispose()
@pytest.fixture(autouse=True)
def test_debug_logger(request):
"""Tests default to DEBUG logging. Use pytest -vvv for TRACE level."""
level = logging.DEBUG
if request.config.option.verbose >= 3:
# -vvv
level = TRACE_LEVEL
with log_level_context(level):
yield
@pytest.fixture
def clear_background_tasks(request):
"""Clear BACKGROUND_TASKS before each test and fail if orphaned tasks remain after.
Tests must properly await all background tasks they create. Orphaned tasks indicate
a test is not properly cleaning up, which can cause cascading failures in other tests.
"""
# Clear before test - cancel any orphaned tasks from previous tests
if BACKGROUND_TASKS:
logger.warning(f'clear_background_tasks: clearing {len(BACKGROUND_TASKS)} orphaned tasks BEFORE test')
for task in list(BACKGROUND_TASKS):
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
logger.warning(f' orphaned task from previous test: {task_name}')
if not task.done():
task.cancel()
BACKGROUND_TASKS.clear()
yield
# After test - fail if orphaned tasks remain
if BACKGROUND_TASKS:
orphaned_tasks = []
for task in list(BACKGROUND_TASKS):
task_name = task.get_name() if hasattr(task, 'get_name') else str(task)
orphaned_tasks.append(f'{task_name} (done={task.done()})')
if not task.done():
task.cancel()
BACKGROUND_TASKS.clear()
# Fail the test
pytest.fail(
f'Test left {len(orphaned_tasks)} orphaned background task(s): {", ".join(orphaned_tasks)}. '
f'Tests must await all background tasks they create using await_background_tasks or '
f'the async_client fixture.'
)
@pytest.fixture
def test_directory() -> Generator[Path, Any, None]:
"""
Overwrite the media directory with a temporary directory.
"""
with tempfile.TemporaryDirectory() as tmp_dir:
tmp_path = pathlib.Path(tmp_dir).resolve()
tmp_path.chmod(0o40755)
assert tmp_path.is_dir()
set_test_media_directory(tmp_path)
yield tmp_path
@pytest.fixture
def test_wrolpi_config(test_directory) -> Generator[Path, Any, None]:
"""
Create a test config based off the example config.
"""
config_path = test_directory / 'config/wrolpi.yaml'
set_test_config(True)
yield config_path
set_test_config(False)
ROUTES_ATTACHED = False
@api_app.on_response
async def background_task_listener(request, response):
"""Wait for all background tasks to finish before returning API response while testing."""
await await_switches_()
class AutoAwaitTestClient:
"""
A wrapper around SanicASGITestClient that automatically awaits background
tasks and switches after modifying HTTP requests (POST, PUT, PATCH, DELETE).
This eliminates the need to manually call await_switches() and
await_background_tasks() after each modifying request in tests.
"""
def __init__(self, client: SanicASGITestClient, session: Session):
self._client = client
self._session = session
async def _await_and_expire(self):
"""Await all background tasks and switches, then expire session objects."""
await await_switches_()
await await_background_tasks_()
self._session.expire_all()
async def post(self, *args, **kwargs):
"""POST request with auto-await."""
result = await self._client.post(*args, **kwargs)
await self._await_and_expire()
return result
async def put(self, *args, **kwargs):
"""PUT request with auto-await."""
result = await self._client.put(*args, **kwargs)
await self._await_and_expire()
return result
async def patch(self, *args, **kwargs):
"""PATCH request with auto-await."""
result = await self._client.patch(*args, **kwargs)
await self._await_and_expire()
return result
async def delete(self, *args, **kwargs):
"""DELETE request with auto-await."""
result = await self._client.delete(*args, **kwargs)
await self._await_and_expire()
return result
async def get(self, *args, **kwargs):
"""GET request (no auto-await)."""
return await self._client.get(*args, **kwargs)
async def head(self, *args, **kwargs):
"""HEAD request (no auto-await)."""
return await self._client.head(*args, **kwargs)
async def options(self, *args, **kwargs):
"""OPTIONS request (no auto-await)."""
return await self._client.options(*args, **kwargs)
async def websocket(self, *args, **kwargs):
"""WebSocket connection (pass-through)."""
return await self._client.websocket(*args, **kwargs)
@property
def sanic_app(self):
"""Access to the underlying Sanic app."""
return self._client.sanic_app
@pytest.fixture
async def async_client(test_directory, test_session, clear_background_tasks) -> AsyncGenerator[
AutoAwaitTestClient, Any]:
"""Get an Async Sanic Test Client with all default routes attached.
Depends on test_session to ensure the database mock is in place for session middleware.
Depends on clear_background_tasks to detect orphaned background tasks.
This client automatically awaits switches and background tasks after
POST, PUT, PATCH, and DELETE requests, and expires session objects.
"""
api_app.signalize()
attach_shared_contexts(api_app)
initialize_configs_contexts(api_app)
client = SanicASGITestClient(api_app)
wrapped_client = AutoAwaitTestClient(client, test_session)
try:
yield wrapped_client
finally:
api_app.stop()
@pytest.fixture
def test_download_manager_config(async_client, test_directory) -> Generator[Path, Any, None]:
with downloads_manager_config_context():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/download_manager.yaml'
yield config_path
@pytest.fixture
def test_download_cache_config(async_client, test_directory) -> Generator[Path, Any, None]:
with download_cache_config_context():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/download_cache.yaml'
yield config_path
@pytest.fixture
async def await_switches(async_client):
"""Returns the `switches.await_switches` function, but primes the Sanic App to actually handle signals."""
await async_client.get('/api')
return await_switches_
@pytest.fixture
async def await_background_tasks(async_client, test_session):
"""Returns a function that awaits background tasks and expires session objects.
After background tasks complete, session objects may be stale, so we automatically
expire them to force a refresh on next access.
Args:
timeout: Optional timeout in seconds (default 15s from await_background_tasks_)
"""
await async_client.get('/api')
async def _await_and_expire(timeout: int = 10):
await await_background_tasks_(timeout=timeout)
test_session.expire_all()
return _await_and_expire
@pytest.fixture
async def test_download_manager(
async_client,
test_session, # session is required because downloads can start without the test DB in place.
test_download_manager_config,
) -> AsyncGenerator[DownloadManager, Any]:
with timer('test_download_manager'):
# Needed to use signals in
manager = DownloadManager()
await manager.enable()
# Disable per-domain rate limiting during tests to avoid timeouts.
get_wrolpi_config().download_wait = 0
yield manager
manager.stop()
@pytest.fixture
def fake_now() -> Generator[Callable[..., Any], Any, None]:
try:
set_test_now(datetime(2000, 1, 1))
yield set_test_now
finally:
# reset now() to its original functionality.
set_test_now(None) # noqa
@pytest.fixture
def successful_download() -> DownloadResult:
return DownloadResult(success=True)
@pytest.fixture
def failed_download() -> DownloadResult:
return DownloadResult(error='wrolpi.conftest failed_download error', success=False)
@pytest.fixture
def assert_download_urls(test_session) -> Callable[[Set[str]], None]:
def asserter(expected_urls: Set[str]):
downloads = test_session.query(Download).all()
urls = {i.url for i in downloads}
if urls != set(expected_urls):
raise AssertionError(f'Download URLs do not match: {urls} != {expected_urls}')
return asserter
@pytest.fixture
def assert_downloads(test_session) -> Callable[[List[Dict]], None]:
from wrolpi.test.common import assert_dict_contains
def asserter(expected: List[Dict]):
downloads = test_session.query(Download).order_by(Download.url)
for download_, expected_ in zip_longest(downloads, expected):
assert_dict_contains(download_.__json__(), expected_)
if (count := test_session.query(Download).count()) != len(expected):
raise AssertionError(f'Download count does not match: {count} != {len(expected)}')
return asserter
def make_test_ctx(*, is_cancelled=None, can_download=None,
outside_download_window=None, run_command_=None,
download_timeout=None,
report_progress=None, clear_progress=None) -> DownloadContext:
"""Build a DownloadContext suitable for downloader unit tests.
Defaults: never-cancelled, can_download True, no window restriction, real run_command,
no global timeout, progress reporting is a no-op. Override any callable to simulate
cancellation, throttling, fake subprocess output, or to record progress events.
Lives in wrolpi/conftest.py so any test (including primitive tests of process_runner /
cancel_wrapper) can construct a ctx without crossing module boundaries.
"""
return DownloadContext(
is_cancelled=is_cancelled or (lambda: False),
can_download=can_download or (lambda: True),
outside_download_window=outside_download_window or (lambda: False),
run_command=run_command_ or run_command,
download_timeout=download_timeout,
report_progress=report_progress or (lambda p: None),
clear_progress=clear_progress or (lambda: None),
)
def probe_write_lock_is_held(db_file: str) -> bool:
"""Return True if a competing connection cannot immediately take the write lock.
Uses a zero busy-timeout so the probe fails instantly if another connection holds a
RESERVED (write) lock, rather than waiting. In WAL mode a plain reader holds no such
lock, so this only returns True when a writer transaction is actually in progress.
"""
probe = sqlite3.connect(db_file, timeout=0)
try:
probe.execute('PRAGMA busy_timeout=0')
try:
probe.execute('BEGIN IMMEDIATE')
except sqlite3.OperationalError as e:
if 'database is locked' in str(e):
return True
raise
probe.execute('ROLLBACK')
return False
finally:
probe.close()
@contextlib.contextmanager
def write_lock_held_briefly(db_file: str, seconds: float = 1.0):
"""Hold the SQLite write lock on `db_file` for `seconds`, from another connection.
Simulates the concurrent writer (refresh, download, config save) that a real WROLPi runs
alongside whatever the test is doing. The lock is released while the test's work is still in
flight, so a correctly-written transaction waits (busy_timeout) and then succeeds; a deferred
one that upgrades its lock mid-transaction fails instantly instead.
Pair with `production_like_sessions` — the shared `test_session` uses one connection, which
cannot contend with itself.
"""
holder = sqlite3.connect(db_file, timeout=30, check_same_thread=False)
holder.isolation_level = None
holder.execute('PRAGMA busy_timeout=30000')
holder.execute('BEGIN IMMEDIATE')
def release():
time.sleep(seconds)
holder.execute('ROLLBACK')
holder.close()
thread = threading.Thread(target=release, daemon=True)
thread.start()
try:
yield
finally:
thread.join(timeout=30)
@contextlib.contextmanager
def production_like_sessions(test_session) -> Generator[sessionmaker, Any, None]:
"""Patch get_db_session to hand out a *fresh* Session per call, like production.
The `test_session` fixture reuses one shared Session for every `get_db_session` call, so an object loaded in
one `with` block stays attached for the next and exceptions never roll back. That convenience masks
detached-object and transaction-rollback bugs that only bite production's per-call sessions. Use this to
exercise that behavior: the yielded sessionmaker creates independent Sessions so a test can verify committed
DB state the way a separate worker would see it.
A dedicated NullPool engine is used and disposed on exit so its connections are released before the test DB
is dropped during fixture teardown.
"""
engine = create_wrolpi_engine(test_session.get_bind().url)
maker = sessionmaker(bind=engine)
sessions = []
def factory():
session = maker()
sessions.append(session)
return engine, session
try:
with mock.patch('wrolpi.db._get_db_session', factory):
yield maker
finally:
for session in sessions:
try:
session.close()
except Exception as e:
# Best-effort: engine.dispose() below releases the connections regardless, so a failed
# close() is inconsequential. Log at debug so it surfaces under -vvv without noising CI.
logger.debug(f'production_like_sessions: failed to close a session: {e}')
engine.dispose()
@pytest.fixture
def test_downloader(test_download_manager) -> Downloader:
class TestDownloader(Downloader, ABC):
"""A testing Downloader.
prepare_download returns a non-None sentinel so the manager's phase-split dispatch
routes through this subclass. execute_download is an AsyncMock — tests wire its
side_effect via the set_test_* methods. Returning a DownloadResult from
execute_download tells the dispatch to skip finalize (see signal_download_download).
"""
name = 'test_downloader'
def __repr__(self):
return '<TESTING Downloader>'
execute_download = AsyncMock()
def prepare_download(self, session, download):
return object() # sentinel; activates the phase-split path
def finalize_download(self, session, download, executed):
# execute_download returns a DownloadResult directly, so the dispatch
# short-circuits before reaching finalize. Provided here only to satisfy
# the abstract contract.
return DownloadResult(success=True)
def set_test_success(self):
async def _(*a, **kwargs):
# Sleep so download happens after testing is waiting.
await asyncio.sleep(1)
return DownloadResult(success=True)
self.execute_download.side_effect = _
def set_test_failure(self):
async def _(*a, **kwargs):
# Sleep so download happens after testing is waiting.
await asyncio.sleep(1)
return DownloadResult(success=False)
self.execute_download.side_effect = _
def set_test_exception(self, exception: Exception = Exception('Test downloader exception')):
async def _(*a, **kwargs):
# Sleep so download happens after testing is waiting.
await asyncio.sleep(1)
raise exception
self.execute_download.side_effect = _
def set_test_unrecoverable_exception(self):
async def _(*a, **kwargs):
# Sleep so download happens after testing is waiting.
await asyncio.sleep(1)
raise UnrecoverableDownloadError()
self.execute_download.side_effect = _
test_downloader = TestDownloader()
# Default to successful download.
test_downloader.set_test_success()
test_download_manager.register_downloader(test_downloader)
return test_downloader
@pytest.fixture
def video_file(test_directory) -> Generator[Path, Any, None]:
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
destination = test_directory / f'video-{uuid4()}.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
yield destination
@pytest.fixture
def corrupted_video_file(test_directory) -> Generator[Path, Any, None]:
"""Return a copy of a corrupted video file in the `test_directory`."""
destination = test_directory / f'corrupted-{uuid4()}.mp4'
shutil.copy(PROJECT_DIR / 'test/corrupted.mp4', destination)
yield destination
@pytest.fixture
def video_file_factory(test_directory) -> Callable[[pathlib.Path], pathlib.Path]:
"""Return a copy of the example Big Buck Bunny video in the `test_directory`."""
def _(path: pathlib.Path = None) -> pathlib.Path:
destination = path or test_directory / f'{uuid4()}.mp4'
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', destination)
return destination
return _
@pytest.fixture
def video_bytes() -> bytes:
return (PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4').read_bytes()
@pytest.fixture
def image_file(test_directory) -> Generator[Path, Any, None]:
"""Create a small image file in the `test_directory`."""
destination = test_directory / f'{uuid4()}.jpeg'
Image.new('RGB', (25, 25), color='grey').save(destination)
yield destination
@pytest.fixture
def vtt_file1(test_directory) -> Generator[Path, Any, None]:
"""Return a copy of the example1 VTT file in the `test_directory`."""
destination = test_directory / f'{uuid4()}.en.vtt'
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', destination)
yield destination
@pytest.fixture
def vtt_file2(test_directory) -> Generator[Path, Any, None]:
"""Return a copy of the example2 VTT file in the `test_directory`."""
destination = test_directory / f'{uuid4()}.en.vtt'
shutil.copy(PROJECT_DIR / 'test/example2.en.vtt', destination)
yield destination
@pytest.fixture
def srt_text() -> str:
return (PROJECT_DIR / 'test/example3.en.srt').read_text()
@pytest.fixture
def srt_file3(test_directory) -> Generator[Path, Any, None]:
"""Return a copy of the example3 SRT file in the `test_directory`."""
destination = test_directory / f'{uuid4()}.en.srt'
shutil.copy(PROJECT_DIR / 'test/example3.en.srt', destination)
yield destination
@pytest.fixture
def example_pdf(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'pdf example.pdf'
shutil.copy(PROJECT_DIR / 'test/pdf example.pdf', destination)
yield destination
@pytest.fixture
def example_epub(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'ebook example.epub'
shutil.copy(PROJECT_DIR / 'test/ebook example.epub', destination)
yield destination
@pytest.fixture
def example_mobi(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'ebook example.mobi'
shutil.copy(PROJECT_DIR / 'test/ebook example.mobi', destination)
yield destination
@pytest.fixture
def example_doc(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'example word.doc'
shutil.copy(PROJECT_DIR / 'test/example word.doc', destination)
yield destination
@pytest.fixture
def example_docx(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'example word.docx'
shutil.copy(PROJECT_DIR / 'test/example word.docx', destination)
yield destination
@pytest.fixture
def example_odt(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'test_book_odt.odt'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/test_book_odt.odt', destination)
yield destination
@pytest.fixture
def example_cbz(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_sample.cbz'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_sample.cbz', destination)
yield destination
@pytest.fixture
def example_cbz_metadata(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_metadata.cbz'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_metadata.cbz', destination)
yield destination
@pytest.fixture
def example_cbt(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_cbt.cbt'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbt.cbt', destination)
yield destination
@pytest.fixture
def example_cbr(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_cbr.cbr'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cbr.cbr', destination)
yield destination
@pytest.fixture
def example_cb7(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_cb7.cb7'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_cb7.cb7', destination)
yield destination
@pytest.fixture
def example_cbz_dir(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_dir_cbz.cbz'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbz.cbz', destination)
yield destination
@pytest.fixture
def example_cbt_dir(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_dir_cbt.cbt'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cbt.cbt', destination)
yield destination
@pytest.fixture
def example_cb7_dir(test_directory) -> Generator[Path, Any, None]:
destination = test_directory / 'bobby_make_believe_dir_cb7.cb7'
shutil.copy(PROJECT_DIR / 'test/sample_reading_media/bobby_make_believe_dir_cb7.cb7', destination)
yield destination
@pytest.fixture
def make_files_structure(test_directory) -> Callable[[Union[List[Union[pathlib.Path, str]], dict]], List[pathlib.Path]]:
"""
A fixture which creates test files passed to it. If a list is provided, empty files will be created at those
locations. If a dict is provided, files will be created containing the value of the dict item.
"""
def create_files(paths: Union[List, Dict], file_groups: bool = False, session: Session = None) \
-> List[pathlib.Path]:
# Note: session is optional here for backwards compatibility - file_groups=True requires session
files = []
@iterify(list)
def touch_paths(paths_):
for name in paths_:
path = test_directory / name
if name.endswith('/'):
path.mkdir(parents=True, exist_ok=True)
else:
path.parent.mkdir(parents=True, exist_ok=True)
path.touch()
yield path
if isinstance(paths, list):
files = touch_paths(paths)
elif isinstance(paths, Dict):
files = touch_paths(paths.keys())
for path, content in paths.items():
path = test_directory / path
if isinstance(content, str):
path.write_text(content)
elif isinstance(content, bytes):
path.write_bytes(content)
if file_groups:
for idx, path in enumerate(files):
fg = FileGroup.from_paths(session, path)
fg.do_index()
files[idx] = fg
return sorted(files) if isinstance(paths, dict) else files
return create_files
async def set_wrol_mode(enabled: bool):
if enabled:
enable_wrol_mode()
else:
await disable_wrol_mode()
@pytest.fixture
def wrol_mode_fixture(test_wrolpi_config, test_download_manager) -> Callable[[bool], Coroutine]:
return set_wrol_mode
@pytest.fixture
def mock_create_subprocess_shell() -> Callable:
def mocker(communicate_return=None, return_code=None, communicate_side_effect=None):
async def communicate(*a, **kw):
if communicate_side_effect:
return communicate_side_effect()
return communicate_return or (None, None)
async def create_subprocess_shell(*a, **kw):
proc = MagicMock()
proc.communicate = communicate
proc.returncode = return_code if return_code is not None else 0
return proc
return create_subprocess_shell
return mocker
@pytest.fixture
def mock_run_command():
"""Mock wrolpi.cmd's `run_command` using `TESTING_RUN_COMMAND_RESULT`."""
default = CommandResult(
return_code=0,
cancelled=False,
stdout=b'mock_run_command fixture stdout',
stderr=b'mock_run_command fixture stderr',
elapsed=0,
pid=123,
)
with mock.patch('wrolpi.cmd.TESTING_RUN_COMMAND_RESULT') as mock_TESTING_RUN_COMMAND_RESULT:
mock_TESTING_RUN_COMMAND_RESULT.return_value = default
def do_mock(new_result):
mock_TESTING_RUN_COMMAND_RESULT.return_value = new_result
yield do_mock
@pytest.fixture
def run_command_bad_result() -> CommandResult:
return CommandResult(1, False, b'run_command_bad_result stdout', b'run_command_bad_result stderr', 0, 123)
class EventsFixture:
"""Fixture for testing events with rich assertions.
Only returns events created after this fixture was instantiated,
making it safe for parallel test execution.
"""
def __init__(self, events_history: list):
self._history = events_history
# Track the starting index to only see new events during this test
self._start_index = len(events_history)
@property
def events(self) -> List[dict]:
"""Return events captured during this test (most recent first)."""
# Only return events added after fixture creation
new_events = list(self._history[self._start_index:])
return list(reversed(new_events))
def get_by_name(self, event_name: str) -> List[dict]:
"""Get all events with the given name."""
return [e for e in self.events if e['event'] == event_name]
def get_by_subject(self, subject: str) -> List[dict]:
"""Get all events with the given subject."""
return [e for e in self.events if e['subject'] == subject]
def assert_has_event(self, event_name: str, message: str = None) -> None:
"""Assert that an event with the given name exists."""
matching = self.get_by_name(event_name)
assert matching, f"No event found with name '{event_name}'. Events: {[e['event'] for e in self.events]}"
if message:
assert any(message in (e.get('message') or '') for e in matching), \
f"No '{event_name}' event contains message '{message}'"
def assert_no_event(self, event_name: str) -> None:
"""Assert that no event with the given name exists."""
matching = self.get_by_name(event_name)
assert not matching, f"Found unexpected event '{event_name}': {matching}"
def assert_events_contain(self, expected: List[dict]) -> None:
"""Assert that events contain all expected events (order matters, most recent first)."""
from wrolpi.test.common import assert_dict_contains
for exp, actual in zip_longest(expected, self.events):
assert actual is not None, f"Expected event {exp} but no more events found"
assert_dict_contains(actual, exp)
def assert_event_count(self, count: int) -> None:
"""Assert the total number of events."""
assert len(self.events) == count, f"Expected {count} events, got {len(self.events)}"
@pytest.fixture
def events_fixture(async_client) -> EventsFixture:
"""Fixture for testing events with rich assertions.
Only sees events created after this fixture is instantiated,
making it safe for parallel test execution.
"""
return EventsFixture(api_app.shared_ctx.events_history)
FLAGS_LOCK = multiprocessing.Lock()
@pytest.fixture()
def flags_lock():
"""Wait for exclusive access to flags before testing."""
try:
FLAGS_LOCK.acquire(timeout=60)
flags.TESTING_LOCK.set()
yield
finally:
flags.TESTING_LOCK.clear()
FLAGS_LOCK.release()
@pytest.fixture()
def tag_factory(test_session) -> Callable[[Optional[str]], Awaitable[Tag]]:
names = ['one', 'two', 'three', 'four', 'five', 'six']
count = 1
async def factory(name: str = None) -> Tag:
if not name:
name = names.pop(0)
tag = await upsert_tag(test_session, name, f'#{str(count) * 6}')
return tag
return factory
@pytest.fixture
def assert_files(test_session, test_directory):
def _(files: List[Dict]):
from wrolpi.files.models import FileGroup
for expected in files:
primary_path = test_directory / expected.pop('path')
try:
file: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
except sqlalchemy.orm.exc.NoResultFound as e:
raise ValueError(f'No FileGroup found with {primary_path=}') from e
for key, value in expected.items():
attr = getattr(file, key)
if attr != value:
raise AssertionError(f'{primary_path} {attr} != {value}')
return _
@pytest.fixture
def assert_file_groups(test_session, test_directory):
from wrolpi.test.common import assert_dict_contains
def _(file_groups: List[Dict], assert_count: bool = True):
from wrolpi.files.models import FileGroup
for expected in file_groups:
if 'primary_path' not in expected:
raise Exception('You must specify the primary path for this fixture!')
primary_path = str(test_directory / expected.pop('primary_path'))
try:
file_group: FileGroup = test_session.query(FileGroup).filter_by(primary_path=primary_path).one()
except Exception as e:
raise ValueError(f'No FileGroup found with {primary_path=}') from e
# Compare file dictionaries.
files = expected.pop('files', None)
if files:
for exp, actual in zip_longest(files, file_group.files):
assert_dict_contains(actual, exp)
for key, value in expected.items():
attr = getattr(file_group, key)
if isinstance(value, dict):
assert_dict_contains(attr, value)
elif attr != value:
raise AssertionError(f'{file_group}.{key} {attr=} != {value}')
if assert_count is True:
if (count := test_session.query(FileGroup).count()) != len(file_groups):
raise AssertionError(f'FileGroup count does not match {count} != {len(file_groups)}')
return _
@pytest.fixture
def assert_files_search(async_client):
from wrolpi.test.common import assert_dict_contains
async def _(search_str: str, expected: List[dict]):
content = json.dumps({'search_str': search_str})
request, response = await async_client.post('/api/files/search', content=content)
for file_group, exp in zip_longest(response.json['file_groups'], expected):
assert_dict_contains(file_group, exp)
assert len(response.json['file_groups']) == len(expected)
return _
SINGLEFILE_CONTENTS = '''<html><!--
Page saved with SingleFile
url: {url}
saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
--><head><meta charset="utf-8">
<meta name="msapplication-TileColor" content="#2d89ef">
<meta name="theme-color" content="#000000">
{title}
<body>
some body contents
</body>
</html>
'''
@pytest.fixture
def singlefile_contents_factory():
def _(title: str = '', url: str = 'https://example.com') -> str:
contents = copy.deepcopy(SINGLEFILE_CONTENTS)
contents = contents.format(
title=f'<title>{title}</title>',
url=url,
)
return contents
return _
@pytest.fixture
def example_singlefile(test_directory, singlefile_contents_factory) -> pathlib.Path:
archive = test_directory / 'archive'
archive.mkdir(exist_ok=True)
singlefile = archive / 'singlefile.html'
singlefile.write_text(singlefile_contents_factory())
return singlefile
@pytest.fixture
def image_bytes_factory():
def _(suffix: str = '.png') -> bytes:
with tempfile.NamedTemporaryFile(suffix=suffix) as fh:
Image.new('RGB', (1, 1), color='grey').save(fh)
fh.seek(0)
image = fh.read()
return image
return _
@pytest.fixture
def insert_file_group(test_session, test_directory):
"""Inserts a FileGroup based on the provided paths. Does not verify if the files exist."""
def _(paths: List[pathlib.Path]):
# Store relative filenames in files (new schema)
files = [dict(path=i.name, mimetype='fake') for i in paths]
primary_path = paths[0]
params = dict(
directory=str(primary_path.parent),
primary_path=str(primary_path),
files=json.dumps(files),
)
test_session.execute('INSERT INTO file_group '
'(indexed, directory, primary_path, files) VALUES '
'(true, :directory, :primary_path, :files)', params)
return _
@pytest.fixture
def test_tags_config(test_directory):
from wrolpi.tags import test_tags_config
with test_tags_config():
(test_directory / 'config').mkdir(exist_ok=True)
config_path = test_directory / 'config/tags.yaml'
yield config_path
@pytest.fixture
def assert_tags_config(test_tags_config):
def _(tags: dict = None):
from wrolpi.test.common import assert_dict_contains
with test_tags_config.open('rt') as fh:
contents = yaml.load(fh, Loader=yaml.Loader)
if tags:
for key, item in tags.items():
assert key in contents['tags']
assert_dict_contains(item, contents['tags'][key])
else:
# Expect no tags to be saved.
assert not contents['tags']
return _
@pytest.fixture
def zip_file_factory(test_directory):
def _() -> bytes:
with tempfile.NamedTemporaryFile() as fh:
with zipfile.ZipFile(fh, 'w') as zip_file:
zip_file.write(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4')
fh.seek(0)
return fh.read()
return _
@pytest.fixture
def assert_directories(test_session, test_directory):
def _(expected: Sequence[str]):
directories = test_session.query(Directory).all()
names = {str(i.path.relative_to(test_directory)) for i in directories}
assert names == set(expected)
return _
FORM_PART = b'''-----------------------------sanic\r
Content-Disposition: form-data; name="{name}"\r
\r
{value}\r
\r
'''
FORM_FILENAME_PART = b'''-----------------------------sanic\r
Content-Disposition: form-data; name="{name}"; filename="{filename}"\r
Content-Type: application/octet-stream\r
\r
{value}\r
\r
'''
@pytest.fixture
def make_multipart_form():
def _(forms: List[dict]):
boundary = b'-----------------------------sanic\r\n'
new_forms = []
for form in forms:
name, value, filename = form['name'].encode(), form['value'], form.get('filename', '').encode()
value = value if isinstance(value, bytes) else str(value).encode()
part = FORM_PART
if filename:
part = FORM_FILENAME_PART
part.replace(b'{filename}', filename)
part = part.replace(b'{name}', name)
part = part.replace(b'{value}', value)
new_forms.append(part)
body = boundary + boundary.join(new_forms) + boundary
return body
return _
@pytest.fixture
def simple_file_server(test_directory):
# Use a random port for testing
server_address = ('', 0) # 0 means pick any available port
class CustomHTTPRequestHandler(http.server.SimpleHTTPRequestHandler):
def __init__(self, *args, **kwargs):
# Pass the directory to the handler
self.directory = kwargs['directory']
super().__init__(*args, **kwargs)
# Create the server
server = http.server.HTTPServer(
server_address,
# Hack to pass `test_directory` as `self.directory` to `SimpleHTTPRequestHandler`
lambda *args, **kwargs:
CustomHTTPRequestHandler(*args, directory=str(test_directory), **kwargs))
# Start the server in a background thread
server_thread = threading.Thread(target=server.serve_forever)
server_thread.daemon = True # So the server dies when the main thread dies
server_thread.start()
try:
yield server
finally:
# Cleanup
server.shutdown()
server.server_close()
server_thread.join()
@pytest.fixture
def mock_downloader_download_file():
"""Mock the Downloader.download_file method."""
contents = None
async def _download_file(self, id_, url, destination, **kwargs):
# **kwargs absorbs ctx= and any future optional args added to Downloader.download_file.
name = url.split('/')[-1]
output_path = destination / name
output_path.write_bytes(contents or b'')
return output_path
with mock.patch('wrolpi.downloader.Downloader.download_file', _download_file):
def set_contents(contents_):
nonlocal contents
contents = contents_
yield set_contents
async def await_file_worker(timeout: int = 30):
"""Wait for FileWorker to complete all pending tasks. Testing only."""
from wrolpi.files.worker import file_worker
start = time.time()
while time.time() - start < timeout:
file_worker.transfer_queue()
if file_worker.private_queue.qsize() > 0:
await file_worker.process_queue()
elif file_worker.public_queue.qsize() == 0:
break
await asyncio.sleep(0.01)
else:
raise RuntimeError('Timed out waiting for file_worker')
@pytest.fixture
def refresh_files(async_client):
"""Test helper that queues and waits for file refresh."""
async def _refresh(paths=None):
from wrolpi.common import get_media_directory
from wrolpi.files.worker import file_worker
paths = paths or [get_media_directory()]
file_worker.queue_refresh(paths)
await await_file_worker()
return _refresh