wrolpi/wrolpi/test/common.py
Roland d9303a0ea8
Postgres -> SQLite hard cutover: the database moves into the media directory.
The database now lives at <media>/config/wrolpi.db so a library (files,
configs, AND database) travels with the drive between WROLPi systems.

Core:
- wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s,
  foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native
  sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3
  adapters keep raw-SQL datetime formats identical to the ORM's.
- wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN.
  Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing
  A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines.
  Stemming contract pinned by dedicated tests (watch/watches/watched/
  watching are interchangeable).
- wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic
  baseline and test DBs; fixes the channel video_count DELETE/UPDATE
  bugs and a latent total_size bug (it used the video id as channel id).
- wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases
  written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor;
  network-filesystem refusal; never creates/migrates/trusts a database
  beneath an unmounted production media directory (shadow-DB guard).
- Alembic: 85 Postgres-era migrations squashed into one baseline;
  revision ids are dates (2026_07_10_1350) so filenames self-sort and
  alembic_version says when a schema was created; render_as_batch=True.
- All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> ->
  HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM,
  generate_series histograms->strftime GROUP BY, DISTINCT ON->
  row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit
  NULLS FIRST/LAST, LIMIT-before-OFFSET).  Simple statements converted
  to SQLAlchemy instead (flag upsert, download retry/summary/tag checks,
  channel-by-tag estimate, Directory delete).
- Config imports count skipped entries; background dumps are blocked
  after a lossy import so partial imports can never overwrite YAML
  configs (they are the sole source of truth for rebuilds now).
- Search tolerates explicit JSON nulls for limit/offset (Postgres
  treated OFFSET NULL as 0; SQLite errors).
- file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite
  (BIGINT is not a rowid alias; FTS5 content_rowid requires it).

Ops teardown: docker-compose db service, CI postgres sidecars,
initialize_api_db.sh, controller postgresql service, psycopg2-binary all
removed.  upgrade.sh stops/disables postgresql (data is NOT deleted;
`systemctl enable --now postgresql` restores it); systemd units no
longer Want postgresql.service.

Existing-user upgrade: first boot creates+migrates the DB, configs
import (tags recreate their FileGroups from disk), a refresh rebuilds
the library index.  QA'd on the 10.0.0.8 RPi: rebuild matched the
Postgres baseline exactly; survived an uncontrolled hard reboot with
clean integrity checks.

Accepted behavior changes: FTS5 has no stopwords; bm25 length
normalization shifts some rank orders; fast-path d_headline is
un-highlighted; all-negative queries match nothing.

Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress
re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked
rebuild refresh with progress UX.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-07-12 14:09:37 -06:00

140 lines
4.5 KiB
Python

import json
import pathlib
import tempfile
from contextlib import contextmanager
from functools import partialmethod
from itertools import zip_longest
from typing import List
import pytest
from wrolpi.api_utils import CustomJSONEncoder
from wrolpi.common import get_media_directory
from wrolpi.conftest import test_db, async_client # noqa
from wrolpi.vars import CIRCLECI, IS_MACOS
TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in Circle CI')
skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on MacOS')
only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS')
class PytestCase:
"""
Replicate unittest.TestCase methods to bridge the gap between it and Pytest.
"""
@staticmethod
def assertGreater(a, b, msg: str = None): # noqa
assert a > b, msg
@staticmethod
def assertLess(a, b, msg: str = None): # noqa
assert a < b, msg
@staticmethod
def assertEqual(a, b, msg: str = None): # noqa
assert a == b, msg or f'{a} != {b}'
@staticmethod
def assertRaises(exception, func, *args, **kwargs): # noqa
try:
func(*args, **kwargs)
except exception:
pass
@classmethod
def assertDictContains(cls, d1: dict, d2: dict): # noqa
assert_dict_contains(d1, d2)
def assertError(self, response, http_status: int, code=None): # noqa
self.assertEqual(response.status_code, http_status)
if code:
self.assertEqual(response.json['code'], code)
@staticmethod
def assertTruth(value, expected): # noqa
"""
Check that a value is Truthy or Falsy.
"""
if expected is True:
assert value, f'Value {value} should have been truthy'
else:
assert not value, f'Value {value} should have been falsey'
assertTruthy = partialmethod(assertTruth, expected=True)
assertFalsey = partialmethod(assertTruth, expected=False)
def assertItemsTruthyOrFalsey(self, items_list: List, expected_list: List):
for d1, d2 in zip_longest(items_list, expected_list):
for d2_key in d2:
if d1 is None:
raise ValueError('d1 is None')
if d2 is None:
raise ValueError('d2 is None')
self.assertTruth(d1[d2_key], d2[d2_key])
@staticmethod
def assertLength(a, b): # noqa
assert len(a) == len(b), f'{len(a)=} != {len(b)=}'
@contextmanager
def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pathlib.Path:
"""
Create directories based on the provided structure.
Example:
>>> build_test_directories([
'channel1/vid1.mp4',
'channel2/vid1.mp4',
'channel2/vid2.mp4',
'channel2/vid2.en.vtt'
])
Creates directories like so:
channel1/vid1.mp4
channel2/vid1.mp4
channel2/vid2.mp4
channel2/vid2.en.vtt
"""
tmp_dir = tmp_dir or get_media_directory()
directories = filter(lambda i: i.endswith('/'), paths)
for directory in directories:
(tmp_dir / directory).mkdir(parents=True)
files = filter(lambda i: not i.endswith('/'), paths)
for file in files:
file = tmp_dir / file
parents = file.parents
parents[0].mkdir(parents=True, exist_ok=True)
(tmp_dir / file).touch()
yield tmp_dir.absolute()
def assert_dict_contains(d1: dict, d2: dict):
if hasattr(d1, '__dict__'):
d1 = d1.__dict__
if hasattr(d2, '__dict__'):
d2 = d2.__dict__
# Sort items so error is easier to read.
d1 = dict(sorted(d1.items()))
d2 = dict(sorted(d2.items()))
for k2 in d2.keys():
try:
assert d1, f'dict 1 is empty: {d1}'
assert d2, f'dict 1 is empty: {d2}'
assert k2 in d1, f'dict 1 does not contain {k2}'
if isinstance(d1[k2], dict):
assert_dict_contains(d1[k2], d2[k2])
else:
assert d1[k2] == d2[k2], f'{repr(k2)} of value {repr(d1[k2])} does not equal {repr(d2[k2])} in dict 1'
except AssertionError as e:
d1_str = json.dumps(d1, indent=1, cls=CustomJSONEncoder)
d2_str = json.dumps(d2, indent=1, cls=CustomJSONEncoder)
raise AssertionError(f'{d1_str}\n\n{d2_str}\n\nDictionaries above are unequal') from e