mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
The database now lives at <media>/config/wrolpi.db so a library (files, configs, AND database) travels with the drive between WROLPi systems. Core: - wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s, foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3 adapters keep raw-SQL datetime formats identical to the ORM's. - wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN. Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines. Stemming contract pinned by dedicated tests (watch/watches/watched/ watching are interchangeable). - wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic baseline and test DBs; fixes the channel video_count DELETE/UPDATE bugs and a latent total_size bug (it used the video id as channel id). - wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor; network-filesystem refusal; never creates/migrates/trusts a database beneath an unmounted production media directory (shadow-DB guard). - Alembic: 85 Postgres-era migrations squashed into one baseline; revision ids are dates (2026_07_10_1350) so filenames self-sort and alembic_version says when a schema was created; render_as_batch=True. - All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> -> HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM, generate_series histograms->strftime GROUP BY, DISTINCT ON-> row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit NULLS FIRST/LAST, LIMIT-before-OFFSET). Simple statements converted to SQLAlchemy instead (flag upsert, download retry/summary/tag checks, channel-by-tag estimate, Directory delete). - Config imports count skipped entries; background dumps are blocked after a lossy import so partial imports can never overwrite YAML configs (they are the sole source of truth for rebuilds now). - Search tolerates explicit JSON nulls for limit/offset (Postgres treated OFFSET NULL as 0; SQLite errors). - file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite (BIGINT is not a rowid alias; FTS5 content_rowid requires it). Ops teardown: docker-compose db service, CI postgres sidecars, initialize_api_db.sh, controller postgresql service, psycopg2-binary all removed. upgrade.sh stops/disables postgresql (data is NOT deleted; `systemctl enable --now postgresql` restores it); systemd units no longer Want postgresql.service. Existing-user upgrade: first boot creates+migrates the DB, configs import (tags recreate their FileGroups from disk), a refresh rebuilds the library index. QA'd on the 10.0.0.8 RPi: rebuild matched the Postgres baseline exactly; survived an uncontrolled hard reboot with clean integrity checks. Accepted behavior changes: FTS5 has no stopwords; bm25 length normalization shifts some rank orders; fast-path d_headline is un-highlighted; all-negative queries match nothing. Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked rebuild refresh with progress UX. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
140 lines
4.5 KiB
Python
140 lines
4.5 KiB
Python
import json
|
|
import pathlib
|
|
import tempfile
|
|
from contextlib import contextmanager
|
|
from functools import partialmethod
|
|
from itertools import zip_longest
|
|
from typing import List
|
|
|
|
import pytest
|
|
|
|
from wrolpi.api_utils import CustomJSONEncoder
|
|
from wrolpi.common import get_media_directory
|
|
from wrolpi.conftest import test_db, async_client # noqa
|
|
from wrolpi.vars import CIRCLECI, IS_MACOS
|
|
|
|
TEST_CONFIG_PATH = tempfile.NamedTemporaryFile(mode='rt', delete=False)
|
|
|
|
skip_circleci = pytest.mark.skipif(CIRCLECI, reason='This test is not supported in Circle CI')
|
|
skip_macos = pytest.mark.skipif(IS_MACOS, reason='This test is not supported on MacOS')
|
|
only_macos = pytest.mark.skipif(not IS_MACOS, reason='This test is only supported on MacOS')
|
|
|
|
|
|
class PytestCase:
|
|
"""
|
|
Replicate unittest.TestCase methods to bridge the gap between it and Pytest.
|
|
"""
|
|
|
|
@staticmethod
|
|
def assertGreater(a, b, msg: str = None): # noqa
|
|
assert a > b, msg
|
|
|
|
@staticmethod
|
|
def assertLess(a, b, msg: str = None): # noqa
|
|
assert a < b, msg
|
|
|
|
@staticmethod
|
|
def assertEqual(a, b, msg: str = None): # noqa
|
|
assert a == b, msg or f'{a} != {b}'
|
|
|
|
@staticmethod
|
|
def assertRaises(exception, func, *args, **kwargs): # noqa
|
|
try:
|
|
func(*args, **kwargs)
|
|
except exception:
|
|
pass
|
|
|
|
@classmethod
|
|
def assertDictContains(cls, d1: dict, d2: dict): # noqa
|
|
assert_dict_contains(d1, d2)
|
|
|
|
def assertError(self, response, http_status: int, code=None): # noqa
|
|
self.assertEqual(response.status_code, http_status)
|
|
if code:
|
|
self.assertEqual(response.json['code'], code)
|
|
|
|
@staticmethod
|
|
def assertTruth(value, expected): # noqa
|
|
"""
|
|
Check that a value is Truthy or Falsy.
|
|
"""
|
|
if expected is True:
|
|
assert value, f'Value {value} should have been truthy'
|
|
else:
|
|
assert not value, f'Value {value} should have been falsey'
|
|
|
|
assertTruthy = partialmethod(assertTruth, expected=True)
|
|
assertFalsey = partialmethod(assertTruth, expected=False)
|
|
|
|
def assertItemsTruthyOrFalsey(self, items_list: List, expected_list: List):
|
|
for d1, d2 in zip_longest(items_list, expected_list):
|
|
for d2_key in d2:
|
|
if d1 is None:
|
|
raise ValueError('d1 is None')
|
|
if d2 is None:
|
|
raise ValueError('d2 is None')
|
|
self.assertTruth(d1[d2_key], d2[d2_key])
|
|
|
|
@staticmethod
|
|
def assertLength(a, b): # noqa
|
|
assert len(a) == len(b), f'{len(a)=} != {len(b)=}'
|
|
|
|
|
|
@contextmanager
|
|
def build_test_directories(paths: List[str], tmp_dir: pathlib.Path = None) -> pathlib.Path:
|
|
"""
|
|
Create directories based on the provided structure.
|
|
|
|
Example:
|
|
>>> build_test_directories([
|
|
'channel1/vid1.mp4',
|
|
'channel2/vid1.mp4',
|
|
'channel2/vid2.mp4',
|
|
'channel2/vid2.en.vtt'
|
|
])
|
|
|
|
Creates directories like so:
|
|
channel1/vid1.mp4
|
|
channel2/vid1.mp4
|
|
channel2/vid2.mp4
|
|
channel2/vid2.en.vtt
|
|
"""
|
|
tmp_dir = tmp_dir or get_media_directory()
|
|
|
|
directories = filter(lambda i: i.endswith('/'), paths)
|
|
for directory in directories:
|
|
(tmp_dir / directory).mkdir(parents=True)
|
|
|
|
files = filter(lambda i: not i.endswith('/'), paths)
|
|
for file in files:
|
|
file = tmp_dir / file
|
|
parents = file.parents
|
|
parents[0].mkdir(parents=True, exist_ok=True)
|
|
(tmp_dir / file).touch()
|
|
|
|
yield tmp_dir.absolute()
|
|
|
|
|
|
def assert_dict_contains(d1: dict, d2: dict):
|
|
if hasattr(d1, '__dict__'):
|
|
d1 = d1.__dict__
|
|
if hasattr(d2, '__dict__'):
|
|
d2 = d2.__dict__
|
|
|
|
# Sort items so error is easier to read.
|
|
d1 = dict(sorted(d1.items()))
|
|
d2 = dict(sorted(d2.items()))
|
|
|
|
for k2 in d2.keys():
|
|
try:
|
|
assert d1, f'dict 1 is empty: {d1}'
|
|
assert d2, f'dict 1 is empty: {d2}'
|
|
assert k2 in d1, f'dict 1 does not contain {k2}'
|
|
if isinstance(d1[k2], dict):
|
|
assert_dict_contains(d1[k2], d2[k2])
|
|
else:
|
|
assert d1[k2] == d2[k2], f'{repr(k2)} of value {repr(d1[k2])} does not equal {repr(d2[k2])} in dict 1'
|
|
except AssertionError as e:
|
|
d1_str = json.dumps(d1, indent=1, cls=CustomJSONEncoder)
|
|
d2_str = json.dumps(d2, indent=1, cls=CustomJSONEncoder)
|
|
raise AssertionError(f'{d1_str}\n\n{d2_str}\n\nDictionaries above are unequal') from e
|