mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
The database now lives at <media>/config/wrolpi.db so a library (files, configs, AND database) travels with the drive between WROLPi systems. Core: - wrolpi/db.py: lazy SQLite engine factory (WAL, busy_timeout=30s, foreign_keys=ON); get_db_curs yields a plain sqlite3.Cursor; native sqlite3 paramstyle everywhere (no compatibility shims); global sqlite3 adapters keep raw-SQL datetime formats identical to the ORM's. - wrolpi/fts.py: FTS5 external-content search replacing tsvector/GIN. Injection-proof websearch->FTS5 translator, bm25(10,4,2,1) reproducing A/B/C/D weighting, fast(abc)/deep column filtering, snippet headlines. Stemming contract pinned by dedicated tests (watch/watches/watched/ watching are interchangeable). - wrolpi/schema_ddl.py: SQLite triggers shared verbatim by the alembic baseline and test DBs; fixes the channel video_count DELETE/UPDATE bugs and a latent total_size bug (it used the video id as channel id). - wrolpi/db_bootstrap.py: startup auto-create/migrate; refuses databases written by a newer WROLPi (db_version_mismatch); SQLite 3.40 floor; network-filesystem refusal; never creates/migrates/trusts a database beneath an unmounted production media directory (shadow-DB guard). - Alembic: 85 Postgres-era migrations squashed into one baseline; revision ids are dates (2026_07_10_1350) so filenames self-sort and alembic_version says when a schema was created; render_as_batch=True. - All raw Postgres SQL rewritten (ANY->json_each IN, array_agg @> -> HAVING COUNT(DISTINCT), unnest->executemany/UPDATE..FROM, generate_series histograms->strftime GROUP BY, DISTINCT ON-> row_number(), writable CTEs->two statements, jsonb_*->json_*, explicit NULLS FIRST/LAST, LIMIT-before-OFFSET). Simple statements converted to SQLAlchemy instead (flag upsert, download retry/summary/tag checks, channel-by-tag estimate, Directory delete). - Config imports count skipped entries; background dumps are blocked after a lossy import so partial imports can never overwrite YAML configs (they are the sole source of truth for rebuilds now). - Search tolerates explicit JSON nulls for limit/offset (Postgres treated OFFSET NULL as 0; SQLite errors). - file_group.id/doc_section.id render as INTEGER PRIMARY KEY on SQLite (BIGINT is not a rowid alias; FTS5 content_rowid requires it). Ops teardown: docker-compose db service, CI postgres sidecars, initialize_api_db.sh, controller postgresql service, psycopg2-binary all removed. upgrade.sh stops/disables postgresql (data is NOT deleted; `systemctl enable --now postgresql` restores it); systemd units no longer Want postgresql.service. Existing-user upgrade: first boot creates+migrates the DB, configs import (tags recreate their FileGroups from disk), a refresh rebuilds the library index. QA'd on the 10.0.0.8 RPi: rebuild matched the Postgres baseline exactly; survived an uncontrolled hard reboot with clean integrity checks. Accepted behavior changes: FTS5 has no stopwords; bm25 length normalization shifts some rank orders; fast-path d_headline is un-highlighted; all-negative queries match nothing. Deferred: Debian/trixie package lists (incl. sqlite3 CLI), Cypress re-baseline, VACUUM INTO snapshots, exFAT/NTFS warning, auto-kicked rebuild refresh with progress UX. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
189 lines
7.3 KiB
Python
189 lines
7.3 KiB
Python
"""Creates and migrates the SQLite database at API startup.
|
|
|
|
The database lives inside the media directory (`<media>/config/wrolpi.db`), so it may belong to
|
|
a drive that was last used by a different WROLPi:
|
|
* no database → create it and run the migrations (the configs will then be imported and a
|
|
refresh rebuilds the library index);
|
|
* database older than this code → migrate it forward;
|
|
* database NEWER than this code → refuse to write; the user must upgrade this WROLPi.
|
|
"""
|
|
import contextlib
|
|
import fcntl
|
|
import sqlite3
|
|
import sys
|
|
from pathlib import Path
|
|
from typing import Optional
|
|
|
|
from wrolpi import flags
|
|
from wrolpi.common import logger
|
|
from wrolpi.db import get_db_file, get_db_uri
|
|
from wrolpi.vars import PROJECT_DIR, PYTEST, DOCKERIZED
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
MINIMUM_SQLITE_VERSION = (3, 40, 0)
|
|
|
|
# WAL locking is unsafe on network filesystems; refuse to run there.
|
|
NETWORK_FS_TYPES = {'nfs', 'nfs4', 'cifs', 'smb3', 'smbfs', 'fuse.sshfs', 'fuse.rclone', '9p'}
|
|
|
|
|
|
def _media_fs_type(path: Path) -> Optional[str]:
|
|
"""The filesystem type of the mount containing `path` (Linux only)."""
|
|
if sys.platform != 'linux':
|
|
return None
|
|
try:
|
|
mounts = Path('/proc/mounts').read_text()
|
|
except OSError:
|
|
return None
|
|
best_prefix, best_type = '', None
|
|
path_str = str(path)
|
|
for line in mounts.splitlines():
|
|
parts = line.split()
|
|
if len(parts) < 3:
|
|
continue
|
|
mount_point, fs_type = parts[1], parts[2]
|
|
if path_str == mount_point or path_str.startswith(mount_point.rstrip('/') + '/') or mount_point == '/':
|
|
if len(mount_point) > len(best_prefix):
|
|
best_prefix, best_type = mount_point, fs_type
|
|
return best_type
|
|
|
|
|
|
def check_sqlite_environment(db_file: Path) -> Optional[str]:
|
|
"""Returns an error message when the environment cannot safely run the database, else None."""
|
|
if sqlite3.sqlite_version_info < MINIMUM_SQLITE_VERSION:
|
|
return f'SQLite {sqlite3.sqlite_version} is too old, WROLPi requires 3.40+'
|
|
fs_type = _media_fs_type(db_file.parent if db_file.parent.exists() else db_file.parent.parent)
|
|
if fs_type in NETWORK_FS_TYPES:
|
|
return f'Refusing to use a database on a network filesystem ({fs_type}); WAL locking is unsafe there'
|
|
return None
|
|
|
|
|
|
def media_directory_is_unmounted_production(media_directory: Path = None) -> bool:
|
|
"""True when this is a production host whose media directory should be a mount, but is not.
|
|
|
|
While true, the database must not be created, migrated, or even considered "up": any file at
|
|
the database path is on the root filesystem, shadowing the real database on the drive. (An
|
|
ORM connection implicitly creates an empty DB file, so file-existence checks cannot be used
|
|
to detect this situation.)"""
|
|
if PYTEST or DOCKERIZED or sys.platform != 'linux':
|
|
return False
|
|
if media_directory is None:
|
|
from wrolpi.common import get_media_directory
|
|
media_directory = get_media_directory()
|
|
if not str(media_directory).startswith('/media/'):
|
|
# A custom MEDIA_DIRECTORY (dev boxes) may legitimately not be a mount.
|
|
return False
|
|
try:
|
|
return not media_directory.is_mount()
|
|
except OSError:
|
|
return True
|
|
|
|
|
|
def _alembic_config():
|
|
from alembic.config import Config
|
|
config = Config(str(PROJECT_DIR / 'alembic.ini'))
|
|
config.set_main_option('sqlalchemy.url', get_db_uri())
|
|
return config
|
|
|
|
|
|
def compare_db_version() -> str:
|
|
"""Compare the database's alembic version to this code's.
|
|
|
|
Returns 'missing' (no/uninitialized DB file), 'current', 'behind' (needs migrations), or
|
|
'ahead' (the DB was written by a newer WROLPi)."""
|
|
from alembic.script import ScriptDirectory
|
|
|
|
db_file = get_db_file()
|
|
if not db_file.is_file():
|
|
return 'missing'
|
|
try:
|
|
# Read-only URI probe: a plain connect would create an empty database file.
|
|
with contextlib.closing(sqlite3.connect(f'file:{db_file}?mode=ro', uri=True)) as conn:
|
|
row = conn.execute('SELECT version_num FROM alembic_version').fetchone()
|
|
current = row[0] if row else None
|
|
except sqlite3.OperationalError:
|
|
current = None
|
|
if not current:
|
|
# The file exists but was never migrated; treat like a fresh database.
|
|
return 'missing'
|
|
|
|
script = ScriptDirectory.from_config(_alembic_config())
|
|
if current == script.get_current_head():
|
|
return 'current'
|
|
try:
|
|
script.get_revision(current)
|
|
return 'behind'
|
|
except Exception:
|
|
# This code does not know the DB's revision: the DB is from a newer WROLPi.
|
|
return 'ahead'
|
|
|
|
|
|
def ensure_db() -> bool:
|
|
"""Create/migrate the database. Returns True when the database is usable at head.
|
|
|
|
Never creates the database beneath an unmounted media directory on a production host
|
|
(that would silently start an empty library on the root filesystem)."""
|
|
from alembic import command
|
|
from wrolpi.common import get_media_directory
|
|
from wrolpi.events import Events
|
|
|
|
db_file = get_db_file()
|
|
|
|
if error := check_sqlite_environment(db_file):
|
|
logger.critical(error)
|
|
with contextlib.suppress(Exception):
|
|
Events.send_config_import_failed(error)
|
|
return False
|
|
|
|
media_directory = get_media_directory()
|
|
if not media_directory.is_dir():
|
|
logger.warning(f'Cannot create/migrate database; media directory is missing: {media_directory}')
|
|
return False
|
|
if media_directory_is_unmounted_production(media_directory):
|
|
# A production media directory that is not mounted: any database created/migrated here
|
|
# would land on the root filesystem and shadow the real database once the drive mounts.
|
|
# (Unconditional: a worker's ORM connection may have already created an empty DB file.)
|
|
logger.warning(f'Refusing to create/migrate database beneath unmounted media directory: {media_directory}')
|
|
return False
|
|
|
|
state = compare_db_version()
|
|
if state == 'ahead':
|
|
flags.db_version_mismatch.set()
|
|
message = 'The database was written by a NEWER WROLPi (drive from another system?). ' \
|
|
'Upgrade this WROLPi before using this drive.'
|
|
logger.critical(message)
|
|
with contextlib.suppress(Exception):
|
|
Events.send_config_import_failed(message)
|
|
return False
|
|
|
|
flags.db_version_mismatch.clear()
|
|
|
|
if state in ('missing', 'behind'):
|
|
db_file.parent.mkdir(parents=True, exist_ok=True)
|
|
logger.warning(f'Migrating database ({state}): {db_file}')
|
|
command.upgrade(_alembic_config(), 'head')
|
|
|
|
return True
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def bootstrap_lock():
|
|
"""A non-blocking cross-process lock; yields True when this process holds the lock.
|
|
|
|
Multiple Sanic workers may try to bootstrap simultaneously; only one needs to."""
|
|
lock_file = get_db_file().with_suffix('.db.lock')
|
|
try:
|
|
lock_file.parent.mkdir(parents=True, exist_ok=True)
|
|
with lock_file.open('w') as fh:
|
|
try:
|
|
fcntl.flock(fh, fcntl.LOCK_EX | fcntl.LOCK_NB)
|
|
except OSError:
|
|
yield False
|
|
return
|
|
try:
|
|
yield True
|
|
finally:
|
|
fcntl.flock(fh, fcntl.LOCK_UN)
|
|
except OSError:
|
|
# Cannot create the lock file (media directory missing/read-only); let the caller try.
|
|
yield True
|