diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py index 42f778d2..89d9192e 100644 --- a/modules/videos/__init__.py +++ b/modules/videos/__init__.py @@ -1,10 +1,12 @@ import asyncio import json +import pathlib from typing import Callable, List, Tuple from sqlalchemy import or_ from sqlalchemy.orm import Session +from modules.videos.common import get_or_create_ffprobe_json from modules.videos.models import Video from wrolpi.common import logger, limit_concurrent, register_modeler, register_refresh_cleanup from wrolpi.db import get_db_curs, get_db_session @@ -23,18 +25,37 @@ VIDEO_PROCESSING_LIMIT = 20 async def video_modeler(progress_callback: Callable[[int], None] = None): total_processed = 0 while True: - with get_db_session(commit=True) as session: - file_groups = session.query(FileGroup, Video).filter( + # Read the batch; nothing is claimed yet, so the write lock stays free while ffprobe runs. + with get_db_session() as session: + batch: List[Tuple[int, pathlib.Path]] = list(session.query(FileGroup.id, FileGroup.primary_path).filter( FileGroup.indexed != True, or_(FileGroup.mimetype.like('video/%'), FileGroup.mimetype.like('audio/%')), - ).outerjoin(Video, Video.file_group_id == FileGroup.id) \ - .limit(VIDEO_PROCESSING_LIMIT) - file_groups: List[Tuple[FileGroup, Video]] = list(file_groups) + ).limit(VIDEO_PROCESSING_LIMIT).all()) + + if not batch: + break + + # ffprobe each file with no transaction open. This is a subprocess per video and can run + # for seconds; inside the write transaction below it would hold the write lock for the whole + # batch, and every other writer on the box would wait out `busy_timeout` (30s). + probed = dict() + for file_group_id, primary_path in batch: + try: + probed[file_group_id] = await get_or_create_ffprobe_json(pathlib.Path(str(primary_path))) + except Exception as e: + if PYTEST: + raise + logger.error(f'Unable to ffprobe Video: {primary_path}', exc_info=e) + # Sleep to catch cancel. + await asyncio.sleep(0) + + with get_db_session(commit=True) as session: + file_groups: List[Tuple[FileGroup, Video]] = list( + session.query(FileGroup, Video) + .filter(FileGroup.id.in_([i for i, _ in batch])) + .outerjoin(Video, Video.file_group_id == FileGroup.id)) - processed = 0 for file_group, video in file_groups: - processed += 1 - video_id = None try: if not video: @@ -45,12 +66,16 @@ async def video_modeler(progress_callback: Callable[[int], None] = None): if not Session.object_session(video): session.add(video) video.flush() - # Extract ffprobe data. - await video.get_ffprobe_json() + # Store the ffprobe data gathered above. + if (result := probed.get(file_group.id)) is not None: + video.ffprobe_json, ffprobe_file = result + if ffprobe_file: + # Track the .ffprobe.json cache file that was just written. + file_group.append_files(ffprobe_file) video.flush(session) - # Validate and index subtitles. + # Validate and index subtitles. (Poster generation happens here when a Channel + # asks for it; it is the remaining slow work inside this transaction.) video.validate(session) - processed += 1 except Exception as e: if PYTEST: raise @@ -59,18 +84,16 @@ async def video_modeler(progress_callback: Callable[[int], None] = None): file_group.indexed = True - session.commit() + # Report batch progress + total_processed += len(batch) + if progress_callback: + progress_callback(total_processed) - # Report batch progress - total_processed += len(file_groups) - if progress_callback and len(file_groups) > 0: - progress_callback(total_processed) + logger.debug(f'Modeled {len(batch)} videos') - if processed < VIDEO_PROCESSING_LIMIT: - # Did not reach limit, do not query again. - break - - logger.debug(f'Modeled {processed} videos') + if len(batch) < VIDEO_PROCESSING_LIMIT: + # Did not reach limit, do not query again. + break # Sleep to catch cancel. await asyncio.sleep(0) @@ -127,8 +150,9 @@ _UNCLAIMED_VIDEOS_SQL = ''' SELECT v.id, c.id FROM video v INNER JOIN file_group fg ON fg.id = v.file_group_id - INNER JOIN collection col ON fg.directory = col.directory - OR fg.directory LIKE col.directory || '/%' + INNER JOIN collection col + ON (fg.directory = col.directory + OR fg.directory LIKE col.directory || '/%') INNER JOIN channel c ON c.collection_id = col.id WHERE v.channel_id IS NULL ''' diff --git a/modules/videos/common.py b/modules/videos/common.py index d1f4eaad..84e86b75 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -319,6 +319,31 @@ def write_ffprobe_json_file(path: Path, data: dict): json.dump(data, f, indent=2) +async def get_or_create_ffprobe_json(video_path: Path) -> Tuple[dict, Optional[Path]]: + """Return `(ffprobe data, the cache file written)` for a video file. + + Deliberately takes a path and touches no session: ffprobe is a subprocess that runs for + seconds, and a caller holding a write transaction while it runs blocks every other writer + until `busy_timeout` expires. The caller persists the returned data afterwards, in a short + write transaction of its own. + """ + ffprobe_path = video_path.with_suffix('.ffprobe.json') + if ffprobe_path.is_file() and (cached_data := read_ffprobe_json_file(ffprobe_path)): + return cached_data, None + + data = await ffprobe_json(video_path) + + written = None + from wrolpi.common import get_wrolpi_config + if get_wrolpi_config().save_ffprobe_json: + try: + write_ffprobe_json_file(ffprobe_path, data) + written = ffprobe_path + except IOError as e: + logger.warning(f'Failed to write ffprobe json cache file {ffprobe_path}', exc_info=e) + return data, written + + def extract_video_duration(video_path: Path) -> Optional[int]: """Get the duration of a video in seconds. Do this using ffprobe.""" if not isinstance(video_path, Path): diff --git a/modules/videos/models.py b/modules/videos/models.py index 6d75256a..ee9a48b1 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -413,28 +413,13 @@ class Video(ModelHelper, Base): if self.ffprobe_json: return self.ffprobe_json - from modules.videos.common import ffprobe_json, read_ffprobe_json_file, write_ffprobe_json_file + from modules.videos.common import get_or_create_ffprobe_json - # 2. Check if .ffprobe.json file exists - ffprobe_path = self.ffprobe_json_path - if ffprobe_path and ffprobe_path.is_file(): - cached_data = read_ffprobe_json_file(ffprobe_path) - if cached_data: - self.ffprobe_json = cached_data - return self.ffprobe_json - - # 3. Run ffprobe and cache the result - self.ffprobe_json = await ffprobe_json(self.video_path) - if ffprobe_path: - # Check if saving ffprobe json files is enabled - from wrolpi.common import get_wrolpi_config - if get_wrolpi_config().save_ffprobe_json: - try: - write_ffprobe_json_file(ffprobe_path, self.ffprobe_json) - # Add the new file to the FileGroup so it's tracked - self.file_group.append_files(ffprobe_path) - except IOError as e: - logger.warning(f'Failed to write ffprobe json cache file {ffprobe_path}', exc_info=e) + # 2/3. Read the cached .ffprobe.json, or run ffprobe and cache it. + self.ffprobe_json, written = await get_or_create_ffprobe_json(self.video_path) + if written: + # Track the new cache file in the FileGroup. + self.file_group.append_files(written) return self.ffprobe_json diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 86607c66..0e66fcf1 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -9,6 +9,7 @@ from PIL import Image from modules.videos.models import Channel, Video from wrolpi.collections import Collection +from wrolpi.conftest import probe_write_lock_is_held, production_like_sessions from wrolpi.common import get_absolute_media_path, get_wrolpi_config, get_relative_to_media_directory from wrolpi.downloader import Download, DownloadFrequency from wrolpi.vars import PROJECT_DIR @@ -608,3 +609,38 @@ def test_get_custom_videos_directory(test_directory, test_wrolpi_config): assert common.get_videos_directory() == (test_directory / 'custom/directory/videos') assert (test_directory / 'custom/directory/videos').is_dir() assert not (test_directory / 'videos').is_dir() + + +@pytest.mark.asyncio +async def test_video_modeler_does_not_hold_write_lock_during_ffprobe(async_client, test_session, video_factory): + """Modeling must not hold the SQLite write lock while ffprobe runs. + + `video_modeler` opened one write session per batch and `await`ed `get_ffprobe_json` -- an ffprobe + subprocess -- for up to 20 videos inside it. Every other writer on the box waits on that lock + and gives up after `busy_timeout` (30s). ffprobe belongs outside the transaction: read the + batch, probe the files, then write what was learned. + """ + import modules.videos as videos_module + from modules.videos import video_modeler + + video = video_factory(with_video_file=True) + video.file_group.indexed = False + video.file_group.model = None + test_session.delete(video) + test_session.commit() + + db_file = test_session.get_bind().url.database + lock_held_during_ffprobe = [] + real_get_or_create_ffprobe_json = videos_module.get_or_create_ffprobe_json + + async def probing_ffprobe(video_path): + lock_held_during_ffprobe.append(probe_write_lock_is_held(db_file)) + return await real_get_or_create_ffprobe_json(video_path) + + with production_like_sessions(test_session): + with mock.patch.object(videos_module, 'get_or_create_ffprobe_json', probing_ffprobe): + await video_modeler() + + assert lock_held_during_ffprobe, 'ffprobe never ran, so the test proves nothing' + assert not any(lock_held_during_ffprobe), \ + 'the write lock was held while ffprobe ran; every other writer waits out busy_timeout' diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py index 89f04f2e..746794dc 100644 --- a/wrolpi/conftest.py +++ b/wrolpi/conftest.py @@ -400,6 +400,28 @@ def make_test_ctx(*, is_cancelled=None, can_download=None, ) +def probe_write_lock_is_held(db_file: str) -> bool: + """Return True if a competing connection cannot immediately take the write lock. + + Uses a zero busy-timeout so the probe fails instantly if another connection holds a + RESERVED (write) lock, rather than waiting. In WAL mode a plain reader holds no such + lock, so this only returns True when a writer transaction is actually in progress. + """ + probe = sqlite3.connect(db_file, timeout=0) + try: + probe.execute('PRAGMA busy_timeout=0') + try: + probe.execute('BEGIN IMMEDIATE') + except sqlite3.OperationalError as e: + if 'database is locked' in str(e): + return True + raise + probe.execute('ROLLBACK') + return False + finally: + probe.close() + + @contextlib.contextmanager def write_lock_held_briefly(db_file: str, seconds: float = 1.0): """Hold the SQLite write lock on `db_file` for `seconds`, from another connection. diff --git a/wrolpi/db.py b/wrolpi/db.py index 4d2a7c4b..a1aa5d75 100644 --- a/wrolpi/db.py +++ b/wrolpi/db.py @@ -204,7 +204,18 @@ def get_db_session(commit: bool = False) -> Generator[Session, Any, None]: readers never block, and `commit=False` sessions stay deferred. """ _, session = get_db_context() - token = _immediate_txn.set(True) if commit else None + if commit: + # Open the transaction here rather than letting it begin lazily at the caller's first + # statement. Two reasons: the write lock is taken up front (the whole point), and the + # `_immediate_txn` flag lives only across this synchronous call. A flag held for the + # session's lifetime would be copied into every task the caller spawns -- + # `asyncio.create_task` snapshots the Context -- and the parent's `reset()` cannot reach a + # child's copy, so those tasks would issue BEGIN IMMEDIATE for their *read* sessions. + token = _immediate_txn.set(True) + try: + session.connection() + finally: + _immediate_txn.reset(token) try: yield session if commit: @@ -213,8 +224,6 @@ def get_db_session(commit: bool = False) -> Generator[Session, Any, None]: session.rollback() raise finally: - if token is not None: - _immediate_txn.reset(token) # Rollback only if a transaction hasn't been committed. # In tests, the test_session fixture manages the session lifecycle, # so we should not rollback here - that would undo other test operations. diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 8e609712..a5da35c6 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -1348,23 +1348,39 @@ class DownloadManager: """Mark any recurring downloads that are due for download as "new". Start a download.""" now_ = now() - # `commit=True` (a writer from BEGIN): this reads the recurring Downloads and then renews - # the due ones. As a read session its commit failed with "database is locked" whenever - # anything else wrote while it was reading -- a config import or refresh, constantly. - with get_db_session(commit=True) as session: - recurring = self.get_recurring_downloads(session) - renewed_count = 0 - for download in recurring: + # Decide what is due in a read session. This runs every download-manager cycle and scans + # every recurring Download -- with a `calculate_next_download` query per row that lacks one + # -- so it must not hold the write lock, least of all on the usual cycle where nothing is + # due. WAL readers block nobody. + due_ids = list() + calculated = dict() + with get_db_session() as session: + for download in self.get_recurring_downloads(session): # A new download may not have a `next_download`, create it if necessary. - download.next_download = download.next_download or self.calculate_next_download(session, download) - if download.next_download < now_ and download.status not in ( + next_download = download.next_download or self.calculate_next_download(session, download) + if next_download != download.next_download: + calculated[download.id] = next_download + if next_download < now_ and download.status not in ( DownloadStatus.new, DownloadStatus.pending): + due_ids.append(download.id) + + if not due_ids and not calculated: + return + + # Take the write lock only for the rows that change. A row's status may have moved on since + # the read above (the dispatcher claims downloads constantly), so re-check it here. + due = set(due_ids) + with get_db_session(commit=True) as session: + renewed_count = 0 + for download in session.query(Download).filter(Download.id.in_(due | set(calculated))): + if (next_download := calculated.get(download.id)) is not None: + download.next_download = download.next_download or next_download + if download.id in due and download.status not in (DownloadStatus.new, DownloadStatus.pending): download.renew() renewed_count += 1 if renewed_count: self.log_debug(f'Renewed {renewed_count} recurring downloads') - session.commit() @staticmethod def get_downloads(session: Session) -> List[Download]: diff --git a/wrolpi/test/test_db.py b/wrolpi/test/test_db.py index a2eb03a4..dd1a62f0 100644 --- a/wrolpi/test/test_db.py +++ b/wrolpi/test/test_db.py @@ -1,6 +1,9 @@ +import asyncio import sqlite3 -from wrolpi.conftest import production_like_sessions +import pytest + +from wrolpi.conftest import production_like_sessions, probe_write_lock_is_held from wrolpi.db import get_db_session, _configure_sqlite_connection @@ -98,28 +101,6 @@ def test_configure_connection_does_not_reattempt_wal_once_ruled_out(): assert 'PRAGMA journal_mode=TRUNCATE' in conn._cursor.executed -def _probe_write_lock_is_held(db_file: str) -> bool: - """Return True if a competing connection cannot immediately take the write lock. - - Uses a zero busy-timeout so the probe fails instantly if another connection holds a - RESERVED (write) lock, rather than waiting. In WAL mode a plain reader holds no such - lock, so this only returns True when a writer transaction is actually in progress. - """ - probe = sqlite3.connect(db_file, timeout=0) - try: - probe.execute('PRAGMA busy_timeout=0') - try: - probe.execute('BEGIN IMMEDIATE') - except sqlite3.OperationalError as e: - if 'database is locked' in str(e): - return True - raise - probe.execute('ROLLBACK') - return False - finally: - probe.close() - - def test_write_session_takes_write_lock_up_front(test_session): """`get_db_session(commit=True)` must acquire the SQLite write lock at BEGIN. @@ -137,7 +118,7 @@ def test_write_session_takes_write_lock_up_front(test_session): # Trigger the transaction's BEGIN with a trivial read (as the download dispatcher # does before writing). This session already holds the write lock. session.execute('SELECT 1') - assert _probe_write_lock_is_held(db_file), \ + assert probe_write_lock_is_held(db_file), \ 'immediate session did not hold the write lock after BEGIN' @@ -148,5 +129,34 @@ def test_read_session_does_not_take_write_lock(test_session): with production_like_sessions(test_session): with get_db_session() as session: session.execute('SELECT 1') - assert not _probe_write_lock_is_held(db_file), \ + assert not probe_write_lock_is_held(db_file), \ 'read-only session unexpectedly holds the write lock' + + +@pytest.mark.asyncio +async def test_background_task_does_not_inherit_write_intent(test_session): + """A task spawned while a write session is open must not inherit its write intent. + + `asyncio.create_task` copies the current Context, so a ContextVar set for the duration of a + write session is inherited by every background task the session's code starts -- permanently, + because the parent's `reset()` cannot reach the child's copy. `create_downloads` spawns + `dispatch_downloads` exactly that way, so every user-created download would leave that task + issuing BEGIN IMMEDIATE for its *read* sessions, serializing reads behind the write lock. + """ + db_file = test_session.get_bind().url.database + held_during_child_read = dict() + + async def child(): + with get_db_session() as session: # A read session: it must stay deferred. + session.execute('SELECT 1') + held_during_child_read['held'] = probe_write_lock_is_held(db_file) + + with production_like_sessions(test_session): + with get_db_session(commit=True) as session: + session.execute('SELECT 1') + task = asyncio.create_task(child()) + # The write session has committed; only the child's own session can hold the lock now. + await task + + assert held_during_child_read['held'] is False, \ + 'a read session in a spawned task took the write lock, inheriting the parent write intent' diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 447db344..ee9b00c6 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -16,6 +16,7 @@ from sqlalchemy.exc import OperationalError from wrolpi.api_utils import api_app from wrolpi.common import get_wrolpi_config, normalize_domain from wrolpi.dates import Seconds, now +from wrolpi.conftest import probe_write_lock_is_held, production_like_sessions from wrolpi.db import get_db_context from wrolpi.downloader import Downloader, Download, DownloadFrequency, import_downloads_config, \ get_download_manager_config, RSSDownloader, parse_aria2c_progress, _parse_size, \ @@ -1713,35 +1714,35 @@ async def test_download_dispatched_outside_immediate_transaction(test_session, t """The download signal must be dispatched AFTER the claiming write transaction commits. Regression test for a production deadlock (found on 10.0.0.9, where downloads silently stopped): - `_dispatch_new_downloads` claimed downloads inside a write (BEGIN - IMMEDIATE) transaction and `await`ed `api_app.dispatch(...)` while that transaction was still - open. Under NullPool (production) the dispatched `signal_download_download` opens its OWN - connection and — inheriting the `_immediate_txn` flag — also issues BEGIN IMMEDIATE, which - blocks on the write lock the outer transaction still holds until `busy_timeout` (30s) expires: - `database is locked`, and no download ever starts. + `_dispatch_new_downloads` claimed downloads inside its write transaction and `await`ed + `api_app.dispatch(...)` while that transaction was still open. Under NullPool (production) the + dispatched `signal_download_download` opens its OWN connection and also issues BEGIN IMMEDIATE, + which blocks on the write lock the outer transaction still holds until `busy_timeout` (30s) + expires: `database is locked`, and no download ever starts. - The test database shares a single connection, so it cannot reproduce the lock itself; instead it - asserts the invariant that prevents it — the signal is dispatched with no immediate transaction - active.""" - from wrolpi.db import _immediate_txn + `production_like_sessions` gives the dispatcher its own connection, so the write lock is real + and a competing connection can probe for it at dispatch time. + """ name = test_downloader.name d = Download(url='https://example.com/new', downloader=name, status='new') test_session.add(d) test_session.commit() + db_file = test_session.get_bind().url.database - immediate_active_at_dispatch = [] + write_lock_held_at_dispatch = [] async def record(event, *args, **kwargs): if event == 'wrolpi.download.download': - immediate_active_at_dispatch.append(getattr(_immediate_txn, 'active', False)) + write_lock_held_at_dispatch.append(probe_write_lock_is_held(db_file)) - with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: - dispatch.side_effect = record - await test_download_manager.dispatch_downloads() + with production_like_sessions(test_session): + with mock.patch.object(type(api_app), 'dispatch', new_callable=mock.AsyncMock) as dispatch: + dispatch.side_effect = record + await test_download_manager.dispatch_downloads() - assert immediate_active_at_dispatch, 'expected the download to be dispatched' - assert not any(immediate_active_at_dispatch), \ - 'download signal was dispatched inside an open BEGIN IMMEDIATE transaction (deadlocks under NullPool)' + assert write_lock_held_at_dispatch, 'expected the download to be dispatched' + assert not any(write_lock_held_at_dispatch), \ + 'download signal was dispatched while the claim still held the write lock (deadlocks under NullPool)' @pytest.mark.asyncio @@ -1974,3 +1975,37 @@ async def test_recurring_downloads_bypass_daily_limits(test_session, test_downlo # Recurring downloads are excluded from the daily count entirely. global_count, domain_counts = test_download_manager.daily_download_counts(test_session) assert 'rumble.com' not in domain_counts + + +@pytest.mark.asyncio +async def test_renew_recurring_downloads_scans_without_the_write_lock(test_session, test_download_manager, + test_downloader): + """Deciding what is due must not hold the write lock. + + The download manager calls this every cycle. Scanning every recurring Download (plus a + `calculate_next_download` query per row) inside the write transaction blocks every other writer + for the length of the scan, even on the common cycle where nothing is due. Read first, then + take the lock only for the rows that are actually being renewed. + """ + db_file = test_session.get_bind().url.database + # A recurring Download that is NOT due, so nothing needs renewing. + d = Download(url='https://example.com/recurring', downloader=test_downloader.name, status='complete', + frequency=DownloadFrequency.weekly, next_download=now() + timedelta(days=3)) + test_session.add(d) + test_session.commit() + + lock_held_during_scan = [] + real_get_recurring_downloads = type(test_download_manager).get_recurring_downloads + + def probing_get_recurring_downloads(self, session, limit: int = None): + lock_held_during_scan.append(probe_write_lock_is_held(db_file)) + return real_get_recurring_downloads(self, session, limit) + + with production_like_sessions(test_session): + with mock.patch.object(type(test_download_manager), 'get_recurring_downloads', + probing_get_recurring_downloads): + test_download_manager.renew_recurring_downloads() + + assert lock_held_during_scan, 'the recurring downloads were never scanned' + assert not any(lock_held_during_scan), \ + 'the write lock was held while scanning recurring downloads'