diff --git a/alembic/versions/2026_07_14_0900_browse_covering_index.py b/alembic/versions/2026_07_14_0900_browse_covering_index.py new file mode 100644 index 00000000..758dba56 --- /dev/null +++ b/alembic/versions/2026_07_14_0900_browse_covering_index.py @@ -0,0 +1,27 @@ +"""Covering index for browse pages. + +The videos/files browse queries filter on mimetype and order by effective_datetime; without a +covering index SQLite scans every file_group row (which carries the indexed texts inline), which +takes ~18 seconds cold on a Pi. With (mimetype, effective_datetime) the filter, count, and sort +resolve index-only. + +Revision ID: 2026_07_14_0900 +Revises: 2026_07_10_1350 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_07_14_0900' +down_revision = '2026_07_10_1350' +branch_labels = None +depends_on = None + + +def upgrade(): + # IF NOT EXISTS: QA boxes may have had the index created manually before this migration. + op.execute('CREATE INDEX IF NOT EXISTS file_group_mimetype_effective_idx ' + 'ON file_group (mimetype, effective_datetime)') + + +def downgrade(): + op.execute('DROP INDEX IF EXISTS file_group_mimetype_effective_idx') diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 7246aaf0..e1422c09 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -1376,8 +1376,8 @@ ARCHIVE_ORDERS = { '-download_datetime': 'fg.download_datetime DESC NULLS LAST', 'rank': f'2 DESC, {_date}', '-rank': f'2 ASC, {date}', - 'size': 'fg.size ASC, LOWER(fg.primary_path) ASC', - '-size': 'fg.size DESC NULLS LAST, LOWER(fg.primary_path) DESC', + 'size': 'fg.size ASC, fg.id ASC', + '-size': 'fg.size DESC NULLS LAST, fg.id DESC', 'viewed': 'fg.viewed ASC', '-viewed': 'fg.viewed DESC', } @@ -1444,15 +1444,24 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order select_columns = f"{select_columns}," if select_columns else "" wheres = '\n AND '.join(wheres) where = f'WHERE\n{wheres}' if wheres else '' + if where or fts_join: + from_clause = f'''FROM archive a + LEFT JOIN file_group fg ON fg.id = a.file_group_id + {fts_join}''' + else: + # Browse (no filters): CROSS JOIN pins file_group as the outer table so SQLite walks its + # date index and probes archive's covering index -- both sides stay index-only. Driving + # from archive probes every (large) file_group row for the sort key, which takes ~90 + # seconds cold on a Pi with 124k archives. + from_clause = 'FROM file_group fg CROSS JOIN archive a ON a.file_group_id = fg.id' + stmt = f''' SELECT a.file_group_id AS id, -- always get `file_group.id` for `handle_file_group_search_results` {select_columns} COUNT(*) OVER() AS total {headline_columns} - FROM archive a - LEFT JOIN file_group fg ON fg.id = a.file_group_id - {fts_join} + {from_clause} {where} ORDER BY {order_by} LIMIT :limit OFFSET :offset diff --git a/modules/videos/test/test_api.py b/modules/videos/test/test_api.py index fe71b97c..fc3b8284 100644 --- a/modules/videos/test/test_api.py +++ b/modules/videos/test/test_api.py @@ -257,9 +257,10 @@ async def test_search_videos(test_session, video_factory, assert_video_search, s offset=1) await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime', channel_id=simple_channel.id) - await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name]) + # No search_str, so the default 'rank' order is a constant; ties fall to fg.id DESC. + await assert_video_search(assert_total=2, assert_ids=[vid2.id, vid1.id], tag_names=[tag1.name]) await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name]) - await assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1) + await assert_video_search(assert_total=2, assert_ids=[vid2.id], tag_names=[tag1.name], limit=1) # Only vid2 has both tags. await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name]) diff --git a/modules/videos/video/lib.py b/modules/videos/video/lib.py index ac87cfec..b497af9b 100644 --- a/modules/videos/video/lib.py +++ b/modules/videos/video/lib.py @@ -50,23 +50,25 @@ def get_video(file_group_id: int) -> Video: VIDEO_ORDERS = { + # fg.id (the rowid) is the pagination tiebreaker: it is present in every index, so these + # orders stay index-only. LOWER(fg.primary_path) forced SQLite to read every (large) row. # Sometimes we don't have a published_datetime. This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime) - 'published_datetime': 'fg.effective_datetime ASC, LOWER(fg.primary_path) ASC', - '-published_datetime': 'fg.effective_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC', - 'rank': '2 DESC, LOWER(fg.primary_path) DESC', - '-rank': '2 ASC, LOWER(fg.primary_path) ASC', - 'size': 'fg.size ASC, LOWER(fg.primary_path) ASC', - '-size': 'fg.size DESC, LOWER(fg.primary_path) DESC', - 'length': 'fg.length ASC, LOWER(fg.primary_path) ASC', - '-length': 'fg.length DESC, LOWER(fg.primary_path) DESC', - 'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, LOWER(fg.primary_path) ASC', - '-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, LOWER(fg.primary_path) DESC', + 'published_datetime': 'fg.effective_datetime ASC, fg.id ASC', + '-published_datetime': 'fg.effective_datetime DESC NULLS LAST, fg.id ASC', + 'rank': '2 DESC, fg.id DESC', + '-rank': '2 ASC, fg.id ASC', + 'size': 'fg.size ASC, fg.id ASC', + '-size': 'fg.size DESC, fg.id DESC', + 'length': 'fg.length ASC, fg.id ASC', + '-length': 'fg.length DESC, fg.id DESC', + 'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, fg.id ASC', + '-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, fg.id DESC', 'viewed': 'fg.viewed ASC', '-viewed': 'fg.viewed DESC', 'view_count': 'v.view_count ASC', '-view_count': 'v.view_count DESC', - 'download_datetime': 'fg.download_datetime ASC, LOWER(fg.primary_path) ASC', - '-download_datetime': 'fg.download_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC', + 'download_datetime': 'fg.download_datetime ASC, fg.id ASC', + '-download_datetime': 'fg.download_datetime DESC NULLS LAST, fg.id ASC', } NO_NULL_ORDERS = { 'viewed': 'fg.viewed IS NOT NULL', diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index 1d37f436..b5656f88 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -72,6 +72,9 @@ class FileGroup(ModelHelper, Base): Index('file_group_download_datetime_idx', 'download_datetime'), Index('file_group_effective_datetime_idx', 'effective_datetime'), Index('file_group_length_idx', 'length'), + # Covering index for browse pages: mimetype filter + date order resolve index-only + # instead of scanning every (large) file_group row. + Index('file_group_mimetype_effective_idx', 'mimetype', 'effective_datetime'), Index('file_group_mimetype_idx', 'mimetype'), Index('file_group_model_idx', 'model'), Index('file_group_modification_datetime_idx', 'modification_datetime'), diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index 445f7eee..88e9c32c 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -1269,6 +1269,12 @@ class FileWorker: # Set refresh_complete flag only when refreshing the entire media directory if is_global_refresh: + # Refresh statistics for the query planner; without sqlite_stat1 (or after the table + # contents change wholesale) SQLite picks full scans over the covering indexes. + from wrolpi.db import get_db_curs + with get_db_curs(commit=True) as curs: + curs.execute('PRAGMA optimize') + flags.refresh_complete.set() flags.global_refresh_active.clear() # After a DB rebuild, playlists.yaml was imported before any files were indexed, so its