SQLite: make the browse pages index-only.

Cold archive/video browsing took 18-90 seconds on a Pi: file_group rows
carry their indexed texts inline (SQLite has no out-of-line TOAST like
Postgres), so any plan that reads file_group rows to filter or sort
drags gigabytes of text pages through USB.

- Browse archives with file_group as the outer table (CROSS JOIN) so
  SQLite walks the date index and probes archive's covering index;
  driving from archive probed every file_group row for the sort key.
- Tie-break orders with fg.id instead of LOWER(fg.primary_path); the
  rowid is present in every index, the path expression forced full-row
  reads (and a full scan on the videos browse).
- Add a (mimetype, effective_datetime) covering index so the videos
  mimetype filter + date order resolve index-only.
- PRAGMA optimize after a global refresh; the planner had no
  sqlite_stat1 at all after the rebuild.

Warm timings on 10.0.0.9 (124k archives / 24k videos): archive browse
0.4s -> 0.22s index-only, videos browse 0.38s -> 0.07s; cold I/O drops
from ~124k random fat-page probes to a few MB of contiguous index.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>

alembic/versions/2026_07_14_0900_browse_covering_index.py

@@ -0,0 +1,27 @@
+"""Covering index for browse pages.
+
+The videos/files browse queries filter on mimetype and order by effective_datetime; without a
+covering index SQLite scans every file_group row (which carries the indexed texts inline), which
+takes ~18 seconds cold on a Pi.  With (mimetype, effective_datetime) the filter, count, and sort
+resolve index-only.
+
+Revision ID: 2026_07_14_0900
+Revises: 2026_07_10_1350
+"""
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '2026_07_14_0900'
+down_revision = '2026_07_10_1350'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    # IF NOT EXISTS: QA boxes may have had the index created manually before this migration.
+    op.execute('CREATE INDEX IF NOT EXISTS file_group_mimetype_effective_idx '
+               'ON file_group (mimetype, effective_datetime)')
+
+
+def downgrade():
+    op.execute('DROP INDEX IF EXISTS file_group_mimetype_effective_idx')

modules/archive/lib.py

@@ -1376,8 +1376,8 @@ ARCHIVE_ORDERS = {
     '-download_datetime': 'fg.download_datetime DESC NULLS LAST',
     'rank': f'2 DESC, {_date}',
     '-rank': f'2 ASC, {date}',
-    'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
-    '-size': 'fg.size DESC NULLS LAST, LOWER(fg.primary_path) DESC',
+    'size': 'fg.size ASC, fg.id ASC',
+    '-size': 'fg.size DESC NULLS LAST, fg.id DESC',
     'viewed': 'fg.viewed ASC',
     '-viewed': 'fg.viewed DESC',
 }
@@ -1444,15 +1444,24 @@ def search_archives(search_str: str, domain: str, limit: int, offset: int, order
     select_columns = f"{select_columns}," if select_columns else ""
     wheres = '\n AND '.join(wheres)
     where = f'WHERE\n{wheres}' if wheres else ''
+    if where or fts_join:
+        from_clause = f'''FROM archive a
+            LEFT JOIN file_group fg ON fg.id = a.file_group_id
+            {fts_join}'''
+    else:
+        # Browse (no filters): CROSS JOIN pins file_group as the outer table so SQLite walks its
+        # date index and probes archive's covering index -- both sides stay index-only.  Driving
+        # from archive probes every (large) file_group row for the sort key, which takes ~90
+        # seconds cold on a Pi with 124k archives.
+        from_clause = 'FROM file_group fg CROSS JOIN archive a ON a.file_group_id = fg.id'
+
     stmt = f'''
             SELECT
                 a.file_group_id AS id, -- always get `file_group.id` for `handle_file_group_search_results`
                 {select_columns}
                 COUNT(*) OVER() AS total
                 {headline_columns}
-            FROM archive a
-            LEFT JOIN file_group fg ON fg.id = a.file_group_id
-            {fts_join}
+            {from_clause}
             {where}
             ORDER BY {order_by}
             LIMIT :limit OFFSET :offset

modules/videos/test/test_api.py

@@ -257,9 +257,10 @@ async def test_search_videos(test_session, video_factory, assert_video_search, s
                               offset=1)
     await assert_video_search(assert_total=1, assert_ids=[vid2.id], order_by='-published_datetime',
                               channel_id=simple_channel.id)
-    await assert_video_search(assert_total=2, assert_ids=[vid1.id, vid2.id], tag_names=[tag1.name])
+    # No search_str, so the default 'rank' order is a constant; ties fall to fg.id DESC.
+    await assert_video_search(assert_total=2, assert_ids=[vid2.id, vid1.id], tag_names=[tag1.name])
     await assert_video_search(assert_total=2, assert_ids=[vid3.id, vid2.id], tag_names=[tag2.name])
-    await assert_video_search(assert_total=2, assert_ids=[vid1.id], tag_names=[tag1.name], limit=1)
+    await assert_video_search(assert_total=2, assert_ids=[vid2.id], tag_names=[tag1.name], limit=1)
     # Only vid2 has both tags.
     await assert_video_search(assert_total=1, assert_ids=[vid2.id], tag_names=[tag1.name, tag2.name])
 

modules/videos/video/lib.py

@@ -50,23 +50,25 @@ def get_video(file_group_id: int) -> Video:
 
 
 VIDEO_ORDERS = {
+    # fg.id (the rowid) is the pagination tiebreaker: it is present in every index, so these
+    # orders stay index-only.  LOWER(fg.primary_path) forced SQLite to read every (large) row.
     # Sometimes we don't have a published_datetime.  This is equivalent to COALESCE(fg.published_datetime, fg.download_datetime)
-    'published_datetime': 'fg.effective_datetime ASC, LOWER(fg.primary_path) ASC',
-    '-published_datetime': 'fg.effective_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
-    'rank': '2 DESC, LOWER(fg.primary_path) DESC',
-    '-rank': '2 ASC, LOWER(fg.primary_path) ASC',
-    'size': 'fg.size ASC, LOWER(fg.primary_path) ASC',
-    '-size': 'fg.size DESC, LOWER(fg.primary_path) DESC',
-    'length': 'fg.length ASC, LOWER(fg.primary_path) ASC',
-    '-length': 'fg.length DESC, LOWER(fg.primary_path) DESC',
-    'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, LOWER(fg.primary_path) ASC',
-    '-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, LOWER(fg.primary_path) DESC',
+    'published_datetime': 'fg.effective_datetime ASC, fg.id ASC',
+    '-published_datetime': 'fg.effective_datetime DESC NULLS LAST, fg.id ASC',
+    'rank': '2 DESC, fg.id DESC',
+    '-rank': '2 ASC, fg.id ASC',
+    'size': 'fg.size ASC, fg.id ASC',
+    '-size': 'fg.size DESC, fg.id DESC',
+    'length': 'fg.length ASC, fg.id ASC',
+    '-length': 'fg.length DESC, fg.id DESC',
+    'size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END ASC, fg.id ASC',
+    '-size_to_duration': 'CASE WHEN fg.length > 0 THEN CAST(fg.size AS REAL) / fg.length ELSE fg.size END DESC, fg.id DESC',
     'viewed': 'fg.viewed ASC',
     '-viewed': 'fg.viewed DESC',
     'view_count': 'v.view_count ASC',
     '-view_count': 'v.view_count DESC',
-    'download_datetime': 'fg.download_datetime ASC, LOWER(fg.primary_path) ASC',
-    '-download_datetime': 'fg.download_datetime DESC NULLS LAST, LOWER(fg.primary_path) ASC',
+    'download_datetime': 'fg.download_datetime ASC, fg.id ASC',
+    '-download_datetime': 'fg.download_datetime DESC NULLS LAST, fg.id ASC',
 }
 NO_NULL_ORDERS = {
     'viewed': 'fg.viewed IS NOT NULL',

wrolpi/files/models.py

@@ -72,6 +72,9 @@ class FileGroup(ModelHelper, Base):
         Index('file_group_download_datetime_idx', 'download_datetime'),
         Index('file_group_effective_datetime_idx', 'effective_datetime'),
         Index('file_group_length_idx', 'length'),
+        # Covering index for browse pages: mimetype filter + date order resolve index-only
+        # instead of scanning every (large) file_group row.
+        Index('file_group_mimetype_effective_idx', 'mimetype', 'effective_datetime'),
         Index('file_group_mimetype_idx', 'mimetype'),
         Index('file_group_model_idx', 'model'),
         Index('file_group_modification_datetime_idx', 'modification_datetime'),

wrolpi/files/worker.py

@@ -1269,6 +1269,12 @@ class FileWorker:
 
         # Set refresh_complete flag only when refreshing the entire media directory
         if is_global_refresh:
+            # Refresh statistics for the query planner; without sqlite_stat1 (or after the table
+            # contents change wholesale) SQLite picks full scans over the covering indexes.
+            from wrolpi.db import get_db_curs
+            with get_db_curs(commit=True) as curs:
+                curs.execute('PRAGMA optimize')
+
             flags.refresh_complete.set()
             flags.global_refresh_active.clear()
             # After a DB rebuild, playlists.yaml was imported before any files were indexed, so its