From 7471b41ae1bb457f6a297067f5b7e2e3c40b383e Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Sun, 13 Sep 2026 12:38:48 -0600 Subject: [PATCH] docs: browse walks the file_group date index instead of probing every doc (#707) * docs: browse walks the file_group date index instead of probing every doc The unfiltered /api/docs/search drove its join from `doc`, so SQLite probed every doc's file_group row for the sort key before returning the first page. With 20k docs scattered through a 10 GB database on a spinning disk, the first request took about a minute cold (0.07s warm). Use the same CROSS JOIN shape as the archive browse: file_group is the outer table, SQLite walks file_group_published_datetime_idx and probes doc's covering unique index on file_group_id. Both sides stay index-only and the scan stops at the page limit. Measured cold: ~60s -> 0.6s, identical rows. Filtered searches and the size/title orderings keep the ORM query. Co-Authored-By: Claude Fable 5.1 * docs: trim browse comments, stabler plan assertion Review feedback: drop the benchmark narrative from docstrings, state only the non-obvious constraint at the call site, and assert the doc covering-index probe by shape rather than by autoindex name. Co-Authored-By: Claude Fable 5.1 --------- Co-authored-by: Claude Fable 5.1 --- modules/docs/lib.py | 31 +++++++++++++++++++++++++++- modules/docs/test/test_lib.py | 38 +++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+), 1 deletion(-) diff --git a/modules/docs/lib.py b/modules/docs/lib.py index 3c2fd967..42ed3fd5 100644 --- a/modules/docs/lib.py +++ b/modules/docs/lib.py @@ -348,6 +348,26 @@ def _get_doc(session, file_group_id: int): return doc +# ORDER BY clauses for the unfiltered docs browse. Only orderings that walk a `file_group` index +# qualify; the rest keep the ORM query. +DOC_BROWSE_ORDERS = { + 'published_datetime': 'fg.published_datetime DESC NULLS LAST, fg.id DESC', + 'id': 'fg.id DESC', +} + + +def _doc_browse_sql(order_by: str) -> Optional[str]: + """SQL for a page of FileGroup ids when browsing docs with no filters, or None when `order_by` has no fast path. + + CROSS JOIN pins file_group as the outer table: SQLite walks its index in sort order and probes doc's covering + (file_group_id) index, so no file_group row is read for the sort key.""" + order = DOC_BROWSE_ORDERS.get(order_by) + if not order: + return None + return f'''SELECT fg.id AS id FROM file_group fg CROSS JOIN doc d ON d.file_group_id = fg.id + ORDER BY {order} LIMIT :limit OFFSET :offset''' + + def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None, limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False): from .models import Doc @@ -418,7 +438,16 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime else: query = query.order_by(desc(FileGroup.id)) - file_groups = query.offset(offset).limit(limit).all() + # `size` sorts on a doc column and `title` has no file_group index, so neither may fall back to `id`. + browse_sql = None + if unfiltered and order_by not in ('size', 'title'): + browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id') + if browse_sql: + fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()] + by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {} + file_groups = [by_id[i] for i in fg_ids] + else: + file_groups = query.offset(offset).limit(limit).all() fg_ids = [fg.id for fg in file_groups] results = [fg.__json__() for fg in file_groups] diff --git a/modules/docs/test/test_lib.py b/modules/docs/test/test_lib.py index 2db26844..56df294b 100644 --- a/modules/docs/test/test_lib.py +++ b/modules/docs/test/test_lib.py @@ -1,4 +1,7 @@ +from datetime import datetime, timezone + import pytest +from sqlalchemy import text from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \ get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \ @@ -334,3 +337,38 @@ async def test_search_docs_no_hint_when_no_query(test_session, test_directory): results, _ = _search_docs(mimetype='application/pdf') assert results and results[0]['id'] == fg.id assert 'section_hint' not in results[0] + + +@pytest.mark.asyncio +async def test_search_docs_browse_walks_file_group_date_index(test_session, test_directory, doc_factory): + """The unfiltered docs browse walks file_group's date index and probes doc's covering index: no file_group + primary-key probe and no temp sort tree. Ordering (NULLS LAST) and paging match the ORM path.""" + from modules.docs.lib import _doc_browse_sql + + docs = [doc_factory() for _ in range(3)] + docs[0].file_group.published_datetime = None + docs[1].file_group.published_datetime = datetime(2020, 1, 1, tzinfo=timezone.utc) + docs[2].file_group.published_datetime = datetime(2021, 1, 1, tzinfo=timezone.utc) + test_session.commit() + + sql = _doc_browse_sql('published_datetime') + plan = ' '.join(row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {sql}'), + dict(limit=20, offset=0)).fetchall()) + assert 'file_group_published_datetime_idx' in plan, plan + assert 'SEARCH d USING COVERING INDEX' in plan and '(file_group_id=?)' in plan, plan + assert 'INTEGER PRIMARY KEY' not in plan, plan + assert 'TEMP B-TREE' not in plan, plan + + # Newest first, NULL published dates last, same as the filtered ORM path. + results, total = _search_docs() + assert total == 3 + assert [i['id'] for i in results] == [docs[2].file_group_id, docs[1].file_group_id, docs[0].file_group_id] + + # Paging still works on the fast path. + results, total = _search_docs(limit=1, offset=1) + assert total == 3 + assert [i['id'] for i in results] == [docs[1].file_group_id] + + # Filtered searches are unaffected. + results, total = _search_docs(mimetype='application/pdf') + assert total == 3