docs: browse walks the file_group date index instead of probing every doc (#707)
Some checks are pending
CodeQL / Analyze (push) Waiting to run
Controller Tests / controller-tests (push) Waiting to run
Jest Tests / jest-tests (push) Waiting to run

* docs: browse walks the file_group date index instead of probing every doc

The unfiltered /api/docs/search drove its join from `doc`, so SQLite probed
every doc's file_group row for the sort key before returning the first page.
With 20k docs scattered through a 10 GB database on a spinning disk, the first
request took about a minute cold (0.07s warm).

Use the same CROSS JOIN shape as the archive browse: file_group is the outer
table, SQLite walks file_group_published_datetime_idx and probes doc's covering
unique index on file_group_id. Both sides stay index-only and the scan stops at
the page limit. Measured cold: ~60s -> 0.6s, identical rows.

Filtered searches and the size/title orderings keep the ORM query.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* docs: trim browse comments, stabler plan assertion

Review feedback: drop the benchmark narrative from docstrings, state only the
non-obvious constraint at the call site, and assert the doc covering-index probe
by shape rather than by autoindex name.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-09-13 12:38:48 -06:00 • committed by Roland
parent e0fdb762fc
commit 7471b41ae1
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 68 additions and 1 deletions

View file

@ -348,6 +348,26 @@ def _get_doc(session, file_group_id: int):
return doc
# ORDER BY clauses for the unfiltered docs browse. Only orderings that walk a `file_group` index
# qualify; the rest keep the ORM query.
DOC_BROWSE_ORDERS = {
'published_datetime': 'fg.published_datetime DESC NULLS LAST, fg.id DESC',
'id': 'fg.id DESC',
}
def _doc_browse_sql(order_by: str) -> Optional[str]:
"""SQL for a page of FileGroup ids when browsing docs with no filters, or None when `order_by` has no fast path.
CROSS JOIN pins file_group as the outer table: SQLite walks its index in sort order and probes doc's covering
(file_group_id) index, so no file_group row is read for the sort key."""
order = DOC_BROWSE_ORDERS.get(order_by)
if not order:
return None
return f'''SELECT fg.id AS id FROM file_group fg CROSS JOIN doc d ON d.file_group_id = fg.id
ORDER BY {order} LIMIT :limit OFFSET :offset'''
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
from .models import Doc
@ -418,7 +438,16 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
else:
query = query.order_by(desc(FileGroup.id))
file_groups = query.offset(offset).limit(limit).all()
# `size` sorts on a doc column and `title` has no file_group index, so neither may fall back to `id`.
browse_sql = None
if unfiltered and order_by not in ('size', 'title'):
browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id')
if browse_sql:
fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()]
by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
file_groups = [by_id[i] for i in fg_ids]
else:
file_groups = query.offset(offset).limit(limit).all()
fg_ids = [fg.id for fg in file_groups]
results = [fg.__json__() for fg in file_groups]

View file

@ -1,4 +1,7 @@
from datetime import datetime, timezone
import pytest
from sqlalchemy import text
from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \
get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \
@ -334,3 +337,38 @@ async def test_search_docs_no_hint_when_no_query(test_session, test_directory):
results, _ = _search_docs(mimetype='application/pdf')
assert results and results[0]['id'] == fg.id
assert 'section_hint' not in results[0]
@pytest.mark.asyncio
async def test_search_docs_browse_walks_file_group_date_index(test_session, test_directory, doc_factory):
"""The unfiltered docs browse walks file_group's date index and probes doc's covering index: no file_group
primary-key probe and no temp sort tree. Ordering (NULLS LAST) and paging match the ORM path."""
from modules.docs.lib import _doc_browse_sql
docs = [doc_factory() for _ in range(3)]
docs[0].file_group.published_datetime = None
docs[1].file_group.published_datetime = datetime(2020, 1, 1, tzinfo=timezone.utc)
docs[2].file_group.published_datetime = datetime(2021, 1, 1, tzinfo=timezone.utc)
test_session.commit()
sql = _doc_browse_sql('published_datetime')
plan = ' '.join(row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {sql}'),
dict(limit=20, offset=0)).fetchall())
assert 'file_group_published_datetime_idx' in plan, plan
assert 'SEARCH d USING COVERING INDEX' in plan and '(file_group_id=?)' in plan, plan
assert 'INTEGER PRIMARY KEY' not in plan, plan
assert 'TEMP B-TREE' not in plan, plan
# Newest first, NULL published dates last, same as the filtered ORM path.
results, total = _search_docs()
assert total == 3
assert [i['id'] for i in results] == [docs[2].file_group_id, docs[1].file_group_id, docs[0].file_group_id]
# Paging still works on the fast path.
results, total = _search_docs(limit=1, offset=1)
assert total == 3
assert [i['id'] for i in results] == [docs[1].file_group_id]
# Filtered searches are unaffected.
results, total = _search_docs(mimetype='application/pdf')
assert total == 3