mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
docs: browse walks the file_group date index instead of probing every doc (#707)
* docs: browse walks the file_group date index instead of probing every doc The unfiltered /api/docs/search drove its join from `doc`, so SQLite probed every doc's file_group row for the sort key before returning the first page. With 20k docs scattered through a 10 GB database on a spinning disk, the first request took about a minute cold (0.07s warm). Use the same CROSS JOIN shape as the archive browse: file_group is the outer table, SQLite walks file_group_published_datetime_idx and probes doc's covering unique index on file_group_id. Both sides stay index-only and the scan stops at the page limit. Measured cold: ~60s -> 0.6s, identical rows. Filtered searches and the size/title orderings keep the ORM query. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * docs: trim browse comments, stabler plan assertion Review feedback: drop the benchmark narrative from docstrings, state only the non-obvious constraint at the call site, and assert the doc covering-index probe by shape rather than by autoindex name. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
e0fdb762fc
commit
7471b41ae1
2 changed files with 68 additions and 1 deletions
|
|
@ -348,6 +348,26 @@ def _get_doc(session, file_group_id: int):
|
|||
return doc
|
||||
|
||||
|
||||
# ORDER BY clauses for the unfiltered docs browse. Only orderings that walk a `file_group` index
|
||||
# qualify; the rest keep the ORM query.
|
||||
DOC_BROWSE_ORDERS = {
|
||||
'published_datetime': 'fg.published_datetime DESC NULLS LAST, fg.id DESC',
|
||||
'id': 'fg.id DESC',
|
||||
}
|
||||
|
||||
|
||||
def _doc_browse_sql(order_by: str) -> Optional[str]:
|
||||
"""SQL for a page of FileGroup ids when browsing docs with no filters, or None when `order_by` has no fast path.
|
||||
|
||||
CROSS JOIN pins file_group as the outer table: SQLite walks its index in sort order and probes doc's covering
|
||||
(file_group_id) index, so no file_group row is read for the sort key."""
|
||||
order = DOC_BROWSE_ORDERS.get(order_by)
|
||||
if not order:
|
||||
return None
|
||||
return f'''SELECT fg.id AS id FROM file_group fg CROSS JOIN doc d ON d.file_group_id = fg.id
|
||||
ORDER BY {order} LIMIT :limit OFFSET :offset'''
|
||||
|
||||
|
||||
def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None,
|
||||
limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False):
|
||||
from .models import Doc
|
||||
|
|
@ -418,7 +438,16 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime
|
|||
else:
|
||||
query = query.order_by(desc(FileGroup.id))
|
||||
|
||||
file_groups = query.offset(offset).limit(limit).all()
|
||||
# `size` sorts on a doc column and `title` has no file_group index, so neither may fall back to `id`.
|
||||
browse_sql = None
|
||||
if unfiltered and order_by not in ('size', 'title'):
|
||||
browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id')
|
||||
if browse_sql:
|
||||
fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()]
|
||||
by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {}
|
||||
file_groups = [by_id[i] for i in fg_ids]
|
||||
else:
|
||||
file_groups = query.offset(offset).limit(limit).all()
|
||||
fg_ids = [fg.id for fg in file_groups]
|
||||
results = [fg.__json__() for fg in file_groups]
|
||||
|
||||
|
|
|
|||
|
|
@ -1,4 +1,7 @@
|
|||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
from sqlalchemy import text
|
||||
|
||||
from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \
|
||||
get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \
|
||||
|
|
@ -334,3 +337,38 @@ async def test_search_docs_no_hint_when_no_query(test_session, test_directory):
|
|||
results, _ = _search_docs(mimetype='application/pdf')
|
||||
assert results and results[0]['id'] == fg.id
|
||||
assert 'section_hint' not in results[0]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_search_docs_browse_walks_file_group_date_index(test_session, test_directory, doc_factory):
|
||||
"""The unfiltered docs browse walks file_group's date index and probes doc's covering index: no file_group
|
||||
primary-key probe and no temp sort tree. Ordering (NULLS LAST) and paging match the ORM path."""
|
||||
from modules.docs.lib import _doc_browse_sql
|
||||
|
||||
docs = [doc_factory() for _ in range(3)]
|
||||
docs[0].file_group.published_datetime = None
|
||||
docs[1].file_group.published_datetime = datetime(2020, 1, 1, tzinfo=timezone.utc)
|
||||
docs[2].file_group.published_datetime = datetime(2021, 1, 1, tzinfo=timezone.utc)
|
||||
test_session.commit()
|
||||
|
||||
sql = _doc_browse_sql('published_datetime')
|
||||
plan = ' '.join(row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {sql}'),
|
||||
dict(limit=20, offset=0)).fetchall())
|
||||
assert 'file_group_published_datetime_idx' in plan, plan
|
||||
assert 'SEARCH d USING COVERING INDEX' in plan and '(file_group_id=?)' in plan, plan
|
||||
assert 'INTEGER PRIMARY KEY' not in plan, plan
|
||||
assert 'TEMP B-TREE' not in plan, plan
|
||||
|
||||
# Newest first, NULL published dates last, same as the filtered ORM path.
|
||||
results, total = _search_docs()
|
||||
assert total == 3
|
||||
assert [i['id'] for i in results] == [docs[2].file_group_id, docs[1].file_group_id, docs[0].file_group_id]
|
||||
|
||||
# Paging still works on the fast path.
|
||||
results, total = _search_docs(limit=1, offset=1)
|
||||
assert total == 3
|
||||
assert [i['id'] for i in results] == [docs[1].file_group_id]
|
||||
|
||||
# Filtered searches are unaffected.
|
||||
results, total = _search_docs(mimetype='application/pdf')
|
||||
assert total == 3
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue