diff --git a/modules/docs/lib.py b/modules/docs/lib.py index 3c2fd967..42ed3fd5 100644 --- a/modules/docs/lib.py +++ b/modules/docs/lib.py @@ -348,6 +348,26 @@ def _get_doc(session, file_group_id: int): return doc +# ORDER BY clauses for the unfiltered docs browse. Only orderings that walk a `file_group` index +# qualify; the rest keep the ORM query. +DOC_BROWSE_ORDERS = { + 'published_datetime': 'fg.published_datetime DESC NULLS LAST, fg.id DESC', + 'id': 'fg.id DESC', +} + + +def _doc_browse_sql(order_by: str) -> Optional[str]: + """SQL for a page of FileGroup ids when browsing docs with no filters, or None when `order_by` has no fast path. + + CROSS JOIN pins file_group as the outer table: SQLite walks its index in sort order and probes doc's covering + (file_group_id) index, so no file_group row is read for the sort key.""" + order = DOC_BROWSE_ORDERS.get(order_by) + if not order: + return None + return f'''SELECT fg.id AS id FROM file_group fg CROSS JOIN doc d ON d.file_group_id = fg.id + ORDER BY {order} LIMIT :limit OFFSET :offset''' + + def _search_docs(search_str=None, author=None, subject=None, language=None, mimetype=None, limit=20, offset=0, order_by='published_datetime', tag_names=None, deep=False): from .models import Doc @@ -418,7 +438,16 @@ def _search_docs(search_str=None, author=None, subject=None, language=None, mime else: query = query.order_by(desc(FileGroup.id)) - file_groups = query.offset(offset).limit(limit).all() + # `size` sorts on a doc column and `title` has no file_group index, so neither may fall back to `id`. + browse_sql = None + if unfiltered and order_by not in ('size', 'title'): + browse_sql = _doc_browse_sql(order_by if order_by in DOC_BROWSE_ORDERS else 'id') + if browse_sql: + fg_ids = [i for i, in session.execute(text(browse_sql), dict(limit=limit, offset=offset)).fetchall()] + by_id = {fg.id: fg for fg in session.query(FileGroup).filter(FileGroup.id.in_(fg_ids))} if fg_ids else {} + file_groups = [by_id[i] for i in fg_ids] + else: + file_groups = query.offset(offset).limit(limit).all() fg_ids = [fg.id for fg in file_groups] results = [fg.__json__() for fg in file_groups] diff --git a/modules/docs/test/test_lib.py b/modules/docs/test/test_lib.py index 2db26844..56df294b 100644 --- a/modules/docs/test/test_lib.py +++ b/modules/docs/test/test_lib.py @@ -1,4 +1,7 @@ +from datetime import datetime, timezone + import pytest +from sqlalchemy import text from modules.docs.lib import is_valid_author, split_authors, get_or_create_author_collection, \ get_or_create_subject_collection, normalize_author, normalize_subject, split_subjects, is_valid_subject, \ @@ -334,3 +337,38 @@ async def test_search_docs_no_hint_when_no_query(test_session, test_directory): results, _ = _search_docs(mimetype='application/pdf') assert results and results[0]['id'] == fg.id assert 'section_hint' not in results[0] + + +@pytest.mark.asyncio +async def test_search_docs_browse_walks_file_group_date_index(test_session, test_directory, doc_factory): + """The unfiltered docs browse walks file_group's date index and probes doc's covering index: no file_group + primary-key probe and no temp sort tree. Ordering (NULLS LAST) and paging match the ORM path.""" + from modules.docs.lib import _doc_browse_sql + + docs = [doc_factory() for _ in range(3)] + docs[0].file_group.published_datetime = None + docs[1].file_group.published_datetime = datetime(2020, 1, 1, tzinfo=timezone.utc) + docs[2].file_group.published_datetime = datetime(2021, 1, 1, tzinfo=timezone.utc) + test_session.commit() + + sql = _doc_browse_sql('published_datetime') + plan = ' '.join(row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {sql}'), + dict(limit=20, offset=0)).fetchall()) + assert 'file_group_published_datetime_idx' in plan, plan + assert 'SEARCH d USING COVERING INDEX' in plan and '(file_group_id=?)' in plan, plan + assert 'INTEGER PRIMARY KEY' not in plan, plan + assert 'TEMP B-TREE' not in plan, plan + + # Newest first, NULL published dates last, same as the filtered ORM path. + results, total = _search_docs() + assert total == 3 + assert [i['id'] for i in results] == [docs[2].file_group_id, docs[1].file_group_id, docs[0].file_group_id] + + # Paging still works on the fast path. + results, total = _search_docs(limit=1, offset=1) + assert total == 3 + assert [i['id'] for i in results] == [docs[1].file_group_id] + + # Filtered searches are unaffected. + results, total = _search_docs(mimetype='application/pdf') + assert total == 3