diff --git a/alembic/versions/2026_10_30_1200_file_group_unindexed_index.py b/alembic/versions/2026_10_30_1200_file_group_unindexed_index.py new file mode 100644 index 00000000..d2e84754 --- /dev/null +++ b/alembic/versions/2026_10_30_1200_file_group_unindexed_index.py @@ -0,0 +1,24 @@ +"""A partial index of the unindexed FileGroups. + +The modelers and indexers look up `file_group WHERE indexed != 1` (and count it for progress) on every +refresh. SQLite cannot use an index on `indexed` for `!=`, so each lookup read every file_group row; on a +large library from a cold disk cache that was close to a minute. This index holds only the unindexed rows. + +Revision ID: 2026_10_30_1200 +Revises: 2026_10_25_1200 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_10_30_1200' +down_revision = '2026_10_25_1200' +branch_labels = None +depends_on = None + + +def upgrade(): + op.execute('CREATE INDEX IF NOT EXISTS file_group_unindexed_idx ON file_group (id) WHERE "indexed" != 1') + + +def downgrade(): + op.execute('DROP INDEX IF EXISTS file_group_unindexed_idx') diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index fb2cd94f..adc88099 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -89,6 +89,9 @@ class FileGroup(ModelHelper, Base): # Identity of a FileGroup. Duplicates are collapsed by # dedupe_file_groups_by_directory_stem before this unique index is created. Index('file_group_directory_stem_idx', 'directory', 'stem', unique=True), + # Holds only the unindexed rows, so the modelers and indexers find them without reading every row. + # SQLite uses a partial index only when the query has this exact term: filter on `indexed != True`. + Index('file_group_unindexed_idx', 'id', sqlite_where=text('"indexed" != 1')), ) # SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid). id: Mapped[int] = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 2093aad7..ea8e48df 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -3,6 +3,7 @@ import itertools import json import os import pathlib +import re import shutil import zipfile from http import HTTPStatus @@ -1992,6 +1993,40 @@ def test_directory_or_descendant(test_session): assert 'file_group_directory_idx' in plan, plan +@pytest.mark.asyncio +async def test_unindexed_lookups_use_index(async_client, test_session, make_files_structure, refresh_files): + """The modelers' and indexers' lookups of unindexed FileGroups read an index, not every file_group row. + + A large library is almost entirely indexed, so a full scan to find the few unindexed rows read the whole + table on every refresh.""" + make_files_structure(['a.txt', 'b.pdf', 'c.html']) + await refresh_files() + test_session.execute(sqlalchemy.text('UPDATE file_group SET "indexed" = 0')) + test_session.commit() + + statements = [] + + def capture(conn, cursor, statement, parameters, context, executemany): + if re.search(r'FROM file_group WHERE file_group\."indexed" != 1( LIMIT|\)|$)', ' '.join(statement.split())): + statements.append((statement, parameters)) + + engine = test_session.get_bind() + sqlalchemy.event.listen(engine, 'before_cursor_execute', capture) + try: + await wrolpi.common.apply_modelers(progress_callback=lambda *a: None) + await lib.apply_indexers(progress_callback=lambda *a: None) + finally: + sqlalchemy.event.remove(engine, 'before_cursor_execute', capture) + + # The progress counts and the indexer's batch select. + assert len(statements) >= 3 + connection = test_session.connection().connection.dbapi_connection + for statement, parameters in statements: + plan = [i[3] for i in connection.execute(f'EXPLAIN QUERY PLAN {statement}', parameters)] + assert 'SCAN file_group' not in plan, (statement, plan) + assert any('file_group_unindexed_idx' in i for i in plan), (statement, plan) + + def test_sha256_files(test_directory): """Each file is hashed; the metadata JSON, which holds the sums, and missing files are skipped.""" video, caption = test_directory / 'foo.mp4', test_directory / 'foo.en.vtt'