The modelers and indexers find unindexed FileGroups through an index (#905)

apply_modelers and apply_indexers count `file_group WHERE indexed != 1` for progress
and apply_indexers selects its batches with it.  SQLite cannot use an index for `!=`,
so each of these read every file_group row; on a large library with a cold disk cache
the first count took close to a minute.  A partial index holds only the unindexed rows.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-30 19:40:46 -06:00 • committed by GitHub
parent 32405a9132
commit 4c8e9fb68c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
3 changed files with 62 additions and 0 deletions

View file

@ -0,0 +1,24 @@
"""A partial index of the unindexed FileGroups.
The modelers and indexers look up `file_group WHERE indexed != 1` (and count it for progress) on every
refresh. SQLite cannot use an index on `indexed` for `!=`, so each lookup read every file_group row; on a
large library from a cold disk cache that was close to a minute. This index holds only the unindexed rows.
Revision ID: 2026_10_30_1200
Revises: 2026_10_25_1200
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = '2026_10_30_1200'
down_revision = '2026_10_25_1200'
branch_labels = None
depends_on = None
def upgrade():
op.execute('CREATE INDEX IF NOT EXISTS file_group_unindexed_idx ON file_group (id) WHERE "indexed" != 1')
def downgrade():
op.execute('DROP INDEX IF EXISTS file_group_unindexed_idx')

View file

@ -89,6 +89,9 @@ class FileGroup(ModelHelper, Base):
# Identity of a FileGroup. Duplicates are collapsed by
# dedupe_file_groups_by_directory_stem before this unique index is created.
Index('file_group_directory_stem_idx', 'directory', 'stem', unique=True),
# Holds only the unindexed rows, so the modelers and indexers find them without reading every row.
# SQLite uses a partial index only when the query has this exact term: filter on `indexed != True`.
Index('file_group_unindexed_idx', 'id', sqlite_where=text('"indexed" != 1')),
)
# SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid).
id: Mapped[int] = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True)

View file

@ -3,6 +3,7 @@ import itertools
import json
import os
import pathlib
import re
import shutil
import zipfile
from http import HTTPStatus
@ -1992,6 +1993,40 @@ def test_directory_or_descendant(test_session):
assert 'file_group_directory_idx' in plan, plan
@pytest.mark.asyncio
async def test_unindexed_lookups_use_index(async_client, test_session, make_files_structure, refresh_files):
"""The modelers' and indexers' lookups of unindexed FileGroups read an index, not every file_group row.
A large library is almost entirely indexed, so a full scan to find the few unindexed rows read the whole
table on every refresh."""
make_files_structure(['a.txt', 'b.pdf', 'c.html'])
await refresh_files()
test_session.execute(sqlalchemy.text('UPDATE file_group SET "indexed" = 0'))
test_session.commit()
statements = []
def capture(conn, cursor, statement, parameters, context, executemany):
if re.search(r'FROM file_group WHERE file_group\."indexed" != 1( LIMIT|\)|$)', ' '.join(statement.split())):
statements.append((statement, parameters))
engine = test_session.get_bind()
sqlalchemy.event.listen(engine, 'before_cursor_execute', capture)
try:
await wrolpi.common.apply_modelers(progress_callback=lambda *a: None)
await lib.apply_indexers(progress_callback=lambda *a: None)
finally:
sqlalchemy.event.remove(engine, 'before_cursor_execute', capture)
# The progress counts and the indexer's batch select.
assert len(statements) >= 3
connection = test_session.connection().connection.dbapi_connection
for statement, parameters in statements:
plan = [i[3] for i in connection.execute(f'EXPLAIN QUERY PLAN {statement}', parameters)]
assert 'SCAN file_group' not in plan, (statement, plan)
assert any('file_group_unindexed_idx' in i for i in plan), (statement, plan)
def test_sha256_files(test_directory):
"""Each file is hashed; the metadata JSON, which holds the sums, and missing files are skipped."""
video, caption = test_directory / 'foo.mp4', test_directory / 'foo.en.vtt'