mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
The modelers and indexers find unindexed FileGroups through an index (#905)
apply_modelers and apply_indexers count `file_group WHERE indexed != 1` for progress and apply_indexers selects its batches with it. SQLite cannot use an index for `!=`, so each of these read every file_group row; on a large library with a cold disk cache the first count took close to a minute. A partial index holds only the unindexed rows. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
32405a9132
commit
4c8e9fb68c
3 changed files with 62 additions and 0 deletions
|
|
@ -0,0 +1,24 @@
|
|||
"""A partial index of the unindexed FileGroups.
|
||||
|
||||
The modelers and indexers look up `file_group WHERE indexed != 1` (and count it for progress) on every
|
||||
refresh. SQLite cannot use an index on `indexed` for `!=`, so each lookup read every file_group row; on a
|
||||
large library from a cold disk cache that was close to a minute. This index holds only the unindexed rows.
|
||||
|
||||
Revision ID: 2026_10_30_1200
|
||||
Revises: 2026_10_25_1200
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '2026_10_30_1200'
|
||||
down_revision = '2026_10_25_1200'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
op.execute('CREATE INDEX IF NOT EXISTS file_group_unindexed_idx ON file_group (id) WHERE "indexed" != 1')
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute('DROP INDEX IF EXISTS file_group_unindexed_idx')
|
||||
|
|
@ -89,6 +89,9 @@ class FileGroup(ModelHelper, Base):
|
|||
# Identity of a FileGroup. Duplicates are collapsed by
|
||||
# dedupe_file_groups_by_directory_stem before this unique index is created.
|
||||
Index('file_group_directory_stem_idx', 'directory', 'stem', unique=True),
|
||||
# Holds only the unindexed rows, so the modelers and indexers find them without reading every row.
|
||||
# SQLite uses a partial index only when the query has this exact term: filter on `indexed != True`.
|
||||
Index('file_group_unindexed_idx', 'id', sqlite_where=text('"indexed" != 1')),
|
||||
)
|
||||
# SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid).
|
||||
id: Mapped[int] = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True)
|
||||
|
|
|
|||
|
|
@ -3,6 +3,7 @@ import itertools
|
|||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shutil
|
||||
import zipfile
|
||||
from http import HTTPStatus
|
||||
|
|
@ -1992,6 +1993,40 @@ def test_directory_or_descendant(test_session):
|
|||
assert 'file_group_directory_idx' in plan, plan
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unindexed_lookups_use_index(async_client, test_session, make_files_structure, refresh_files):
|
||||
"""The modelers' and indexers' lookups of unindexed FileGroups read an index, not every file_group row.
|
||||
|
||||
A large library is almost entirely indexed, so a full scan to find the few unindexed rows read the whole
|
||||
table on every refresh."""
|
||||
make_files_structure(['a.txt', 'b.pdf', 'c.html'])
|
||||
await refresh_files()
|
||||
test_session.execute(sqlalchemy.text('UPDATE file_group SET "indexed" = 0'))
|
||||
test_session.commit()
|
||||
|
||||
statements = []
|
||||
|
||||
def capture(conn, cursor, statement, parameters, context, executemany):
|
||||
if re.search(r'FROM file_group WHERE file_group\."indexed" != 1( LIMIT|\)|$)', ' '.join(statement.split())):
|
||||
statements.append((statement, parameters))
|
||||
|
||||
engine = test_session.get_bind()
|
||||
sqlalchemy.event.listen(engine, 'before_cursor_execute', capture)
|
||||
try:
|
||||
await wrolpi.common.apply_modelers(progress_callback=lambda *a: None)
|
||||
await lib.apply_indexers(progress_callback=lambda *a: None)
|
||||
finally:
|
||||
sqlalchemy.event.remove(engine, 'before_cursor_execute', capture)
|
||||
|
||||
# The progress counts and the indexer's batch select.
|
||||
assert len(statements) >= 3
|
||||
connection = test_session.connection().connection.dbapi_connection
|
||||
for statement, parameters in statements:
|
||||
plan = [i[3] for i in connection.execute(f'EXPLAIN QUERY PLAN {statement}', parameters)]
|
||||
assert 'SCAN file_group' not in plan, (statement, plan)
|
||||
assert any('file_group_unindexed_idx' in i for i in plan), (statement, plan)
|
||||
|
||||
|
||||
def test_sha256_files(test_directory):
|
||||
"""Each file is hashed; the metadata JSON, which holds the sums, and missing files are skipped."""
|
||||
video, caption = test_directory / 'foo.mp4', test_directory / 'foo.en.vtt'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue