A directory lookup uses the directory index and matches only that directory (#900)

Uploads, deletes, and file refreshes looked up a missing path's descendants with
`directory LIKE 'path/%'`.  SQLite cannot use an index for that LIKE, so each lookup
scanned the whole file_group table; with a cold page cache on a large library this
held the first upload chunk or a delete for tens of seconds.  LIKE also treated `_`
and `%` in a path as wildcards, so refreshing a missing `a_b` removed the FileGroups of
an existing `aXb/sub`.

`directory_or_descendant` matches with a range the directory index can serve.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-29 19:56:48 -06:00 • committed by Roland
parent 49eeef68e8
commit 56fee562b7
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 57 additions and 12 deletions

View file

@ -21,7 +21,7 @@ from pathlib import Path
from typing import Callable, ContextManager, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set
import cachetools.func
from sqlalchemy import asc, or_, text as sa_text
from sqlalchemy import and_, asc, or_, text as sa_text
from sqlalchemy.exc import IntegrityError
from sqlalchemy.orm import Session, undefer, joinedload
@ -308,6 +308,15 @@ def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) -
return [fg.__json__() for fg in rows]
def directory_or_descendant(column, directory: Union[str, pathlib.Path]):
"""A filter matching `directory` and every directory beneath it.
A range is used instead of `LIKE 'directory/%'` so SQLite can use the column's index (it cannot for a
case-insensitive LIKE), and so `_` and `%` in a path are not wildcards. '0' is the character after '/'."""
directory = str(directory)
return or_(column == directory, and_(column >= f'{directory}/', column < f'{directory}0'))
def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]:
"""Return FileGroup objects under any of the given paths that have at least one TagFile.
@ -331,11 +340,7 @@ def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Pa
else:
# Search for any FileGroups that have been tagged under this directory.
# Use indexed directory column for efficient lookup.
path_str = str(path)
query = query.filter(or_(
FileGroup.directory == path_str,
FileGroup.directory.like(f'{path_str}/%'),
))
query = query.filter(directory_or_descendant(FileGroup.directory, path))
for file_group in query:
if path.is_dir() or any(p for p in paths if p in file_group.my_paths()):
if file_group.id not in seen_ids:

View file

@ -1945,3 +1945,30 @@ async def test_remove_files_in_ignored_directories(async_client, test_directory,
assert lib.remove_files_in_ignored_directories([private, nested, sibling, other]) == [sibling, other]
# The ignored directory itself is ignored.
assert lib.remove_files_in_ignored_directories([test_directory / 'private']) == []
def test_directory_or_descendant(test_session):
"""Matches a directory and everything beneath it, using the directory index."""
directories = ['/media/a', '/media/a/b', '/media/a/b/c', '/media/ab', '/media/a0', '/media/A/b', '/media/aXb/c']
for i, directory in enumerate(directories):
test_session.add(FileGroup(primary_path=f'{directory}/file{i}.txt', directory=directory))
for i, directory in enumerate(['/media/a_b/c', '/media/a%']):
test_session.add(FileGroup(primary_path=f'{directory}/wild{i}.txt', directory=directory))
test_session.commit()
def matching(directory):
query = test_session.query(FileGroup).filter(lib.directory_or_descendant(FileGroup.directory, directory))
return sorted(str(i.directory) for i in query)
assert matching('/media/a') == ['/media/a', '/media/a/b', '/media/a/b/c']
assert matching(pathlib.Path('/media/a/b')) == ['/media/a/b', '/media/a/b/c']
# `_` and `%` are not wildcards.
assert matching('/media/a_b') == ['/media/a_b/c']
assert matching('/media/a%') == ['/media/a%']
statement = test_session.query(FileGroup.id) \
.filter(lib.directory_or_descendant(FileGroup.directory, '/media/a')).statement \
.compile(compile_kwargs={'literal_binds': True})
plan = ' '.join(row[3] for row in test_session.execute(sqlalchemy.text(f'EXPLAIN QUERY PLAN {statement}')))
assert 'SCAN file_group' not in plan, plan
assert 'file_group_directory_idx' in plan, plan

View file

@ -2017,3 +2017,19 @@ async def test_queue_first_refresh_skipped(async_client, test_session, test_dire
flags.refresh_complete.clear()
await wrol_mode_fixture(True)
assert file_worker.queue_first_refresh() is None
@pytest.mark.asyncio
async def test_refresh_missing_path_keeps_lookalike_directories(
async_client, test_session, test_directory, make_files_structure
):
"""A missing path is refreshed as a possibly-deleted directory; only that directory and its descendants are
matched, never a directory whose name differs by a character `LIKE` would treat as a wildcard."""
kept, = make_files_structure(['docs/aXb/sub/kept.txt'])
await file_worker.refresh_sync([kept])
assert test_session.query(FileGroup).count() == 1
await file_worker.refresh_sync([test_directory / 'docs/a_b'])
test_session.expire_all()
assert [i.primary_path for i in test_session.query(FileGroup)] == [kept]

View file

@ -17,7 +17,7 @@ from multiprocessing.queues import Queue
from pathlib import Path
from typing import Callable, Set, Dict, List, Tuple
from sqlalchemy import text, or_
from sqlalchemy import text
from wrolpi import flags
from wrolpi.common import apply_modelers, apply_refresh_cleanup
@ -31,6 +31,7 @@ from wrolpi.files.lib import (
split_path_stem_and_suffix, _upsert_files, glob_shared_stem,
group_files_by_stem, choose_primary_file, apply_indexers,
get_normalized_ignored_directories, remove_files_in_ignored_directories, merge_into_surviving_duplicates,
directory_or_descendant,
)
from wrolpi.files.refresh import (
FileGroupDiff, FileComparisonResult, classify_file_group_diffs, compare_file_groups,
@ -1096,12 +1097,8 @@ class FileWorker(FileMoveMixin, FileReorganizeMixin):
# Also query for FileGroups contained in deleted directories
for deleted_path in deleted_paths:
deleted_path_str = str(deleted_path)
dir_deleted_fgs = session.query(FileGroup).filter(
or_(
FileGroup.directory == deleted_path_str,
FileGroup.directory.like(f'{deleted_path_str}/%')
),
directory_or_descendant(FileGroup.directory, deleted_path),
~FileGroup.id.in_(seen_fg_ids) if seen_fg_ids else True,
).all()
for fg in dir_deleted_fgs: