mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 07:56:18 -05:00
A directory lookup uses the directory index and matches only that directory (#900)
Uploads, deletes, and file refreshes looked up a missing path's descendants with `directory LIKE 'path/%'`. SQLite cannot use an index for that LIKE, so each lookup scanned the whole file_group table; with a cold page cache on a large library this held the first upload chunk or a delete for tens of seconds. LIKE also treated `_` and `%` in a path as wildcards, so refreshing a missing `a_b` removed the FileGroups of an existing `aXb/sub`. `directory_or_descendant` matches with a range the directory index can serve. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
49eeef68e8
commit
56fee562b7
4 changed files with 57 additions and 12 deletions
|
|
@ -21,7 +21,7 @@ from pathlib import Path
|
|||
from typing import Callable, ContextManager, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set
|
||||
|
||||
import cachetools.func
|
||||
from sqlalchemy import asc, or_, text as sa_text
|
||||
from sqlalchemy import and_, asc, or_, text as sa_text
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
from sqlalchemy.orm import Session, undefer, joinedload
|
||||
|
||||
|
|
@ -308,6 +308,15 @@ def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) -
|
|||
return [fg.__json__() for fg in rows]
|
||||
|
||||
|
||||
def directory_or_descendant(column, directory: Union[str, pathlib.Path]):
|
||||
"""A filter matching `directory` and every directory beneath it.
|
||||
|
||||
A range is used instead of `LIKE 'directory/%'` so SQLite can use the column's index (it cannot for a
|
||||
case-insensitive LIKE), and so `_` and `%` in a path are not wildcards. '0' is the character after '/'."""
|
||||
directory = str(directory)
|
||||
return or_(column == directory, and_(column >= f'{directory}/', column < f'{directory}0'))
|
||||
|
||||
|
||||
def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]:
|
||||
"""Return FileGroup objects under any of the given paths that have at least one TagFile.
|
||||
|
||||
|
|
@ -331,11 +340,7 @@ def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Pa
|
|||
else:
|
||||
# Search for any FileGroups that have been tagged under this directory.
|
||||
# Use indexed directory column for efficient lookup.
|
||||
path_str = str(path)
|
||||
query = query.filter(or_(
|
||||
FileGroup.directory == path_str,
|
||||
FileGroup.directory.like(f'{path_str}/%'),
|
||||
))
|
||||
query = query.filter(directory_or_descendant(FileGroup.directory, path))
|
||||
for file_group in query:
|
||||
if path.is_dir() or any(p for p in paths if p in file_group.my_paths()):
|
||||
if file_group.id not in seen_ids:
|
||||
|
|
|
|||
|
|
@ -1945,3 +1945,30 @@ async def test_remove_files_in_ignored_directories(async_client, test_directory,
|
|||
assert lib.remove_files_in_ignored_directories([private, nested, sibling, other]) == [sibling, other]
|
||||
# The ignored directory itself is ignored.
|
||||
assert lib.remove_files_in_ignored_directories([test_directory / 'private']) == []
|
||||
|
||||
|
||||
def test_directory_or_descendant(test_session):
|
||||
"""Matches a directory and everything beneath it, using the directory index."""
|
||||
directories = ['/media/a', '/media/a/b', '/media/a/b/c', '/media/ab', '/media/a0', '/media/A/b', '/media/aXb/c']
|
||||
for i, directory in enumerate(directories):
|
||||
test_session.add(FileGroup(primary_path=f'{directory}/file{i}.txt', directory=directory))
|
||||
for i, directory in enumerate(['/media/a_b/c', '/media/a%']):
|
||||
test_session.add(FileGroup(primary_path=f'{directory}/wild{i}.txt', directory=directory))
|
||||
test_session.commit()
|
||||
|
||||
def matching(directory):
|
||||
query = test_session.query(FileGroup).filter(lib.directory_or_descendant(FileGroup.directory, directory))
|
||||
return sorted(str(i.directory) for i in query)
|
||||
|
||||
assert matching('/media/a') == ['/media/a', '/media/a/b', '/media/a/b/c']
|
||||
assert matching(pathlib.Path('/media/a/b')) == ['/media/a/b', '/media/a/b/c']
|
||||
# `_` and `%` are not wildcards.
|
||||
assert matching('/media/a_b') == ['/media/a_b/c']
|
||||
assert matching('/media/a%') == ['/media/a%']
|
||||
|
||||
statement = test_session.query(FileGroup.id) \
|
||||
.filter(lib.directory_or_descendant(FileGroup.directory, '/media/a')).statement \
|
||||
.compile(compile_kwargs={'literal_binds': True})
|
||||
plan = ' '.join(row[3] for row in test_session.execute(sqlalchemy.text(f'EXPLAIN QUERY PLAN {statement}')))
|
||||
assert 'SCAN file_group' not in plan, plan
|
||||
assert 'file_group_directory_idx' in plan, plan
|
||||
|
|
|
|||
|
|
@ -2017,3 +2017,19 @@ async def test_queue_first_refresh_skipped(async_client, test_session, test_dire
|
|||
flags.refresh_complete.clear()
|
||||
await wrol_mode_fixture(True)
|
||||
assert file_worker.queue_first_refresh() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_missing_path_keeps_lookalike_directories(
|
||||
async_client, test_session, test_directory, make_files_structure
|
||||
):
|
||||
"""A missing path is refreshed as a possibly-deleted directory; only that directory and its descendants are
|
||||
matched, never a directory whose name differs by a character `LIKE` would treat as a wildcard."""
|
||||
kept, = make_files_structure(['docs/aXb/sub/kept.txt'])
|
||||
await file_worker.refresh_sync([kept])
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
await file_worker.refresh_sync([test_directory / 'docs/a_b'])
|
||||
|
||||
test_session.expire_all()
|
||||
assert [i.primary_path for i in test_session.query(FileGroup)] == [kept]
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ from multiprocessing.queues import Queue
|
|||
from pathlib import Path
|
||||
from typing import Callable, Set, Dict, List, Tuple
|
||||
|
||||
from sqlalchemy import text, or_
|
||||
from sqlalchemy import text
|
||||
|
||||
from wrolpi import flags
|
||||
from wrolpi.common import apply_modelers, apply_refresh_cleanup
|
||||
|
|
@ -31,6 +31,7 @@ from wrolpi.files.lib import (
|
|||
split_path_stem_and_suffix, _upsert_files, glob_shared_stem,
|
||||
group_files_by_stem, choose_primary_file, apply_indexers,
|
||||
get_normalized_ignored_directories, remove_files_in_ignored_directories, merge_into_surviving_duplicates,
|
||||
directory_or_descendant,
|
||||
)
|
||||
from wrolpi.files.refresh import (
|
||||
FileGroupDiff, FileComparisonResult, classify_file_group_diffs, compare_file_groups,
|
||||
|
|
@ -1096,12 +1097,8 @@ class FileWorker(FileMoveMixin, FileReorganizeMixin):
|
|||
|
||||
# Also query for FileGroups contained in deleted directories
|
||||
for deleted_path in deleted_paths:
|
||||
deleted_path_str = str(deleted_path)
|
||||
dir_deleted_fgs = session.query(FileGroup).filter(
|
||||
or_(
|
||||
FileGroup.directory == deleted_path_str,
|
||||
FileGroup.directory.like(f'{deleted_path_str}/%')
|
||||
),
|
||||
directory_or_descendant(FileGroup.directory, deleted_path),
|
||||
~FileGroup.id.in_(seen_fg_ids) if seen_fg_ids else True,
|
||||
).all()
|
||||
for fg in dir_deleted_fgs:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue