diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 9240a38d..b9d9b2c1 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -21,7 +21,7 @@ from pathlib import Path from typing import Callable, ContextManager, List, Optional, Tuple, Union, Dict, Generator, Iterable, Set import cachetools.func -from sqlalchemy import asc, or_, text as sa_text +from sqlalchemy import and_, asc, or_, text as sa_text from sqlalchemy.exc import IntegrityError from sqlalchemy.orm import Session, undefer, joinedload @@ -308,6 +308,15 @@ def get_tagged_file_groups_by_ids(session: Session, file_group_ids: List[int]) - return [fg.__json__() for fg in rows] +def directory_or_descendant(column, directory: Union[str, pathlib.Path]): + """A filter matching `directory` and every directory beneath it. + + A range is used instead of `LIKE 'directory/%'` so SQLite can use the column's index (it cannot for a + case-insensitive LIKE), and so `_` and `%` in a path are not wildcards. '0' is the character after '/'.""" + directory = str(directory) + return or_(column == directory, and_(column >= f'{directory}/', column < f'{directory}0')) + + def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Path]) -> List[FileGroup]: """Return FileGroup objects under any of the given paths that have at least one TagFile. @@ -331,11 +340,7 @@ def get_tagged_file_groups_by_paths(session: Session, paths: Iterable[pathlib.Pa else: # Search for any FileGroups that have been tagged under this directory. # Use indexed directory column for efficient lookup. - path_str = str(path) - query = query.filter(or_( - FileGroup.directory == path_str, - FileGroup.directory.like(f'{path_str}/%'), - )) + query = query.filter(directory_or_descendant(FileGroup.directory, path)) for file_group in query: if path.is_dir() or any(p for p in paths if p in file_group.my_paths()): if file_group.id not in seen_ids: diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index b8028fc0..685abe33 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1945,3 +1945,30 @@ async def test_remove_files_in_ignored_directories(async_client, test_directory, assert lib.remove_files_in_ignored_directories([private, nested, sibling, other]) == [sibling, other] # The ignored directory itself is ignored. assert lib.remove_files_in_ignored_directories([test_directory / 'private']) == [] + + +def test_directory_or_descendant(test_session): + """Matches a directory and everything beneath it, using the directory index.""" + directories = ['/media/a', '/media/a/b', '/media/a/b/c', '/media/ab', '/media/a0', '/media/A/b', '/media/aXb/c'] + for i, directory in enumerate(directories): + test_session.add(FileGroup(primary_path=f'{directory}/file{i}.txt', directory=directory)) + for i, directory in enumerate(['/media/a_b/c', '/media/a%']): + test_session.add(FileGroup(primary_path=f'{directory}/wild{i}.txt', directory=directory)) + test_session.commit() + + def matching(directory): + query = test_session.query(FileGroup).filter(lib.directory_or_descendant(FileGroup.directory, directory)) + return sorted(str(i.directory) for i in query) + + assert matching('/media/a') == ['/media/a', '/media/a/b', '/media/a/b/c'] + assert matching(pathlib.Path('/media/a/b')) == ['/media/a/b', '/media/a/b/c'] + # `_` and `%` are not wildcards. + assert matching('/media/a_b') == ['/media/a_b/c'] + assert matching('/media/a%') == ['/media/a%'] + + statement = test_session.query(FileGroup.id) \ + .filter(lib.directory_or_descendant(FileGroup.directory, '/media/a')).statement \ + .compile(compile_kwargs={'literal_binds': True}) + plan = ' '.join(row[3] for row in test_session.execute(sqlalchemy.text(f'EXPLAIN QUERY PLAN {statement}'))) + assert 'SCAN file_group' not in plan, plan + assert 'file_group_directory_idx' in plan, plan diff --git a/wrolpi/files/test/test_worker.py b/wrolpi/files/test/test_worker.py index b3345161..30265dfa 100644 --- a/wrolpi/files/test/test_worker.py +++ b/wrolpi/files/test/test_worker.py @@ -2017,3 +2017,19 @@ async def test_queue_first_refresh_skipped(async_client, test_session, test_dire flags.refresh_complete.clear() await wrol_mode_fixture(True) assert file_worker.queue_first_refresh() is None + + +@pytest.mark.asyncio +async def test_refresh_missing_path_keeps_lookalike_directories( + async_client, test_session, test_directory, make_files_structure +): + """A missing path is refreshed as a possibly-deleted directory; only that directory and its descendants are + matched, never a directory whose name differs by a character `LIKE` would treat as a wildcard.""" + kept, = make_files_structure(['docs/aXb/sub/kept.txt']) + await file_worker.refresh_sync([kept]) + assert test_session.query(FileGroup).count() == 1 + + await file_worker.refresh_sync([test_directory / 'docs/a_b']) + + test_session.expire_all() + assert [i.primary_path for i in test_session.query(FileGroup)] == [kept] diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index cffdf76c..d1895cbb 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -17,7 +17,7 @@ from multiprocessing.queues import Queue from pathlib import Path from typing import Callable, Set, Dict, List, Tuple -from sqlalchemy import text, or_ +from sqlalchemy import text from wrolpi import flags from wrolpi.common import apply_modelers, apply_refresh_cleanup @@ -31,6 +31,7 @@ from wrolpi.files.lib import ( split_path_stem_and_suffix, _upsert_files, glob_shared_stem, group_files_by_stem, choose_primary_file, apply_indexers, get_normalized_ignored_directories, remove_files_in_ignored_directories, merge_into_surviving_duplicates, + directory_or_descendant, ) from wrolpi.files.refresh import ( FileGroupDiff, FileComparisonResult, classify_file_group_diffs, compare_file_groups, @@ -1096,12 +1097,8 @@ class FileWorker(FileMoveMixin, FileReorganizeMixin): # Also query for FileGroups contained in deleted directories for deleted_path in deleted_paths: - deleted_path_str = str(deleted_path) dir_deleted_fgs = session.query(FileGroup).filter( - or_( - FileGroup.directory == deleted_path_str, - FileGroup.directory.like(f'{deleted_path_str}/%') - ), + directory_or_descendant(FileGroup.directory, deleted_path), ~FileGroup.id.in_(seen_fg_ids) if seen_fg_ids else True, ).all() for fg in dir_deleted_fgs: