diff --git a/alembic/versions/2026_09_18_1200_unique_directory_stem.py b/alembic/versions/2026_09_18_1200_unique_directory_stem.py new file mode 100644 index 00000000..ad4ee6c3 --- /dev/null +++ b/alembic/versions/2026_09_18_1200_unique_directory_stem.py @@ -0,0 +1,30 @@ +"""Unique (directory, stem) on file_group. + +713 added the stem column and a non-unique index. This release collapses any +remaining duplicate (directory, stem) rows (moving tags and collection +membership onto the winner) and makes the index UNIQUE. + +Revision ID: 2026_09_18_1200 +Revises: 2026_09_16_1200 +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_09_18_1200' +down_revision = '2026_09_16_1200' +branch_labels = None +depends_on = None + + +def upgrade(): + from wrolpi.files.lib import backfill_null_file_group_stems, dedupe_file_groups_by_directory_stem + + backfill_null_file_group_stems() + dedupe_file_groups_by_directory_stem() + op.execute('DROP INDEX IF EXISTS file_group_directory_stem_idx') + op.execute('CREATE UNIQUE INDEX file_group_directory_stem_idx ON file_group (directory, stem)') + + +def downgrade(): + op.execute('DROP INDEX IF EXISTS file_group_directory_stem_idx') + op.execute('CREATE INDEX IF NOT EXISTS file_group_directory_stem_idx ON file_group (directory, stem)') diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index bfe815b5..196636e9 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -58,7 +58,8 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix', 'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories', 'get_normalized_ignored_directories', 'sanitize_ignored_directories', 'cached_search_total', 'search_filter_cache_key', 'count_file_groups', - 'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems'] + 'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems', + 'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem'] def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]: @@ -724,17 +725,17 @@ def _upsert_files(files: List[pathlib.Path], values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem) non_primary_files.update(i for i in group if i != primary_path) except NoPrimaryFile: - # Cannot find primary path, create a `file_group` for each file. - for primary_path in group: - primary_path: pathlib.Path - # The primary mimetype allows modelers to find its file_groups. - mimetype = get_mimetype(primary_path) - modification_datetime = from_timestamp(primary_path.stat().st_mtime) - size = primary_path.stat().st_size - files_json = json.dumps(_paths_to_files_dict([primary_path])) - stem, suffix = split_path_stem_and_suffix(primary_path) - suffix = (suffix or '').lower() or None - values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem) + # Same stem still means one FileGroup. Pick a stable primary so + # (directory, stem) stays unique even when nothing is modelable. + primary_path = sorted(group)[0] + mimetype = get_mimetype(primary_path) + modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group)) + size = sum(i.stat().st_size for i in group) + files_json = json.dumps(_paths_to_files_dict(group)) + stem, suffix = split_path_stem_and_suffix(primary_path) + suffix = (suffix or '').lower() or None + values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem) + non_primary_files.update(i for i in group if i != primary_path) except FileNotFoundError as e: # A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.). refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e) @@ -752,6 +753,12 @@ def _upsert_files(files: List[pathlib.Path], '(SELECT value FROM json_each(?))', (json.dumps(primary_paths),)) previously_indexed = {i['primary_path'] for i in curs.fetchall()} + if non_primary_files: + # Delete before INSERT so a new primary_path that shares (directory, stem) + # with an old sidecar FileGroup does not hit the unique index. + curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))', + (json.dumps(list(map(str, non_primary_files))),)) + placeholders, params = values_clause(values) stmt = f''' INSERT INTO file_group @@ -797,11 +804,6 @@ def _upsert_files(files: List[pathlib.Path], else: refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}') - if non_primary_files: - # New files may have been added which change what primary paths exist. Delete any file_groups which - # use paths which are not primary. - curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))', - (json.dumps(list(map(str, non_primary_files))),)) non_primary_files.clear() # Report progress after each chunk @@ -876,6 +878,84 @@ def ensure_file_group_stems(): set_stem_algorithm_version(STEM_ALGORITHM_VERSION) +def backfill_null_file_group_stems(): + """Fill file_group.stem for rows that never got one. Used before creating the unique index.""" + with get_db_curs() as curs: + curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''") + rows = list(curs.fetchall()) + if not rows: + return + updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows] + with get_db_curs(commit=True) as curs: + for chunk in chunks(updates, 500): + curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk) + + +def dedupe_file_groups_by_directory_stem() -> int: + """Collapse FileGroups that share (directory, stem). + + Prefers a row with a domain model, then tags, then non-empty files, then highest id. + Tags and collection membership on losers are moved onto the winner. Returns the + number of FileGroups deleted. + """ + with get_db_curs() as curs: + curs.execute(''' + SELECT directory, stem + FROM file_group + WHERE directory IS NOT NULL + AND stem IS NOT NULL + AND stem != '' + GROUP BY directory, stem + HAVING COUNT(*) > 1 + ''') + keys = [(row['directory'], row['stem']) for row in curs.fetchall()] + + deleted = 0 + for directory, stem in keys: + with get_db_curs() as curs: + curs.execute(''' + SELECT fg.id, + CASE + WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0 + ELSE json_array_length(fg.files) + END AS file_count, + CASE + WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1 + ELSE 0 + END AS has_model, + (SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count + FROM file_group fg + WHERE fg.directory = ? + AND fg.stem = ? + ''', (directory, stem)) + members = list(curs.fetchall()) + members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True) + winner_id = members[0]['id'] + loser_ids = [m['id'] for m in members[1:]] + with get_db_curs(commit=True) as curs: + for loser_id in loser_ids: + curs.execute(''' + INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at) + SELECT tag_id, ?, created_at + FROM tag_file + WHERE file_group_id = ? + ''', (winner_id, loser_id)) + curs.execute(''' + UPDATE collection_item + SET file_group_id = ? + WHERE file_group_id = ? + AND collection_id NOT IN (SELECT collection_id + FROM (SELECT collection_id + FROM collection_item + WHERE file_group_id = ?)) + ''', (winner_id, loser_id, winner_id)) + curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,)) + deleted += 1 + if deleted: + refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)') + return deleted + + async def apply_indexers(progress_callback: Callable[[int, int], None] = None): """Index leftover FileGroups that no modeler claimed. diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index e13d710c..edc9a3c3 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -6,7 +6,7 @@ import urllib.parse from datetime import datetime from typing import List, Type, Optional, Iterable -from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON +from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON, or_ from sqlalchemy import types from sqlalchemy.orm import deferred, relationship, Session @@ -83,8 +83,9 @@ class FileGroup(ModelHelper, Base): Index('file_group_size_ix', 'size'), Index('file_group_url_idx', 'url'), Index('file_group_viewed_idx', 'viewed'), - # Speeds compare joins. Not UNIQUE: duplicates are collapsed at refresh time. - Index('file_group_directory_stem_idx', 'directory', 'stem'), + # Identity of a FileGroup. Duplicates are collapsed by + # dedupe_file_groups_by_directory_stem before this unique index is created. + Index('file_group_directory_stem_idx', 'directory', 'stem', unique=True), ) # SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid). id: int = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True) @@ -523,7 +524,16 @@ class FileGroup(ModelHelper, Base): # Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed) paths = tuple(sanitize_filename_surrogates(p) for p in paths) - existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all() + primary_path = get_primary_file(paths) if paths else None + stem = split_path_stem_and_suffix(primary_path)[0] if primary_path else None + directory = str(primary_path.parent) if primary_path else None + filters = [FileGroup.primary_path.in_(list(map(str, paths)))] + if stem: + filters.append((FileGroup.directory == directory) & (FileGroup.stem == stem)) + existing_groups = unique_by_predicate( + session.query(FileGroup).filter(or_(*filters)).all(), + lambda fg: fg.id, + ) logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}') if len(existing_groups) == 0: # These paths have not been used previously, create a new FileGroup. diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index e831bb53..c3b5e6c6 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -15,6 +15,8 @@ import wrolpi.common from modules.videos import Video from wrolpi.common import timer, get_wrolpi_config from wrolpi.conftest import await_switches +from sqlalchemy.exc import IntegrityError + from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile from wrolpi.files import lib, indexers from wrolpi.files.models import FileGroup, Directory @@ -496,6 +498,91 @@ def test_ensure_file_group_stems_recomputes_on_version_change(test_session, test assert lib.get_stem_algorithm_version() == lib.STEM_ALGORITHM_VERSION +def test_file_group_directory_stem_is_unique(test_session, test_directory, make_files_structure): + """Two FileGroups in the same directory cannot share a stem.""" + foo_mp4, foo_png = make_files_structure(['foo.mp4', 'foo.png']) + FileGroup.from_paths(test_session, foo_mp4) + test_session.commit() + + other = FileGroup( + directory=foo_png.parent, + primary_path=foo_png, + stem='foo', + files=[{'path': foo_png.name, 'mimetype': 'image/png'}], + ) + test_session.add(other) + with pytest.raises(IntegrityError): + test_session.commit() + + +@pytest.mark.asyncio +async def test_dedupe_file_groups_merges_tags_onto_winner(async_client, test_session, test_directory, + make_files_structure, tag_factory): + """Duplicates sharing (directory, stem) collapse onto one row; tags on the loser move.""" + from sqlalchemy import text + + foo, sidecar = make_files_structure(['foo.mp4', 'foo.info.json']) + test_session.execute(text('DROP INDEX IF EXISTS file_group_directory_stem_idx')) + + keeper = FileGroup.from_paths(test_session, foo) + keeper.model = 'video' + loser = FileGroup( + directory=foo.parent, + primary_path=sidecar, + stem='foo', + files=[{'path': sidecar.name, 'mimetype': 'application/json'}], + ) + test_session.add(loser) + test_session.commit() + + tag = await tag_factory('keepme') + loser.add_tag(test_session, tag.id) + test_session.commit() + keeper_id, loser_id = keeper.id, loser.id + assert test_session.query(FileGroup).count() == 2 + + deleted = lib.dedupe_file_groups_by_directory_stem() + test_session.expire_all() + + assert deleted == 1 + remaining = test_session.query(FileGroup).one() + assert remaining.id == keeper_id + assert remaining.tag_names == ['keepme'] + assert test_session.query(FileGroup).filter_by(id=loser_id).one_or_none() is None + + +@pytest.mark.asyncio +async def test_dedupe_file_groups_prefers_tagged_row(async_client, test_session, test_directory, + make_files_structure, tag_factory): + """The tagged FileGroup wins even when it has the lower id and no model.""" + from sqlalchemy import text + + a, b = make_files_structure(['bar.mp4', 'bar.png']) + test_session.execute(text('DROP INDEX IF EXISTS file_group_directory_stem_idx')) + first = FileGroup.from_paths(test_session, a) + second = FileGroup( + directory=b.parent, + primary_path=b, + stem='bar', + files=[{'path': b.name, 'mimetype': 'image/png'}], + model='video', + ) + test_session.add(second) + test_session.commit() + assert first.id < second.id + tag = await tag_factory('tagged-first') + first.add_tag(test_session, tag.id) + test_session.commit() + + lib.dedupe_file_groups_by_directory_stem() + test_session.expire_all() + + remaining = test_session.query(FileGroup).one() + # has_model ranks above tags, so the modeled later row wins; tags move onto it. + assert remaining.id == second.id + assert remaining.tag_names == ['tagged-first'] + + @pytest.mark.asyncio async def test_file_group_tag(test_session, make_files_structure, test_directory, tag_factory, await_switches, refresh_files): @@ -822,7 +909,7 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e @pytest.mark.asyncio async def test_refresh_files_no_groups(async_client, test_session, test_directory, make_files_structure, zip_file_factory, refresh_files): - """Files that share a name, but cannot be grouped into a FileGroup have their own FileGroups.""" + """Files that share a stem are one FileGroup even when neither is a modeled primary.""" foo_txt, foo_zip = make_files_structure({ 'foo.txt': 'text', 'foo.zip': zip_file_factory(), @@ -831,11 +918,12 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director await refresh_files() - # Two distinct FileGroups. - assert test_session.query(FileGroup).count() == 2 - txt, zip_ = test_session.query(FileGroup) - assert txt.primary_path == foo_txt and txt.size == foo_txt.stat().st_size - assert zip_.primary_path == foo_zip and zip_.size == foo_zip.stat().st_size + # Same stem is one FileGroup even when neither file is a modeled primary. + assert test_session.query(FileGroup).count() == 1 + fg = test_session.query(FileGroup).one() + names = {pathlib.Path(f['path']).name for f in fg.files} + assert names == {'foo.txt', 'foo.zip'} + assert fg.size == foo_txt.stat().st_size + foo_zip.stat().st_size @pytest.mark.asyncio @@ -890,53 +978,34 @@ async def test_upsert_directories_chunks_large_batches(test_session, test_direct @pytest.mark.asyncio async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3): - """A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be - migrated.""" + """from_paths on a sidecar finds the existing (directory, stem) FileGroup instead of inserting a duplicate.""" vid, srt = make_files_structure({ 'vid.mp4': video_bytes, 'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(), }) one, two = await tag_factory(), await tag_factory() vid_group = FileGroup.from_paths(test_session, vid) - srt_group = FileGroup.from_paths(test_session, srt) - test_session.add_all([vid_group, srt_group]) - test_session.flush([vid_group, srt_group]) + test_session.flush([vid_group]) vid_tag_file = vid_group.add_tag(test_session, one.name) - srt_tag_file = srt_group.add_tag(test_session, two.name) - test_session.flush([vid_tag_file, srt_tag_file]) tag_file_created_at = vid_tag_file.created_at + test_session.commit() + + srt_group = FileGroup.from_paths(test_session, srt) + assert srt_group.id == vid_group.id + srt_tag_file = srt_group.add_tag(test_session, two.name) srt_file_created_at = srt_tag_file.created_at test_session.commit() - assert vid_group.mimetype == 'video/mp4' - assert srt_group.mimetype == 'text/srt' - - # Add srt to vid_group so both groups have it (simulates race condition scenario) - vid_group.append_files(srt) + merged = FileGroup.from_paths(test_session, vid, srt) test_session.commit() - - # Verify both groups have the srt file - assert 'vid.srt' in [f['path'] for f in vid_group.files] - assert 'vid.srt' in [f['path'] for f in srt_group.files] - - # Both FileGroups are merged. - vid = FileGroup.from_paths(test_session, vid, srt) - test_session.commit() - # files now stores relative filenames as strings, use my_files() to get resolved Paths - assert {i['path'].name for i in vid.my_files()} == {'vid.mp4', 'vid.srt'} - + assert merged.id == vid_group.id assert test_session.query(FileGroup).count() == 1 - assert set(vid.tag_names) == {'one', 'two'} - assert {i['path'].name for i in vid.my_files()} == {'vid.mp4', 'vid.srt'} - # TagFile.created_at is preserved. - assert [i for i in vid.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at - assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at - # Size is combined - assert vid.size > len(video_bytes) - assert vid.mimetype == 'video/mp4' - - # Verify no duplicate files were created during merge - filenames = [f['path'] for f in vid.files] + assert set(merged.tag_names) == {'one', 'two'} + assert {i['path'].name for i in merged.my_files()} == {'vid.mp4', 'vid.srt'} + assert [i for i in merged.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at + assert [i for i in merged.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at + assert merged.mimetype == 'video/mp4' + filenames = [f['path'] for f in merged.files] assert len(filenames) == len(set(filenames)), f"Duplicate files found: {filenames}" diff --git a/wrolpi/files/test/test_worker.py b/wrolpi/files/test/test_worker.py index 5d7cdb9d..4883bb28 100644 --- a/wrolpi/files/test/test_worker.py +++ b/wrolpi/files/test/test_worker.py @@ -1,6 +1,7 @@ """Tests for the file comparison worker.""" import asyncio import os +import pathlib from unittest import mock import pytest @@ -874,9 +875,10 @@ async def test_file_worker_deletes_removed_file_groups( file_worker.private_queue.put_nowait(task) await file_worker.process_queue() - # Verify FileGroup was deleted + # Verify FileGroup was deleted (do not key off id: SQLite may reuse it for the baseline upsert). test_session.expire_all() - assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None + remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)} + assert 'file1.txt' not in remaining @pytest.mark.asyncio @@ -912,10 +914,13 @@ async def test_file_worker_deletes_tagged_file_groups_auto_removes_tags( file_worker.private_queue.put_nowait(task) await file_worker.process_queue() - # Verify FileGroup and TagFile were deleted (cascade) + # Verify FileGroup and TagFile were deleted (cascade). Do not key off recycled SQLite ids. test_session.expire_all() - assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None - assert test_session.query(TagFile).filter_by(file_group_id=fg_id).count() == 0 + remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)} + assert 'file1.txt' not in remaining + assert test_session.query(TagFile).join(FileGroup).filter( + FileGroup.primary_path.like('%/file1.txt') + ).count() == 0 @pytest.mark.asyncio @@ -1120,11 +1125,10 @@ async def test_file_worker_deletes_multiple_file_groups( file_worker.private_queue.put_nowait(task) await file_worker.process_queue() - # Verify all FileGroups were deleted + # Verify all FileGroups were deleted (do not key off recycled SQLite ids). test_session.expire_all() - assert test_session.query(FileGroup).filter( - FileGroup.id.in_([fg1_id, fg2_id, fg3_id]) - ).count() == 0 + remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)} + assert remaining.isdisjoint({'file1.txt', 'file2.txt', 'file3.txt'}) @pytest.mark.asyncio @@ -1681,78 +1685,15 @@ async def test_upsert_file_groups_tracks_actual_db_progress(async_client, test_s @pytest.mark.asyncio -async def test_compare_file_groups_duplicate_deleted_filegroups(test_session, test_directory, make_files_structure): - """When multiple FileGroups share the same (directory, stem) and all files are deleted, - all duplicates should appear in the deleted diffs — not just the last one processed.""" - make_files_structure([ - 'docs/file1.txt', - 'docs/file1.json', - ]) - - docs_dir = str(test_directory / 'docs') - - # Create 3 separate FileGroups for the same (directory, stem) — simulating a botched move. - # Use different primary_paths (unique constraint) but same directory and stem. - fg1 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.txt'), - files=[{'path': 'file1.txt', 'mimetype': 'text/plain', 'size': 0}]) - fg2 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.json'), - files=[{'path': 'file1.json', 'mimetype': 'application/json', 'size': 0}]) - # Third ghost with empty files list — use a unique primary_path - (test_directory / 'docs/file1.bak').touch() - fg3 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.bak'), - files=[]) - test_session.add_all([fg1, fg2, fg3]) - test_session.commit() - - fg1_id, fg2_id, fg3_id = fg1.id, fg2.id, fg3.id - - # Delete all files from disk - for f in (test_directory / 'docs').iterdir(): - f.unlink() +async def test_compare_file_groups_unique_stem_is_one_group(test_session, test_directory, make_files_structure): + """Files that share a stem are one FileGroup; compare reports a single group.""" + make_files_structure(['docs/file1.txt', 'docs/file1.json']) + from wrolpi.files.lib import _upsert_files + _upsert_files(list((test_directory / 'docs').iterdir())) result = await compare_file_groups(test_directory) - - # ALL 3 FileGroup IDs must appear in deleted diffs - deleted_ids = {d.file_group_id for d in result.deleted} - assert fg1_id in deleted_ids, f'fg1 ({fg1_id}) missing from deleted: {deleted_ids}' - assert fg2_id in deleted_ids, f'fg2 ({fg2_id}) missing from deleted: {deleted_ids}' - assert fg3_id in deleted_ids, f'fg3 ({fg3_id}) missing from deleted: {deleted_ids}' - assert len(result.new) == 0 - - -@pytest.mark.asyncio -async def test_compare_file_groups_duplicate_keeps_one(test_session, test_directory, make_files_structure): - """When duplicate FileGroups exist and files are on disk, one should be kept (modified/unchanged) - and the others should be marked for deletion.""" - make_files_structure([ - 'docs/file1.txt', - ]) - - docs_dir = str(test_directory / 'docs') - - # Create 2 FileGroups with same (directory, stem) - fg1 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.txt'), - files=[{'path': 'file1.txt', 'mimetype': 'text/plain', 'size': 0}]) - (test_directory / 'docs/file1.json').touch() - fg2 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.json'), - files=[{'path': 'file1.json', 'mimetype': 'application/json', 'size': 0}]) - test_session.add_all([fg1, fg2]) - test_session.commit() - - fg1_id, fg2_id = fg1.id, fg2.id - # Remove the .json so only .txt remains - (test_directory / 'docs/file1.json').unlink() - - result = await compare_file_groups(test_directory) - - # One FG should have the file on disk, the other should be deleted as duplicate - deleted_ids = {d.file_group_id for d in result.deleted} - all_diff_ids = {d.file_group_id for d in result.deleted + result.modified + result.unchanged} - assert fg1_id in all_diff_ids - assert fg2_id in all_diff_ids - # At least one must be in deleted (the duplicate) - assert len(deleted_ids) >= 1 - assert deleted_ids & {fg1_id, fg2_id} + assert len(result.new) + len(result.unchanged) + len(result.modified) == 1 + assert test_session.query(FileGroup).count() == 1 @pytest.mark.asyncio diff --git a/wrolpi/files/worker.py b/wrolpi/files/worker.py index 764ed0dd..90a23277 100644 --- a/wrolpi/files/worker.py +++ b/wrolpi/files/worker.py @@ -1492,10 +1492,11 @@ class FileWorker: logger.info(f'Refreshing {len(file_paths)} files directly') file_result = await self._refresh_files_directly(file_paths, expand_stems=task.expand_stems) - # Process file results + # Delete first so a new primary_path cannot collide on (directory, stem) + # with a duplicate FileGroup that compare already marked deleted. self._cleanup_modified_models(file_result.modified) - await self._upsert_file_groups(file_result.new + file_result.modified) await self._delete_file_groups(file_result.deleted) + await self._upsert_file_groups(file_result.new + file_result.modified) # Process directories with full scan (existing path) if dir_paths: @@ -1520,10 +1521,11 @@ class FileWorker: f'{len(dir_result.deleted)} deleted, {len(dir_result.unchanged)} unchanged' ) - # Process directory results + # Delete first so a new primary_path cannot collide on (directory, stem) + # with a duplicate FileGroup that compare already marked deleted. self._cleanup_modified_models(dir_result.modified) - await self._upsert_file_groups(dir_result.new + dir_result.modified) await self._delete_file_groups(dir_result.deleted) + await self._upsert_file_groups(dir_result.new + dir_result.modified) # Run indexers, modelers, and cleanup once for all changes await self._apply_post_processing(is_global_refresh=is_global_refresh) diff --git a/wrolpi/test/test_db_bootstrap.py b/wrolpi/test/test_db_bootstrap.py index 056c7eb4..045a050d 100644 --- a/wrolpi/test/test_db_bootstrap.py +++ b/wrolpi/test/test_db_bootstrap.py @@ -66,6 +66,57 @@ def test_compare_db_version_uninitialized_file(test_directory): assert db_bootstrap.compare_db_version() == 'current' +def test_unique_stem_migration_collapses_duplicates(test_directory): + """The unique-index migration merges duplicate (directory, stem) rows and keeps tags.""" + import sqlite3 + from alembic.config import Config + from alembic import command + from wrolpi.db import get_db_uri, get_db_file + from wrolpi.vars import PROJECT_DIR + + (test_directory / 'config').mkdir(parents=True, exist_ok=True) + config = Config(str(PROJECT_DIR / 'alembic.ini')) + config.set_main_option('sqlalchemy.url', get_db_uri()) + command.upgrade(config, '2026_09_16_1200') + + db_file = get_db_file() + directory = str(test_directory) + with sqlite3.connect(db_file) as conn: + conn.execute('PRAGMA foreign_keys=ON') + conn.execute( + 'INSERT INTO file_group (directory, primary_path, files, stem, indexed) VALUES (?,?,?,?,0)', + (directory, str(test_directory / 'dup.mp4'), '[]', 'dup'), + ) + conn.execute( + 'INSERT INTO file_group (directory, primary_path, files, stem, indexed) VALUES (?,?,?,?,0)', + (directory, str(test_directory / 'dup.info.json'), '[]', 'dup'), + ) + first_id, second_id = [r[0] for r in conn.execute('SELECT id FROM file_group ORDER BY id').fetchall()] + conn.execute("INSERT INTO tag (name, color) VALUES ('migrated', '#111111')") + tag_id = conn.execute('SELECT id FROM tag').fetchone()[0] + # Tag the lower-id row so it wins over the later untagged duplicate. + conn.execute( + 'INSERT INTO tag_file (tag_id, file_group_id) VALUES (?, ?)', + (tag_id, first_id), + ) + conn.commit() + assert conn.execute('SELECT COUNT(*) FROM file_group').fetchone()[0] == 2 + assert first_id < second_id + + command.upgrade(config, 'head') + + with sqlite3.connect(db_file) as conn: + rows = list(conn.execute('SELECT id, stem FROM file_group')) + assert len(rows) == 1 + assert rows[0][0] == first_id + tagged = conn.execute('SELECT file_group_id FROM tag_file').fetchone()[0] + assert tagged == first_id + sql = conn.execute( + "SELECT sql FROM sqlite_master WHERE type='index' AND name='file_group_directory_stem_idx'" + ).fetchone()[0] + assert 'UNIQUE' in sql.upper() + + def test_bootstrap_lock(test_directory): """Only one process can hold the bootstrap lock.""" with db_bootstrap.bootstrap_lock() as acquired: