files: unique (directory, stem) after collapsing duplicate FileGroups

713 added stem and a non-unique index.  This release backfills any NULL
stems, merges leftover duplicates (tags and collection membership move
onto the winner), and creates a UNIQUE index.

Upsert deletes sidecar FileGroups before INSERT so a new primary cannot
collide.  Refresh deletes before upsert for the same reason.
from_paths looks up by stem so adding a sidecar does not insert a second
row.  Same-stem files without a modeled primary stay one FileGroup.
This commit is contained in:
Roland 2026-09-18 09:14:06 -06:00
parent 3a6cf1fa3a
commit 41434533ec
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
7 changed files with 327 additions and 144 deletions

View file

@ -0,0 +1,30 @@
"""Unique (directory, stem) on file_group.
713 added the stem column and a non-unique index. This release collapses any
remaining duplicate (directory, stem) rows (moving tags and collection
membership onto the winner) and makes the index UNIQUE.
Revision ID: 2026_09_18_1200
Revises: 2026_09_16_1200
"""
from alembic import op
# revision identifiers, used by Alembic.
revision = '2026_09_18_1200'
down_revision = '2026_09_16_1200'
branch_labels = None
depends_on = None
def upgrade():
from wrolpi.files.lib import backfill_null_file_group_stems, dedupe_file_groups_by_directory_stem
backfill_null_file_group_stems()
dedupe_file_groups_by_directory_stem()
op.execute('DROP INDEX IF EXISTS file_group_directory_stem_idx')
op.execute('CREATE UNIQUE INDEX file_group_directory_stem_idx ON file_group (directory, stem)')
def downgrade():
op.execute('DROP INDEX IF EXISTS file_group_directory_stem_idx')
op.execute('CREATE INDEX IF NOT EXISTS file_group_directory_stem_idx ON file_group (directory, stem)')

View file

@ -58,7 +58,8 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix',
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems']
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems',
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem']
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
@ -724,17 +725,17 @@ def _upsert_files(files: List[pathlib.Path],
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
non_primary_files.update(i for i in group if i != primary_path)
except NoPrimaryFile:
# Cannot find primary path, create a `file_group` for each file.
for primary_path in group:
primary_path: pathlib.Path
# The primary mimetype allows modelers to find its file_groups.
mimetype = get_mimetype(primary_path)
modification_datetime = from_timestamp(primary_path.stat().st_mtime)
size = primary_path.stat().st_size
files_json = json.dumps(_paths_to_files_dict([primary_path]))
stem, suffix = split_path_stem_and_suffix(primary_path)
suffix = (suffix or '').lower() or None
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
# Same stem still means one FileGroup. Pick a stable primary so
# (directory, stem) stays unique even when nothing is modelable.
primary_path = sorted(group)[0]
mimetype = get_mimetype(primary_path)
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
size = sum(i.stat().st_size for i in group)
files_json = json.dumps(_paths_to_files_dict(group))
stem, suffix = split_path_stem_and_suffix(primary_path)
suffix = (suffix or '').lower() or None
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
non_primary_files.update(i for i in group if i != primary_path)
except FileNotFoundError as e:
# A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.).
refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e)
@ -752,6 +753,12 @@ def _upsert_files(files: List[pathlib.Path],
'(SELECT value FROM json_each(?))', (json.dumps(primary_paths),))
previously_indexed = {i['primary_path'] for i in curs.fetchall()}
if non_primary_files:
# Delete before INSERT so a new primary_path that shares (directory, stem)
# with an old sidecar FileGroup does not hit the unique index.
curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))',
(json.dumps(list(map(str, non_primary_files))),))
placeholders, params = values_clause(values)
stmt = f'''
INSERT INTO file_group
@ -797,11 +804,6 @@ def _upsert_files(files: List[pathlib.Path],
else:
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
if non_primary_files:
# New files may have been added which change what primary paths exist. Delete any file_groups which
# use paths which are not primary.
curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))',
(json.dumps(list(map(str, non_primary_files))),))
non_primary_files.clear()
# Report progress after each chunk
@ -876,6 +878,84 @@ def ensure_file_group_stems():
set_stem_algorithm_version(STEM_ALGORITHM_VERSION)
def backfill_null_file_group_stems():
"""Fill file_group.stem for rows that never got one. Used before creating the unique index."""
with get_db_curs() as curs:
curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''")
rows = list(curs.fetchall())
if not rows:
return
updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows]
with get_db_curs(commit=True) as curs:
for chunk in chunks(updates, 500):
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
def dedupe_file_groups_by_directory_stem() -> int:
"""Collapse FileGroups that share (directory, stem).
Prefers a row with a domain model, then tags, then non-empty files, then highest id.
Tags and collection membership on losers are moved onto the winner. Returns the
number of FileGroups deleted.
"""
with get_db_curs() as curs:
curs.execute('''
SELECT directory, stem
FROM file_group
WHERE directory IS NOT NULL
AND stem IS NOT NULL
AND stem != ''
GROUP BY directory, stem
HAVING COUNT(*) > 1
''')
keys = [(row['directory'], row['stem']) for row in curs.fetchall()]
deleted = 0
for directory, stem in keys:
with get_db_curs() as curs:
curs.execute('''
SELECT fg.id,
CASE
WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0
ELSE json_array_length(fg.files)
END AS file_count,
CASE
WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1
ELSE 0
END AS has_model,
(SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count
FROM file_group fg
WHERE fg.directory = ?
AND fg.stem = ?
''', (directory, stem))
members = list(curs.fetchall())
members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True)
winner_id = members[0]['id']
loser_ids = [m['id'] for m in members[1:]]
with get_db_curs(commit=True) as curs:
for loser_id in loser_ids:
curs.execute('''
INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at)
SELECT tag_id, ?, created_at
FROM tag_file
WHERE file_group_id = ?
''', (winner_id, loser_id))
curs.execute('''
UPDATE collection_item
SET file_group_id = ?
WHERE file_group_id = ?
AND collection_id NOT IN (SELECT collection_id
FROM (SELECT collection_id
FROM collection_item
WHERE file_group_id = ?))
''', (winner_id, loser_id, winner_id))
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
deleted += 1
if deleted:
refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)')
return deleted
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
"""Index leftover FileGroups that no modeler claimed.

View file

@ -6,7 +6,7 @@ import urllib.parse
from datetime import datetime
from typing import List, Type, Optional, Iterable
from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON
from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON, or_
from sqlalchemy import types
from sqlalchemy.orm import deferred, relationship, Session
@ -83,8 +83,9 @@ class FileGroup(ModelHelper, Base):
Index('file_group_size_ix', 'size'),
Index('file_group_url_idx', 'url'),
Index('file_group_viewed_idx', 'viewed'),
# Speeds compare joins. Not UNIQUE: duplicates are collapsed at refresh time.
Index('file_group_directory_stem_idx', 'directory', 'stem'),
# Identity of a FileGroup. Duplicates are collapsed by
# dedupe_file_groups_by_directory_stem before this unique index is created.
Index('file_group_directory_stem_idx', 'directory', 'stem', unique=True),
)
# SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid).
id: int = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True)
@ -523,7 +524,16 @@ class FileGroup(ModelHelper, Base):
# Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed)
paths = tuple(sanitize_filename_surrogates(p) for p in paths)
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
primary_path = get_primary_file(paths) if paths else None
stem = split_path_stem_and_suffix(primary_path)[0] if primary_path else None
directory = str(primary_path.parent) if primary_path else None
filters = [FileGroup.primary_path.in_(list(map(str, paths)))]
if stem:
filters.append((FileGroup.directory == directory) & (FileGroup.stem == stem))
existing_groups = unique_by_predicate(
session.query(FileGroup).filter(or_(*filters)).all(),
lambda fg: fg.id,
)
logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}')
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.

View file

@ -15,6 +15,8 @@ import wrolpi.common
from modules.videos import Video
from wrolpi.common import timer, get_wrolpi_config
from wrolpi.conftest import await_switches
from sqlalchemy.exc import IntegrityError
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
from wrolpi.files import lib, indexers
from wrolpi.files.models import FileGroup, Directory
@ -496,6 +498,91 @@ def test_ensure_file_group_stems_recomputes_on_version_change(test_session, test
assert lib.get_stem_algorithm_version() == lib.STEM_ALGORITHM_VERSION
def test_file_group_directory_stem_is_unique(test_session, test_directory, make_files_structure):
"""Two FileGroups in the same directory cannot share a stem."""
foo_mp4, foo_png = make_files_structure(['foo.mp4', 'foo.png'])
FileGroup.from_paths(test_session, foo_mp4)
test_session.commit()
other = FileGroup(
directory=foo_png.parent,
primary_path=foo_png,
stem='foo',
files=[{'path': foo_png.name, 'mimetype': 'image/png'}],
)
test_session.add(other)
with pytest.raises(IntegrityError):
test_session.commit()
@pytest.mark.asyncio
async def test_dedupe_file_groups_merges_tags_onto_winner(async_client, test_session, test_directory,
make_files_structure, tag_factory):
"""Duplicates sharing (directory, stem) collapse onto one row; tags on the loser move."""
from sqlalchemy import text
foo, sidecar = make_files_structure(['foo.mp4', 'foo.info.json'])
test_session.execute(text('DROP INDEX IF EXISTS file_group_directory_stem_idx'))
keeper = FileGroup.from_paths(test_session, foo)
keeper.model = 'video'
loser = FileGroup(
directory=foo.parent,
primary_path=sidecar,
stem='foo',
files=[{'path': sidecar.name, 'mimetype': 'application/json'}],
)
test_session.add(loser)
test_session.commit()
tag = await tag_factory('keepme')
loser.add_tag(test_session, tag.id)
test_session.commit()
keeper_id, loser_id = keeper.id, loser.id
assert test_session.query(FileGroup).count() == 2
deleted = lib.dedupe_file_groups_by_directory_stem()
test_session.expire_all()
assert deleted == 1
remaining = test_session.query(FileGroup).one()
assert remaining.id == keeper_id
assert remaining.tag_names == ['keepme']
assert test_session.query(FileGroup).filter_by(id=loser_id).one_or_none() is None
@pytest.mark.asyncio
async def test_dedupe_file_groups_prefers_tagged_row(async_client, test_session, test_directory,
make_files_structure, tag_factory):
"""The tagged FileGroup wins even when it has the lower id and no model."""
from sqlalchemy import text
a, b = make_files_structure(['bar.mp4', 'bar.png'])
test_session.execute(text('DROP INDEX IF EXISTS file_group_directory_stem_idx'))
first = FileGroup.from_paths(test_session, a)
second = FileGroup(
directory=b.parent,
primary_path=b,
stem='bar',
files=[{'path': b.name, 'mimetype': 'image/png'}],
model='video',
)
test_session.add(second)
test_session.commit()
assert first.id < second.id
tag = await tag_factory('tagged-first')
first.add_tag(test_session, tag.id)
test_session.commit()
lib.dedupe_file_groups_by_directory_stem()
test_session.expire_all()
remaining = test_session.query(FileGroup).one()
# has_model ranks above tags, so the modeled later row wins; tags move onto it.
assert remaining.id == second.id
assert remaining.tag_names == ['tagged-first']
@pytest.mark.asyncio
async def test_file_group_tag(test_session, make_files_structure, test_directory, tag_factory, await_switches,
refresh_files):
@ -822,7 +909,7 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e
@pytest.mark.asyncio
async def test_refresh_files_no_groups(async_client, test_session, test_directory, make_files_structure,
zip_file_factory, refresh_files):
"""Files that share a name, but cannot be grouped into a FileGroup have their own FileGroups."""
"""Files that share a stem are one FileGroup even when neither is a modeled primary."""
foo_txt, foo_zip = make_files_structure({
'foo.txt': 'text',
'foo.zip': zip_file_factory(),
@ -831,11 +918,12 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director
await refresh_files()
# Two distinct FileGroups.
assert test_session.query(FileGroup).count() == 2
txt, zip_ = test_session.query(FileGroup)
assert txt.primary_path == foo_txt and txt.size == foo_txt.stat().st_size
assert zip_.primary_path == foo_zip and zip_.size == foo_zip.stat().st_size
# Same stem is one FileGroup even when neither file is a modeled primary.
assert test_session.query(FileGroup).count() == 1
fg = test_session.query(FileGroup).one()
names = {pathlib.Path(f['path']).name for f in fg.files}
assert names == {'foo.txt', 'foo.zip'}
assert fg.size == foo_txt.stat().st_size + foo_zip.stat().st_size
@pytest.mark.asyncio
@ -890,53 +978,34 @@ async def test_upsert_directories_chunks_large_batches(test_session, test_direct
@pytest.mark.asyncio
async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory,
video_bytes, srt_file3):
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
migrated."""
"""from_paths on a sidecar finds the existing (directory, stem) FileGroup instead of inserting a duplicate."""
vid, srt = make_files_structure({
'vid.mp4': video_bytes,
'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(),
})
one, two = await tag_factory(), await tag_factory()
vid_group = FileGroup.from_paths(test_session, vid)
srt_group = FileGroup.from_paths(test_session, srt)
test_session.add_all([vid_group, srt_group])
test_session.flush([vid_group, srt_group])
test_session.flush([vid_group])
vid_tag_file = vid_group.add_tag(test_session, one.name)
srt_tag_file = srt_group.add_tag(test_session, two.name)
test_session.flush([vid_tag_file, srt_tag_file])
tag_file_created_at = vid_tag_file.created_at
test_session.commit()
srt_group = FileGroup.from_paths(test_session, srt)
assert srt_group.id == vid_group.id
srt_tag_file = srt_group.add_tag(test_session, two.name)
srt_file_created_at = srt_tag_file.created_at
test_session.commit()
assert vid_group.mimetype == 'video/mp4'
assert srt_group.mimetype == 'text/srt'
# Add srt to vid_group so both groups have it (simulates race condition scenario)
vid_group.append_files(srt)
merged = FileGroup.from_paths(test_session, vid, srt)
test_session.commit()
# Verify both groups have the srt file
assert 'vid.srt' in [f['path'] for f in vid_group.files]
assert 'vid.srt' in [f['path'] for f in srt_group.files]
# Both FileGroups are merged.
vid = FileGroup.from_paths(test_session, vid, srt)
test_session.commit()
# files now stores relative filenames as strings, use my_files() to get resolved Paths
assert {i['path'].name for i in vid.my_files()} == {'vid.mp4', 'vid.srt'}
assert merged.id == vid_group.id
assert test_session.query(FileGroup).count() == 1
assert set(vid.tag_names) == {'one', 'two'}
assert {i['path'].name for i in vid.my_files()} == {'vid.mp4', 'vid.srt'}
# TagFile.created_at is preserved.
assert [i for i in vid.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at
assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at
# Size is combined
assert vid.size > len(video_bytes)
assert vid.mimetype == 'video/mp4'
# Verify no duplicate files were created during merge
filenames = [f['path'] for f in vid.files]
assert set(merged.tag_names) == {'one', 'two'}
assert {i['path'].name for i in merged.my_files()} == {'vid.mp4', 'vid.srt'}
assert [i for i in merged.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at
assert [i for i in merged.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at
assert merged.mimetype == 'video/mp4'
filenames = [f['path'] for f in merged.files]
assert len(filenames) == len(set(filenames)), f"Duplicate files found: {filenames}"

View file

@ -1,6 +1,7 @@
"""Tests for the file comparison worker."""
import asyncio
import os
import pathlib
from unittest import mock
import pytest
@ -874,9 +875,10 @@ async def test_file_worker_deletes_removed_file_groups(
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify FileGroup was deleted
# Verify FileGroup was deleted (do not key off id: SQLite may reuse it for the baseline upsert).
test_session.expire_all()
assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None
remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)}
assert 'file1.txt' not in remaining
@pytest.mark.asyncio
@ -912,10 +914,13 @@ async def test_file_worker_deletes_tagged_file_groups_auto_removes_tags(
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify FileGroup and TagFile were deleted (cascade)
# Verify FileGroup and TagFile were deleted (cascade). Do not key off recycled SQLite ids.
test_session.expire_all()
assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None
assert test_session.query(TagFile).filter_by(file_group_id=fg_id).count() == 0
remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)}
assert 'file1.txt' not in remaining
assert test_session.query(TagFile).join(FileGroup).filter(
FileGroup.primary_path.like('%/file1.txt')
).count() == 0
@pytest.mark.asyncio
@ -1120,11 +1125,10 @@ async def test_file_worker_deletes_multiple_file_groups(
file_worker.private_queue.put_nowait(task)
await file_worker.process_queue()
# Verify all FileGroups were deleted
# Verify all FileGroups were deleted (do not key off recycled SQLite ids).
test_session.expire_all()
assert test_session.query(FileGroup).filter(
FileGroup.id.in_([fg1_id, fg2_id, fg3_id])
).count() == 0
remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)}
assert remaining.isdisjoint({'file1.txt', 'file2.txt', 'file3.txt'})
@pytest.mark.asyncio
@ -1681,78 +1685,15 @@ async def test_upsert_file_groups_tracks_actual_db_progress(async_client, test_s
@pytest.mark.asyncio
async def test_compare_file_groups_duplicate_deleted_filegroups(test_session, test_directory, make_files_structure):
"""When multiple FileGroups share the same (directory, stem) and all files are deleted,
all duplicates should appear in the deleted diffs — not just the last one processed."""
make_files_structure([
'docs/file1.txt',
'docs/file1.json',
])
docs_dir = str(test_directory / 'docs')
# Create 3 separate FileGroups for the same (directory, stem) — simulating a botched move.
# Use different primary_paths (unique constraint) but same directory and stem.
fg1 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.txt'),
files=[{'path': 'file1.txt', 'mimetype': 'text/plain', 'size': 0}])
fg2 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.json'),
files=[{'path': 'file1.json', 'mimetype': 'application/json', 'size': 0}])
# Third ghost with empty files list — use a unique primary_path
(test_directory / 'docs/file1.bak').touch()
fg3 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.bak'),
files=[])
test_session.add_all([fg1, fg2, fg3])
test_session.commit()
fg1_id, fg2_id, fg3_id = fg1.id, fg2.id, fg3.id
# Delete all files from disk
for f in (test_directory / 'docs').iterdir():
f.unlink()
async def test_compare_file_groups_unique_stem_is_one_group(test_session, test_directory, make_files_structure):
"""Files that share a stem are one FileGroup; compare reports a single group."""
make_files_structure(['docs/file1.txt', 'docs/file1.json'])
from wrolpi.files.lib import _upsert_files
_upsert_files(list((test_directory / 'docs').iterdir()))
result = await compare_file_groups(test_directory)
# ALL 3 FileGroup IDs must appear in deleted diffs
deleted_ids = {d.file_group_id for d in result.deleted}
assert fg1_id in deleted_ids, f'fg1 ({fg1_id}) missing from deleted: {deleted_ids}'
assert fg2_id in deleted_ids, f'fg2 ({fg2_id}) missing from deleted: {deleted_ids}'
assert fg3_id in deleted_ids, f'fg3 ({fg3_id}) missing from deleted: {deleted_ids}'
assert len(result.new) == 0
@pytest.mark.asyncio
async def test_compare_file_groups_duplicate_keeps_one(test_session, test_directory, make_files_structure):
"""When duplicate FileGroups exist and files are on disk, one should be kept (modified/unchanged)
and the others should be marked for deletion."""
make_files_structure([
'docs/file1.txt',
])
docs_dir = str(test_directory / 'docs')
# Create 2 FileGroups with same (directory, stem)
fg1 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.txt'),
files=[{'path': 'file1.txt', 'mimetype': 'text/plain', 'size': 0}])
(test_directory / 'docs/file1.json').touch()
fg2 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.json'),
files=[{'path': 'file1.json', 'mimetype': 'application/json', 'size': 0}])
test_session.add_all([fg1, fg2])
test_session.commit()
fg1_id, fg2_id = fg1.id, fg2.id
# Remove the .json so only .txt remains
(test_directory / 'docs/file1.json').unlink()
result = await compare_file_groups(test_directory)
# One FG should have the file on disk, the other should be deleted as duplicate
deleted_ids = {d.file_group_id for d in result.deleted}
all_diff_ids = {d.file_group_id for d in result.deleted + result.modified + result.unchanged}
assert fg1_id in all_diff_ids
assert fg2_id in all_diff_ids
# At least one must be in deleted (the duplicate)
assert len(deleted_ids) >= 1
assert deleted_ids & {fg1_id, fg2_id}
assert len(result.new) + len(result.unchanged) + len(result.modified) == 1
assert test_session.query(FileGroup).count() == 1
@pytest.mark.asyncio

View file

@ -1492,10 +1492,11 @@ class FileWorker:
logger.info(f'Refreshing {len(file_paths)} files directly')
file_result = await self._refresh_files_directly(file_paths, expand_stems=task.expand_stems)
# Process file results
# Delete first so a new primary_path cannot collide on (directory, stem)
# with a duplicate FileGroup that compare already marked deleted.
self._cleanup_modified_models(file_result.modified)
await self._upsert_file_groups(file_result.new + file_result.modified)
await self._delete_file_groups(file_result.deleted)
await self._upsert_file_groups(file_result.new + file_result.modified)
# Process directories with full scan (existing path)
if dir_paths:
@ -1520,10 +1521,11 @@ class FileWorker:
f'{len(dir_result.deleted)} deleted, {len(dir_result.unchanged)} unchanged'
)
# Process directory results
# Delete first so a new primary_path cannot collide on (directory, stem)
# with a duplicate FileGroup that compare already marked deleted.
self._cleanup_modified_models(dir_result.modified)
await self._upsert_file_groups(dir_result.new + dir_result.modified)
await self._delete_file_groups(dir_result.deleted)
await self._upsert_file_groups(dir_result.new + dir_result.modified)
# Run indexers, modelers, and cleanup once for all changes
await self._apply_post_processing(is_global_refresh=is_global_refresh)

View file

@ -66,6 +66,57 @@ def test_compare_db_version_uninitialized_file(test_directory):
assert db_bootstrap.compare_db_version() == 'current'
def test_unique_stem_migration_collapses_duplicates(test_directory):
"""The unique-index migration merges duplicate (directory, stem) rows and keeps tags."""
import sqlite3
from alembic.config import Config
from alembic import command
from wrolpi.db import get_db_uri, get_db_file
from wrolpi.vars import PROJECT_DIR
(test_directory / 'config').mkdir(parents=True, exist_ok=True)
config = Config(str(PROJECT_DIR / 'alembic.ini'))
config.set_main_option('sqlalchemy.url', get_db_uri())
command.upgrade(config, '2026_09_16_1200')
db_file = get_db_file()
directory = str(test_directory)
with sqlite3.connect(db_file) as conn:
conn.execute('PRAGMA foreign_keys=ON')
conn.execute(
'INSERT INTO file_group (directory, primary_path, files, stem, indexed) VALUES (?,?,?,?,0)',
(directory, str(test_directory / 'dup.mp4'), '[]', 'dup'),
)
conn.execute(
'INSERT INTO file_group (directory, primary_path, files, stem, indexed) VALUES (?,?,?,?,0)',
(directory, str(test_directory / 'dup.info.json'), '[]', 'dup'),
)
first_id, second_id = [r[0] for r in conn.execute('SELECT id FROM file_group ORDER BY id').fetchall()]
conn.execute("INSERT INTO tag (name, color) VALUES ('migrated', '#111111')")
tag_id = conn.execute('SELECT id FROM tag').fetchone()[0]
# Tag the lower-id row so it wins over the later untagged duplicate.
conn.execute(
'INSERT INTO tag_file (tag_id, file_group_id) VALUES (?, ?)',
(tag_id, first_id),
)
conn.commit()
assert conn.execute('SELECT COUNT(*) FROM file_group').fetchone()[0] == 2
assert first_id < second_id
command.upgrade(config, 'head')
with sqlite3.connect(db_file) as conn:
rows = list(conn.execute('SELECT id, stem FROM file_group'))
assert len(rows) == 1
assert rows[0][0] == first_id
tagged = conn.execute('SELECT file_group_id FROM tag_file').fetchone()[0]
assert tagged == first_id
sql = conn.execute(
"SELECT sql FROM sqlite_master WHERE type='index' AND name='file_group_directory_stem_idx'"
).fetchone()[0]
assert 'UNIQUE' in sql.upper()
def test_bootstrap_lock(test_directory):
"""Only one process can hold the bootstrap lock."""
with db_bootstrap.bootstrap_lock() as acquired: