mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 06:56:18 -05:00
files: unique (directory, stem) after collapsing duplicate FileGroups
713 added stem and a non-unique index. This release backfills any NULL stems, merges leftover duplicates (tags and collection membership move onto the winner), and creates a UNIQUE index. Upsert deletes sidecar FileGroups before INSERT so a new primary cannot collide. Refresh deletes before upsert for the same reason. from_paths looks up by stem so adding a sidecar does not insert a second row. Same-stem files without a modeled primary stay one FileGroup.
This commit is contained in:
parent
3a6cf1fa3a
commit
41434533ec
7 changed files with 327 additions and 144 deletions
30
alembic/versions/2026_09_18_1200_unique_directory_stem.py
Normal file
30
alembic/versions/2026_09_18_1200_unique_directory_stem.py
Normal file
|
|
@ -0,0 +1,30 @@
|
|||
"""Unique (directory, stem) on file_group.
|
||||
|
||||
713 added the stem column and a non-unique index. This release collapses any
|
||||
remaining duplicate (directory, stem) rows (moving tags and collection
|
||||
membership onto the winner) and makes the index UNIQUE.
|
||||
|
||||
Revision ID: 2026_09_18_1200
|
||||
Revises: 2026_09_16_1200
|
||||
"""
|
||||
from alembic import op
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '2026_09_18_1200'
|
||||
down_revision = '2026_09_16_1200'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
|
||||
def upgrade():
|
||||
from wrolpi.files.lib import backfill_null_file_group_stems, dedupe_file_groups_by_directory_stem
|
||||
|
||||
backfill_null_file_group_stems()
|
||||
dedupe_file_groups_by_directory_stem()
|
||||
op.execute('DROP INDEX IF EXISTS file_group_directory_stem_idx')
|
||||
op.execute('CREATE UNIQUE INDEX file_group_directory_stem_idx ON file_group (directory, stem)')
|
||||
|
||||
|
||||
def downgrade():
|
||||
op.execute('DROP INDEX IF EXISTS file_group_directory_stem_idx')
|
||||
op.execute('CREATE INDEX IF NOT EXISTS file_group_directory_stem_idx ON file_group (directory, stem)')
|
||||
|
|
@ -58,7 +58,8 @@ __all__ = ['list_directories_contents', 'delete', 'split_path_stem_and_suffix',
|
|||
'get_tagged_file_groups_by_ids', 'delete_file_groups', 'get_special_directories',
|
||||
'get_normalized_ignored_directories', 'sanitize_ignored_directories',
|
||||
'cached_search_total', 'search_filter_cache_key', 'count_file_groups',
|
||||
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems']
|
||||
'STEM_ALGORITHM_VERSION', 'ensure_file_group_stems',
|
||||
'backfill_null_file_group_stems', 'dedupe_file_groups_by_directory_stem']
|
||||
|
||||
|
||||
def get_file_tag_names(session: Session, file: pathlib.Path) -> List[str]:
|
||||
|
|
@ -724,17 +725,17 @@ def _upsert_files(files: List[pathlib.Path],
|
|||
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
|
||||
non_primary_files.update(i for i in group if i != primary_path)
|
||||
except NoPrimaryFile:
|
||||
# Cannot find primary path, create a `file_group` for each file.
|
||||
for primary_path in group:
|
||||
primary_path: pathlib.Path
|
||||
# The primary mimetype allows modelers to find its file_groups.
|
||||
mimetype = get_mimetype(primary_path)
|
||||
modification_datetime = from_timestamp(primary_path.stat().st_mtime)
|
||||
size = primary_path.stat().st_size
|
||||
files_json = json.dumps(_paths_to_files_dict([primary_path]))
|
||||
stem, suffix = split_path_stem_and_suffix(primary_path)
|
||||
suffix = (suffix or '').lower() or None
|
||||
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
|
||||
# Same stem still means one FileGroup. Pick a stable primary so
|
||||
# (directory, stem) stays unique even when nothing is modelable.
|
||||
primary_path = sorted(group)[0]
|
||||
mimetype = get_mimetype(primary_path)
|
||||
modification_datetime = from_timestamp(max(i.stat().st_mtime for i in group))
|
||||
size = sum(i.stat().st_size for i in group)
|
||||
files_json = json.dumps(_paths_to_files_dict(group))
|
||||
stem, suffix = split_path_stem_and_suffix(primary_path)
|
||||
suffix = (suffix or '').lower() or None
|
||||
values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem)
|
||||
non_primary_files.update(i for i in group if i != primary_path)
|
||||
except FileNotFoundError as e:
|
||||
# A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.).
|
||||
refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e)
|
||||
|
|
@ -752,6 +753,12 @@ def _upsert_files(files: List[pathlib.Path],
|
|||
'(SELECT value FROM json_each(?))', (json.dumps(primary_paths),))
|
||||
previously_indexed = {i['primary_path'] for i in curs.fetchall()}
|
||||
|
||||
if non_primary_files:
|
||||
# Delete before INSERT so a new primary_path that shares (directory, stem)
|
||||
# with an old sidecar FileGroup does not hit the unique index.
|
||||
curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))',
|
||||
(json.dumps(list(map(str, non_primary_files))),))
|
||||
|
||||
placeholders, params = values_clause(values)
|
||||
stmt = f'''
|
||||
INSERT INTO file_group
|
||||
|
|
@ -797,11 +804,6 @@ def _upsert_files(files: List[pathlib.Path],
|
|||
else:
|
||||
refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}')
|
||||
|
||||
if non_primary_files:
|
||||
# New files may have been added which change what primary paths exist. Delete any file_groups which
|
||||
# use paths which are not primary.
|
||||
curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))',
|
||||
(json.dumps(list(map(str, non_primary_files))),))
|
||||
non_primary_files.clear()
|
||||
|
||||
# Report progress after each chunk
|
||||
|
|
@ -876,6 +878,84 @@ def ensure_file_group_stems():
|
|||
set_stem_algorithm_version(STEM_ALGORITHM_VERSION)
|
||||
|
||||
|
||||
def backfill_null_file_group_stems():
|
||||
"""Fill file_group.stem for rows that never got one. Used before creating the unique index."""
|
||||
with get_db_curs() as curs:
|
||||
curs.execute("SELECT id, primary_path FROM file_group WHERE stem IS NULL OR stem = ''")
|
||||
rows = list(curs.fetchall())
|
||||
if not rows:
|
||||
return
|
||||
updates = [(split_path_stem_and_suffix(row['primary_path'])[0], row['id']) for row in rows]
|
||||
with get_db_curs(commit=True) as curs:
|
||||
for chunk in chunks(updates, 500):
|
||||
curs.executemany('UPDATE file_group SET stem = ? WHERE id = ?', chunk)
|
||||
|
||||
|
||||
def dedupe_file_groups_by_directory_stem() -> int:
|
||||
"""Collapse FileGroups that share (directory, stem).
|
||||
|
||||
Prefers a row with a domain model, then tags, then non-empty files, then highest id.
|
||||
Tags and collection membership on losers are moved onto the winner. Returns the
|
||||
number of FileGroups deleted.
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
SELECT directory, stem
|
||||
FROM file_group
|
||||
WHERE directory IS NOT NULL
|
||||
AND stem IS NOT NULL
|
||||
AND stem != ''
|
||||
GROUP BY directory, stem
|
||||
HAVING COUNT(*) > 1
|
||||
''')
|
||||
keys = [(row['directory'], row['stem']) for row in curs.fetchall()]
|
||||
|
||||
deleted = 0
|
||||
for directory, stem in keys:
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('''
|
||||
SELECT fg.id,
|
||||
CASE
|
||||
WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0
|
||||
ELSE json_array_length(fg.files)
|
||||
END AS file_count,
|
||||
CASE
|
||||
WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1
|
||||
ELSE 0
|
||||
END AS has_model,
|
||||
(SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count
|
||||
FROM file_group fg
|
||||
WHERE fg.directory = ?
|
||||
AND fg.stem = ?
|
||||
''', (directory, stem))
|
||||
members = list(curs.fetchall())
|
||||
members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True)
|
||||
winner_id = members[0]['id']
|
||||
loser_ids = [m['id'] for m in members[1:]]
|
||||
with get_db_curs(commit=True) as curs:
|
||||
for loser_id in loser_ids:
|
||||
curs.execute('''
|
||||
INSERT OR IGNORE INTO tag_file (tag_id, file_group_id, created_at)
|
||||
SELECT tag_id, ?, created_at
|
||||
FROM tag_file
|
||||
WHERE file_group_id = ?
|
||||
''', (winner_id, loser_id))
|
||||
curs.execute('''
|
||||
UPDATE collection_item
|
||||
SET file_group_id = ?
|
||||
WHERE file_group_id = ?
|
||||
AND collection_id NOT IN (SELECT collection_id
|
||||
FROM (SELECT collection_id
|
||||
FROM collection_item
|
||||
WHERE file_group_id = ?))
|
||||
''', (winner_id, loser_id, winner_id))
|
||||
curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,))
|
||||
deleted += 1
|
||||
if deleted:
|
||||
refresh_logger.info(f'Collapsed {deleted} duplicate FileGroups sharing (directory, stem)')
|
||||
return deleted
|
||||
|
||||
|
||||
async def apply_indexers(progress_callback: Callable[[int, int], None] = None):
|
||||
"""Index leftover FileGroups that no modeler claimed.
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,7 @@ import urllib.parse
|
|||
from datetime import datetime
|
||||
from typing import List, Type, Optional, Iterable
|
||||
|
||||
from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON
|
||||
from sqlalchemy import Column, String, BigInteger, Boolean, event, Index, Integer, JSON, or_
|
||||
from sqlalchemy import types
|
||||
from sqlalchemy.orm import deferred, relationship, Session
|
||||
|
||||
|
|
@ -83,8 +83,9 @@ class FileGroup(ModelHelper, Base):
|
|||
Index('file_group_size_ix', 'size'),
|
||||
Index('file_group_url_idx', 'url'),
|
||||
Index('file_group_viewed_idx', 'viewed'),
|
||||
# Speeds compare joins. Not UNIQUE: duplicates are collapsed at refresh time.
|
||||
Index('file_group_directory_stem_idx', 'directory', 'stem'),
|
||||
# Identity of a FileGroup. Duplicates are collapsed by
|
||||
# dedupe_file_groups_by_directory_stem before this unique index is created.
|
||||
Index('file_group_directory_stem_idx', 'directory', 'stem', unique=True),
|
||||
)
|
||||
# SQLite requires exactly "INTEGER PRIMARY KEY" for the rowid alias (FTS5 content_rowid).
|
||||
id: int = Column(BigInteger().with_variant(Integer, 'sqlite'), primary_key=True)
|
||||
|
|
@ -523,7 +524,16 @@ class FileGroup(ModelHelper, Base):
|
|||
# Sanitize any paths with invalid UTF-8 characters (renames files on disk if needed)
|
||||
paths = tuple(sanitize_filename_surrogates(p) for p in paths)
|
||||
|
||||
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
|
||||
primary_path = get_primary_file(paths) if paths else None
|
||||
stem = split_path_stem_and_suffix(primary_path)[0] if primary_path else None
|
||||
directory = str(primary_path.parent) if primary_path else None
|
||||
filters = [FileGroup.primary_path.in_(list(map(str, paths)))]
|
||||
if stem:
|
||||
filters.append((FileGroup.directory == directory) & (FileGroup.stem == stem))
|
||||
existing_groups = unique_by_predicate(
|
||||
session.query(FileGroup).filter(or_(*filters)).all(),
|
||||
lambda fg: fg.id,
|
||||
)
|
||||
logger.trace(f'FileGroup.from_paths: {len(existing_groups)=}')
|
||||
if len(existing_groups) == 0:
|
||||
# These paths have not been used previously, create a new FileGroup.
|
||||
|
|
|
|||
|
|
@ -15,6 +15,8 @@ import wrolpi.common
|
|||
from modules.videos import Video
|
||||
from wrolpi.common import timer, get_wrolpi_config
|
||||
from wrolpi.conftest import await_switches
|
||||
from sqlalchemy.exc import IntegrityError
|
||||
|
||||
from wrolpi.errors import InvalidFile, UnknownDirectory, FileGroupIsTagged, NoPrimaryFile
|
||||
from wrolpi.files import lib, indexers
|
||||
from wrolpi.files.models import FileGroup, Directory
|
||||
|
|
@ -496,6 +498,91 @@ def test_ensure_file_group_stems_recomputes_on_version_change(test_session, test
|
|||
assert lib.get_stem_algorithm_version() == lib.STEM_ALGORITHM_VERSION
|
||||
|
||||
|
||||
def test_file_group_directory_stem_is_unique(test_session, test_directory, make_files_structure):
|
||||
"""Two FileGroups in the same directory cannot share a stem."""
|
||||
foo_mp4, foo_png = make_files_structure(['foo.mp4', 'foo.png'])
|
||||
FileGroup.from_paths(test_session, foo_mp4)
|
||||
test_session.commit()
|
||||
|
||||
other = FileGroup(
|
||||
directory=foo_png.parent,
|
||||
primary_path=foo_png,
|
||||
stem='foo',
|
||||
files=[{'path': foo_png.name, 'mimetype': 'image/png'}],
|
||||
)
|
||||
test_session.add(other)
|
||||
with pytest.raises(IntegrityError):
|
||||
test_session.commit()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedupe_file_groups_merges_tags_onto_winner(async_client, test_session, test_directory,
|
||||
make_files_structure, tag_factory):
|
||||
"""Duplicates sharing (directory, stem) collapse onto one row; tags on the loser move."""
|
||||
from sqlalchemy import text
|
||||
|
||||
foo, sidecar = make_files_structure(['foo.mp4', 'foo.info.json'])
|
||||
test_session.execute(text('DROP INDEX IF EXISTS file_group_directory_stem_idx'))
|
||||
|
||||
keeper = FileGroup.from_paths(test_session, foo)
|
||||
keeper.model = 'video'
|
||||
loser = FileGroup(
|
||||
directory=foo.parent,
|
||||
primary_path=sidecar,
|
||||
stem='foo',
|
||||
files=[{'path': sidecar.name, 'mimetype': 'application/json'}],
|
||||
)
|
||||
test_session.add(loser)
|
||||
test_session.commit()
|
||||
|
||||
tag = await tag_factory('keepme')
|
||||
loser.add_tag(test_session, tag.id)
|
||||
test_session.commit()
|
||||
keeper_id, loser_id = keeper.id, loser.id
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
|
||||
deleted = lib.dedupe_file_groups_by_directory_stem()
|
||||
test_session.expire_all()
|
||||
|
||||
assert deleted == 1
|
||||
remaining = test_session.query(FileGroup).one()
|
||||
assert remaining.id == keeper_id
|
||||
assert remaining.tag_names == ['keepme']
|
||||
assert test_session.query(FileGroup).filter_by(id=loser_id).one_or_none() is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_dedupe_file_groups_prefers_tagged_row(async_client, test_session, test_directory,
|
||||
make_files_structure, tag_factory):
|
||||
"""The tagged FileGroup wins even when it has the lower id and no model."""
|
||||
from sqlalchemy import text
|
||||
|
||||
a, b = make_files_structure(['bar.mp4', 'bar.png'])
|
||||
test_session.execute(text('DROP INDEX IF EXISTS file_group_directory_stem_idx'))
|
||||
first = FileGroup.from_paths(test_session, a)
|
||||
second = FileGroup(
|
||||
directory=b.parent,
|
||||
primary_path=b,
|
||||
stem='bar',
|
||||
files=[{'path': b.name, 'mimetype': 'image/png'}],
|
||||
model='video',
|
||||
)
|
||||
test_session.add(second)
|
||||
test_session.commit()
|
||||
assert first.id < second.id
|
||||
tag = await tag_factory('tagged-first')
|
||||
first.add_tag(test_session, tag.id)
|
||||
test_session.commit()
|
||||
|
||||
lib.dedupe_file_groups_by_directory_stem()
|
||||
test_session.expire_all()
|
||||
|
||||
remaining = test_session.query(FileGroup).one()
|
||||
# has_model ranks above tags, so the modeled later row wins; tags move onto it.
|
||||
assert remaining.id == second.id
|
||||
assert remaining.tag_names == ['tagged-first']
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_file_group_tag(test_session, make_files_structure, test_directory, tag_factory, await_switches,
|
||||
refresh_files):
|
||||
|
|
@ -822,7 +909,7 @@ def test_get_primary_file(test_directory, video_file, srt_file3, example_epub, e
|
|||
@pytest.mark.asyncio
|
||||
async def test_refresh_files_no_groups(async_client, test_session, test_directory, make_files_structure,
|
||||
zip_file_factory, refresh_files):
|
||||
"""Files that share a name, but cannot be grouped into a FileGroup have their own FileGroups."""
|
||||
"""Files that share a stem are one FileGroup even when neither is a modeled primary."""
|
||||
foo_txt, foo_zip = make_files_structure({
|
||||
'foo.txt': 'text',
|
||||
'foo.zip': zip_file_factory(),
|
||||
|
|
@ -831,11 +918,12 @@ async def test_refresh_files_no_groups(async_client, test_session, test_director
|
|||
|
||||
await refresh_files()
|
||||
|
||||
# Two distinct FileGroups.
|
||||
assert test_session.query(FileGroup).count() == 2
|
||||
txt, zip_ = test_session.query(FileGroup)
|
||||
assert txt.primary_path == foo_txt and txt.size == foo_txt.stat().st_size
|
||||
assert zip_.primary_path == foo_zip and zip_.size == foo_zip.stat().st_size
|
||||
# Same stem is one FileGroup even when neither file is a modeled primary.
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
fg = test_session.query(FileGroup).one()
|
||||
names = {pathlib.Path(f['path']).name for f in fg.files}
|
||||
assert names == {'foo.txt', 'foo.zip'}
|
||||
assert fg.size == foo_txt.stat().st_size + foo_zip.stat().st_size
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -890,53 +978,34 @@ async def test_upsert_directories_chunks_large_batches(test_session, test_direct
|
|||
@pytest.mark.asyncio
|
||||
async def test_file_group_merge(async_client, test_session, test_directory, make_files_structure, tag_factory,
|
||||
video_bytes, srt_file3):
|
||||
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
|
||||
migrated."""
|
||||
"""from_paths on a sidecar finds the existing (directory, stem) FileGroup instead of inserting a duplicate."""
|
||||
vid, srt = make_files_structure({
|
||||
'vid.mp4': video_bytes,
|
||||
'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(),
|
||||
})
|
||||
one, two = await tag_factory(), await tag_factory()
|
||||
vid_group = FileGroup.from_paths(test_session, vid)
|
||||
srt_group = FileGroup.from_paths(test_session, srt)
|
||||
test_session.add_all([vid_group, srt_group])
|
||||
test_session.flush([vid_group, srt_group])
|
||||
test_session.flush([vid_group])
|
||||
vid_tag_file = vid_group.add_tag(test_session, one.name)
|
||||
srt_tag_file = srt_group.add_tag(test_session, two.name)
|
||||
test_session.flush([vid_tag_file, srt_tag_file])
|
||||
tag_file_created_at = vid_tag_file.created_at
|
||||
test_session.commit()
|
||||
|
||||
srt_group = FileGroup.from_paths(test_session, srt)
|
||||
assert srt_group.id == vid_group.id
|
||||
srt_tag_file = srt_group.add_tag(test_session, two.name)
|
||||
srt_file_created_at = srt_tag_file.created_at
|
||||
test_session.commit()
|
||||
|
||||
assert vid_group.mimetype == 'video/mp4'
|
||||
assert srt_group.mimetype == 'text/srt'
|
||||
|
||||
# Add srt to vid_group so both groups have it (simulates race condition scenario)
|
||||
vid_group.append_files(srt)
|
||||
merged = FileGroup.from_paths(test_session, vid, srt)
|
||||
test_session.commit()
|
||||
|
||||
# Verify both groups have the srt file
|
||||
assert 'vid.srt' in [f['path'] for f in vid_group.files]
|
||||
assert 'vid.srt' in [f['path'] for f in srt_group.files]
|
||||
|
||||
# Both FileGroups are merged.
|
||||
vid = FileGroup.from_paths(test_session, vid, srt)
|
||||
test_session.commit()
|
||||
# files now stores relative filenames as strings, use my_files() to get resolved Paths
|
||||
assert {i['path'].name for i in vid.my_files()} == {'vid.mp4', 'vid.srt'}
|
||||
|
||||
assert merged.id == vid_group.id
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
assert set(vid.tag_names) == {'one', 'two'}
|
||||
assert {i['path'].name for i in vid.my_files()} == {'vid.mp4', 'vid.srt'}
|
||||
# TagFile.created_at is preserved.
|
||||
assert [i for i in vid.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at
|
||||
assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at
|
||||
# Size is combined
|
||||
assert vid.size > len(video_bytes)
|
||||
assert vid.mimetype == 'video/mp4'
|
||||
|
||||
# Verify no duplicate files were created during merge
|
||||
filenames = [f['path'] for f in vid.files]
|
||||
assert set(merged.tag_names) == {'one', 'two'}
|
||||
assert {i['path'].name for i in merged.my_files()} == {'vid.mp4', 'vid.srt'}
|
||||
assert [i for i in merged.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at
|
||||
assert [i for i in merged.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at
|
||||
assert merged.mimetype == 'video/mp4'
|
||||
filenames = [f['path'] for f in merged.files]
|
||||
assert len(filenames) == len(set(filenames)), f"Duplicate files found: {filenames}"
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
"""Tests for the file comparison worker."""
|
||||
import asyncio
|
||||
import os
|
||||
import pathlib
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -874,9 +875,10 @@ async def test_file_worker_deletes_removed_file_groups(
|
|||
file_worker.private_queue.put_nowait(task)
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify FileGroup was deleted
|
||||
# Verify FileGroup was deleted (do not key off id: SQLite may reuse it for the baseline upsert).
|
||||
test_session.expire_all()
|
||||
assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None
|
||||
remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)}
|
||||
assert 'file1.txt' not in remaining
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -912,10 +914,13 @@ async def test_file_worker_deletes_tagged_file_groups_auto_removes_tags(
|
|||
file_worker.private_queue.put_nowait(task)
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify FileGroup and TagFile were deleted (cascade)
|
||||
# Verify FileGroup and TagFile were deleted (cascade). Do not key off recycled SQLite ids.
|
||||
test_session.expire_all()
|
||||
assert test_session.query(FileGroup).filter_by(id=fg_id).one_or_none() is None
|
||||
assert test_session.query(TagFile).filter_by(file_group_id=fg_id).count() == 0
|
||||
remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)}
|
||||
assert 'file1.txt' not in remaining
|
||||
assert test_session.query(TagFile).join(FileGroup).filter(
|
||||
FileGroup.primary_path.like('%/file1.txt')
|
||||
).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1120,11 +1125,10 @@ async def test_file_worker_deletes_multiple_file_groups(
|
|||
file_worker.private_queue.put_nowait(task)
|
||||
await file_worker.process_queue()
|
||||
|
||||
# Verify all FileGroups were deleted
|
||||
# Verify all FileGroups were deleted (do not key off recycled SQLite ids).
|
||||
test_session.expire_all()
|
||||
assert test_session.query(FileGroup).filter(
|
||||
FileGroup.id.in_([fg1_id, fg2_id, fg3_id])
|
||||
).count() == 0
|
||||
remaining = {pathlib.Path(str(fg.primary_path)).name for fg in test_session.query(FileGroup)}
|
||||
assert remaining.isdisjoint({'file1.txt', 'file2.txt', 'file3.txt'})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
@ -1681,78 +1685,15 @@ async def test_upsert_file_groups_tracks_actual_db_progress(async_client, test_s
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compare_file_groups_duplicate_deleted_filegroups(test_session, test_directory, make_files_structure):
|
||||
"""When multiple FileGroups share the same (directory, stem) and all files are deleted,
|
||||
all duplicates should appear in the deleted diffs — not just the last one processed."""
|
||||
make_files_structure([
|
||||
'docs/file1.txt',
|
||||
'docs/file1.json',
|
||||
])
|
||||
|
||||
docs_dir = str(test_directory / 'docs')
|
||||
|
||||
# Create 3 separate FileGroups for the same (directory, stem) — simulating a botched move.
|
||||
# Use different primary_paths (unique constraint) but same directory and stem.
|
||||
fg1 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.txt'),
|
||||
files=[{'path': 'file1.txt', 'mimetype': 'text/plain', 'size': 0}])
|
||||
fg2 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.json'),
|
||||
files=[{'path': 'file1.json', 'mimetype': 'application/json', 'size': 0}])
|
||||
# Third ghost with empty files list — use a unique primary_path
|
||||
(test_directory / 'docs/file1.bak').touch()
|
||||
fg3 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.bak'),
|
||||
files=[])
|
||||
test_session.add_all([fg1, fg2, fg3])
|
||||
test_session.commit()
|
||||
|
||||
fg1_id, fg2_id, fg3_id = fg1.id, fg2.id, fg3.id
|
||||
|
||||
# Delete all files from disk
|
||||
for f in (test_directory / 'docs').iterdir():
|
||||
f.unlink()
|
||||
async def test_compare_file_groups_unique_stem_is_one_group(test_session, test_directory, make_files_structure):
|
||||
"""Files that share a stem are one FileGroup; compare reports a single group."""
|
||||
make_files_structure(['docs/file1.txt', 'docs/file1.json'])
|
||||
from wrolpi.files.lib import _upsert_files
|
||||
_upsert_files(list((test_directory / 'docs').iterdir()))
|
||||
|
||||
result = await compare_file_groups(test_directory)
|
||||
|
||||
# ALL 3 FileGroup IDs must appear in deleted diffs
|
||||
deleted_ids = {d.file_group_id for d in result.deleted}
|
||||
assert fg1_id in deleted_ids, f'fg1 ({fg1_id}) missing from deleted: {deleted_ids}'
|
||||
assert fg2_id in deleted_ids, f'fg2 ({fg2_id}) missing from deleted: {deleted_ids}'
|
||||
assert fg3_id in deleted_ids, f'fg3 ({fg3_id}) missing from deleted: {deleted_ids}'
|
||||
assert len(result.new) == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_compare_file_groups_duplicate_keeps_one(test_session, test_directory, make_files_structure):
|
||||
"""When duplicate FileGroups exist and files are on disk, one should be kept (modified/unchanged)
|
||||
and the others should be marked for deletion."""
|
||||
make_files_structure([
|
||||
'docs/file1.txt',
|
||||
])
|
||||
|
||||
docs_dir = str(test_directory / 'docs')
|
||||
|
||||
# Create 2 FileGroups with same (directory, stem)
|
||||
fg1 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.txt'),
|
||||
files=[{'path': 'file1.txt', 'mimetype': 'text/plain', 'size': 0}])
|
||||
(test_directory / 'docs/file1.json').touch()
|
||||
fg2 = FileGroup(directory=docs_dir, primary_path=str(test_directory / 'docs/file1.json'),
|
||||
files=[{'path': 'file1.json', 'mimetype': 'application/json', 'size': 0}])
|
||||
test_session.add_all([fg1, fg2])
|
||||
test_session.commit()
|
||||
|
||||
fg1_id, fg2_id = fg1.id, fg2.id
|
||||
# Remove the .json so only .txt remains
|
||||
(test_directory / 'docs/file1.json').unlink()
|
||||
|
||||
result = await compare_file_groups(test_directory)
|
||||
|
||||
# One FG should have the file on disk, the other should be deleted as duplicate
|
||||
deleted_ids = {d.file_group_id for d in result.deleted}
|
||||
all_diff_ids = {d.file_group_id for d in result.deleted + result.modified + result.unchanged}
|
||||
assert fg1_id in all_diff_ids
|
||||
assert fg2_id in all_diff_ids
|
||||
# At least one must be in deleted (the duplicate)
|
||||
assert len(deleted_ids) >= 1
|
||||
assert deleted_ids & {fg1_id, fg2_id}
|
||||
assert len(result.new) + len(result.unchanged) + len(result.modified) == 1
|
||||
assert test_session.query(FileGroup).count() == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
|
|
|
|||
|
|
@ -1492,10 +1492,11 @@ class FileWorker:
|
|||
logger.info(f'Refreshing {len(file_paths)} files directly')
|
||||
file_result = await self._refresh_files_directly(file_paths, expand_stems=task.expand_stems)
|
||||
|
||||
# Process file results
|
||||
# Delete first so a new primary_path cannot collide on (directory, stem)
|
||||
# with a duplicate FileGroup that compare already marked deleted.
|
||||
self._cleanup_modified_models(file_result.modified)
|
||||
await self._upsert_file_groups(file_result.new + file_result.modified)
|
||||
await self._delete_file_groups(file_result.deleted)
|
||||
await self._upsert_file_groups(file_result.new + file_result.modified)
|
||||
|
||||
# Process directories with full scan (existing path)
|
||||
if dir_paths:
|
||||
|
|
@ -1520,10 +1521,11 @@ class FileWorker:
|
|||
f'{len(dir_result.deleted)} deleted, {len(dir_result.unchanged)} unchanged'
|
||||
)
|
||||
|
||||
# Process directory results
|
||||
# Delete first so a new primary_path cannot collide on (directory, stem)
|
||||
# with a duplicate FileGroup that compare already marked deleted.
|
||||
self._cleanup_modified_models(dir_result.modified)
|
||||
await self._upsert_file_groups(dir_result.new + dir_result.modified)
|
||||
await self._delete_file_groups(dir_result.deleted)
|
||||
await self._upsert_file_groups(dir_result.new + dir_result.modified)
|
||||
|
||||
# Run indexers, modelers, and cleanup once for all changes
|
||||
await self._apply_post_processing(is_global_refresh=is_global_refresh)
|
||||
|
|
|
|||
|
|
@ -66,6 +66,57 @@ def test_compare_db_version_uninitialized_file(test_directory):
|
|||
assert db_bootstrap.compare_db_version() == 'current'
|
||||
|
||||
|
||||
def test_unique_stem_migration_collapses_duplicates(test_directory):
|
||||
"""The unique-index migration merges duplicate (directory, stem) rows and keeps tags."""
|
||||
import sqlite3
|
||||
from alembic.config import Config
|
||||
from alembic import command
|
||||
from wrolpi.db import get_db_uri, get_db_file
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
(test_directory / 'config').mkdir(parents=True, exist_ok=True)
|
||||
config = Config(str(PROJECT_DIR / 'alembic.ini'))
|
||||
config.set_main_option('sqlalchemy.url', get_db_uri())
|
||||
command.upgrade(config, '2026_09_16_1200')
|
||||
|
||||
db_file = get_db_file()
|
||||
directory = str(test_directory)
|
||||
with sqlite3.connect(db_file) as conn:
|
||||
conn.execute('PRAGMA foreign_keys=ON')
|
||||
conn.execute(
|
||||
'INSERT INTO file_group (directory, primary_path, files, stem, indexed) VALUES (?,?,?,?,0)',
|
||||
(directory, str(test_directory / 'dup.mp4'), '[]', 'dup'),
|
||||
)
|
||||
conn.execute(
|
||||
'INSERT INTO file_group (directory, primary_path, files, stem, indexed) VALUES (?,?,?,?,0)',
|
||||
(directory, str(test_directory / 'dup.info.json'), '[]', 'dup'),
|
||||
)
|
||||
first_id, second_id = [r[0] for r in conn.execute('SELECT id FROM file_group ORDER BY id').fetchall()]
|
||||
conn.execute("INSERT INTO tag (name, color) VALUES ('migrated', '#111111')")
|
||||
tag_id = conn.execute('SELECT id FROM tag').fetchone()[0]
|
||||
# Tag the lower-id row so it wins over the later untagged duplicate.
|
||||
conn.execute(
|
||||
'INSERT INTO tag_file (tag_id, file_group_id) VALUES (?, ?)',
|
||||
(tag_id, first_id),
|
||||
)
|
||||
conn.commit()
|
||||
assert conn.execute('SELECT COUNT(*) FROM file_group').fetchone()[0] == 2
|
||||
assert first_id < second_id
|
||||
|
||||
command.upgrade(config, 'head')
|
||||
|
||||
with sqlite3.connect(db_file) as conn:
|
||||
rows = list(conn.execute('SELECT id, stem FROM file_group'))
|
||||
assert len(rows) == 1
|
||||
assert rows[0][0] == first_id
|
||||
tagged = conn.execute('SELECT file_group_id FROM tag_file').fetchone()[0]
|
||||
assert tagged == first_id
|
||||
sql = conn.execute(
|
||||
"SELECT sql FROM sqlite_master WHERE type='index' AND name='file_group_directory_stem_idx'"
|
||||
).fetchone()[0]
|
||||
assert 'UNIQUE' in sql.upper()
|
||||
|
||||
|
||||
def test_bootstrap_lock(test_directory):
|
||||
"""Only one process can hold the bootstrap lock."""
|
||||
with db_bootstrap.bootstrap_lock() as acquired:
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue