A refresh re-indexes a FileGroup whose files changed even when their count did not (#933)

_upsert_files kept a FileGroup's files and indexed state when the size,
newest modification time and number of files were unchanged.  A file that
replaced another (foo.png for foo.jpg, same size, older than foo.mp4) left the
old file in `files` and the FileGroup indexed.

The upsert now compares the sorted file names.  FileGroups that still store
absolute paths (the old format) compare by name too, so they are not
re-indexed when nothing changed.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-01 13:04:32 -07:00 • committed by GitHub
parent 21006eae09
commit 6fc147ccdf
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
2 changed files with 53 additions and 2 deletions

View file

@ -831,6 +831,18 @@ def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]
return sorted(paths)[0]
def _file_names_sql(table: str) -> str:
"""SQL: the sorted names in `table`.files, as a JSON array.
Old FileGroups store each file's absolute path, new ones its name; both give the name."""
path = "json_extract(j.value, '$.path')"
return f'''(SELECT json_group_array(name) FROM (
SELECT CASE WHEN {path} LIKE '/%' THEN substr({path}, length({table}.directory) + 2) ELSE {path} END
AS name
FROM json_each({table}.files) AS j
ORDER BY name))'''
# About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks.
UPSERT_CHUNK_SIZE = 500
@ -895,6 +907,8 @@ def _upsert_files(files: List[pathlib.Path],
(json.dumps(list(map(str, non_primary_files))),))
placeholders, params = values_clause(values)
# Compare names, not counts: a file can replace another without changing the size or newest time.
same_files = f'{_file_names_sql("file_group")} = {_file_names_sql("EXCLUDED")}'
stmt = f'''
INSERT INTO file_group
(primary_path, directory, indexed, modification_datetime, mimetype, size, files,
@ -915,7 +929,7 @@ def _upsert_files(files: List[pathlib.Path],
WHEN
file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
AND {same_files}
THEN file_group.files
ELSE EXCLUDED.files
END
@ -925,7 +939,7 @@ def _upsert_files(files: List[pathlib.Path],
file_group.indexed = true
AND file_group.modification_datetime = EXCLUDED.modification_datetime
AND file_group.size = EXCLUDED.size
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
AND {same_files}
)
RETURNING id, primary_path, indexed
'''

View file

@ -495,6 +495,43 @@ async def test__upsert_files_interleaved_stems(test_session, make_files_structur
])
@pytest.mark.asyncio
async def test__upsert_files_same_count_different_members(test_session, make_files_structure, video_bytes):
"""A FileGroup whose files changed is re-indexed even when the count, total size and newest modification time
did not change."""
jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes})
lib._upsert_files([mp4, jpg])
test_session.query(FileGroup).one().indexed = True
test_session.commit()
# foo.png replaces foo.jpg: same size, and foo.mp4 is still the newest file.
png = jpg.rename(jpg.with_suffix('.png'))
os.utime(png, (mp4.stat().st_mtime - 10,) * 2)
lib._upsert_files([mp4, png])
test_session.expire_all()
foo = test_session.query(FileGroup).one()
assert {i['path'] for i in foo.files} == {'foo.mp4', 'foo.png'}
assert foo.indexed is False
@pytest.mark.asyncio
async def test__upsert_files_unchanged_legacy_absolute_paths(test_session, make_files_structure, video_bytes):
"""A FileGroup whose files are stored with absolute paths (the old format) is not re-indexed when its files did
not change."""
jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes})
lib._upsert_files([mp4, jpg])
foo = test_session.query(FileGroup).one()
foo.files = [{**i, 'path': str(foo.directory / i['path'])} for i in foo.files]
foo.indexed = True
test_session.commit()
lib._upsert_files([mp4, jpg])
test_session.expire_all()
assert test_session.query(FileGroup).one().indexed is True
@pytest.mark.asyncio
async def test_refresh_interleaved_stems(test_session, make_files_structure, test_directory, video_bytes,
refresh_files):