mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
A refresh re-indexes a FileGroup whose files changed even when their count did not (#933)
_upsert_files kept a FileGroup's files and indexed state when the size, newest modification time and number of files were unchanged. A file that replaced another (foo.png for foo.jpg, same size, older than foo.mp4) left the old file in `files` and the FileGroup indexed. The upsert now compares the sorted file names. FileGroups that still store absolute paths (the old format) compare by name too, so they are not re-indexed when nothing changed. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
21006eae09
commit
6fc147ccdf
2 changed files with 53 additions and 2 deletions
|
|
@ -831,6 +831,18 @@ def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path]
|
|||
return sorted(paths)[0]
|
||||
|
||||
|
||||
def _file_names_sql(table: str) -> str:
|
||||
"""SQL: the sorted names in `table`.files, as a JSON array.
|
||||
|
||||
Old FileGroups store each file's absolute path, new ones its name; both give the name."""
|
||||
path = "json_extract(j.value, '$.path')"
|
||||
return f'''(SELECT json_group_array(name) FROM (
|
||||
SELECT CASE WHEN {path} LIKE '/%' THEN substr({path}, length({table}.directory) + 2) ELSE {path} END
|
||||
AS name
|
||||
FROM json_each({table}.files) AS j
|
||||
ORDER BY name))'''
|
||||
|
||||
|
||||
# About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks.
|
||||
UPSERT_CHUNK_SIZE = 500
|
||||
|
||||
|
|
@ -895,6 +907,8 @@ def _upsert_files(files: List[pathlib.Path],
|
|||
(json.dumps(list(map(str, non_primary_files))),))
|
||||
|
||||
placeholders, params = values_clause(values)
|
||||
# Compare names, not counts: a file can replace another without changing the size or newest time.
|
||||
same_files = f'{_file_names_sql("file_group")} = {_file_names_sql("EXCLUDED")}'
|
||||
stmt = f'''
|
||||
INSERT INTO file_group
|
||||
(primary_path, directory, indexed, modification_datetime, mimetype, size, files,
|
||||
|
|
@ -915,7 +929,7 @@ def _upsert_files(files: List[pathlib.Path],
|
|||
WHEN
|
||||
file_group.modification_datetime = EXCLUDED.modification_datetime
|
||||
AND file_group.size = EXCLUDED.size
|
||||
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
|
||||
AND {same_files}
|
||||
THEN file_group.files
|
||||
ELSE EXCLUDED.files
|
||||
END
|
||||
|
|
@ -925,7 +939,7 @@ def _upsert_files(files: List[pathlib.Path],
|
|||
file_group.indexed = true
|
||||
AND file_group.modification_datetime = EXCLUDED.modification_datetime
|
||||
AND file_group.size = EXCLUDED.size
|
||||
AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files)
|
||||
AND {same_files}
|
||||
)
|
||||
RETURNING id, primary_path, indexed
|
||||
'''
|
||||
|
|
|
|||
|
|
@ -495,6 +495,43 @@ async def test__upsert_files_interleaved_stems(test_session, make_files_structur
|
|||
])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test__upsert_files_same_count_different_members(test_session, make_files_structure, video_bytes):
|
||||
"""A FileGroup whose files changed is re-indexed even when the count, total size and newest modification time
|
||||
did not change."""
|
||||
jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes})
|
||||
lib._upsert_files([mp4, jpg])
|
||||
test_session.query(FileGroup).one().indexed = True
|
||||
test_session.commit()
|
||||
|
||||
# foo.png replaces foo.jpg: same size, and foo.mp4 is still the newest file.
|
||||
png = jpg.rename(jpg.with_suffix('.png'))
|
||||
os.utime(png, (mp4.stat().st_mtime - 10,) * 2)
|
||||
lib._upsert_files([mp4, png])
|
||||
|
||||
test_session.expire_all()
|
||||
foo = test_session.query(FileGroup).one()
|
||||
assert {i['path'] for i in foo.files} == {'foo.mp4', 'foo.png'}
|
||||
assert foo.indexed is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test__upsert_files_unchanged_legacy_absolute_paths(test_session, make_files_structure, video_bytes):
|
||||
"""A FileGroup whose files are stored with absolute paths (the old format) is not re-indexed when its files did
|
||||
not change."""
|
||||
jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes})
|
||||
lib._upsert_files([mp4, jpg])
|
||||
foo = test_session.query(FileGroup).one()
|
||||
foo.files = [{**i, 'path': str(foo.directory / i['path'])} for i in foo.files]
|
||||
foo.indexed = True
|
||||
test_session.commit()
|
||||
|
||||
lib._upsert_files([mp4, jpg])
|
||||
|
||||
test_session.expire_all()
|
||||
assert test_session.query(FileGroup).one().indexed is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_refresh_interleaved_stems(test_session, make_files_structure, test_directory, video_bytes,
|
||||
refresh_files):
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue