diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 66c8ccd2..312f69d8 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -831,6 +831,18 @@ def choose_primary_file(files: Union[Tuple[pathlib.Path], Iterable[pathlib.Path] return sorted(paths)[0] +def _file_names_sql(table: str) -> str: + """SQL: the sorted names in `table`.files, as a JSON array. + + Old FileGroups store each file's absolute path, new ones its name; both give the name.""" + path = "json_extract(j.value, '$.path')" + return f'''(SELECT json_group_array(name) FROM ( + SELECT CASE WHEN {path} LIKE '/%' THEN substr({path}, length({table}.directory) + 2) ELSE {path} END + AS name + FROM json_each({table}.files) AS j + ORDER BY name))''' + + # About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks. UPSERT_CHUNK_SIZE = 500 @@ -895,6 +907,8 @@ def _upsert_files(files: List[pathlib.Path], (json.dumps(list(map(str, non_primary_files))),)) placeholders, params = values_clause(values) + # Compare names, not counts: a file can replace another without changing the size or newest time. + same_files = f'{_file_names_sql("file_group")} = {_file_names_sql("EXCLUDED")}' stmt = f''' INSERT INTO file_group (primary_path, directory, indexed, modification_datetime, mimetype, size, files, @@ -915,7 +929,7 @@ def _upsert_files(files: List[pathlib.Path], WHEN file_group.modification_datetime = EXCLUDED.modification_datetime AND file_group.size = EXCLUDED.size - AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files) + AND {same_files} THEN file_group.files ELSE EXCLUDED.files END @@ -925,7 +939,7 @@ def _upsert_files(files: List[pathlib.Path], file_group.indexed = true AND file_group.modification_datetime = EXCLUDED.modification_datetime AND file_group.size = EXCLUDED.size - AND json_array_length(file_group.files) = json_array_length(EXCLUDED.files) + AND {same_files} ) RETURNING id, primary_path, indexed ''' diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 1d5080ae..038bbc19 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -495,6 +495,43 @@ async def test__upsert_files_interleaved_stems(test_session, make_files_structur ]) +@pytest.mark.asyncio +async def test__upsert_files_same_count_different_members(test_session, make_files_structure, video_bytes): + """A FileGroup whose files changed is re-indexed even when the count, total size and newest modification time + did not change.""" + jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes}) + lib._upsert_files([mp4, jpg]) + test_session.query(FileGroup).one().indexed = True + test_session.commit() + + # foo.png replaces foo.jpg: same size, and foo.mp4 is still the newest file. + png = jpg.rename(jpg.with_suffix('.png')) + os.utime(png, (mp4.stat().st_mtime - 10,) * 2) + lib._upsert_files([mp4, png]) + + test_session.expire_all() + foo = test_session.query(FileGroup).one() + assert {i['path'] for i in foo.files} == {'foo.mp4', 'foo.png'} + assert foo.indexed is False + + +@pytest.mark.asyncio +async def test__upsert_files_unchanged_legacy_absolute_paths(test_session, make_files_structure, video_bytes): + """A FileGroup whose files are stored with absolute paths (the old format) is not re-indexed when its files did + not change.""" + jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes}) + lib._upsert_files([mp4, jpg]) + foo = test_session.query(FileGroup).one() + foo.files = [{**i, 'path': str(foo.directory / i['path'])} for i in foo.files] + foo.indexed = True + test_session.commit() + + lib._upsert_files([mp4, jpg]) + + test_session.expire_all() + assert test_session.query(FileGroup).one().indexed is True + + @pytest.mark.asyncio async def test_refresh_interleaved_stems(test_session, make_files_structure, test_directory, video_bytes, refresh_files):