From ee6c384dc6e30de915fc5acfdfb6f25a79b875d0 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Sun, 1 Nov 2026 13:11:24 -0700 Subject: [PATCH] A refresh keeps a FileGroup's tags and viewing state when its primary file changes (#932) _upsert_files deleted the FileGroup of every companion file before inserting, so a FileGroup whose primary file became a companion (foo.jpg, then foo.mp4 arrives) was replaced by a new, untagged row. A FileGroup whose primary file was replaced (foo.txt deleted, foo.mp4 added) still owned the (directory, stem), and the INSERT failed on the unique index. Before inserting, the FileGroup owning each incoming (directory, stem) is renamed to the new primary path, keeping its id, tags, collection membership, model and viewing state. When a different FileGroup owns the primary path, the two collapse into the one dedupe prefers (modeled, then tagged). A companion's leftover FileGroup gives its tags, collection membership and viewing state to its group's FileGroup before it is deleted. Co-authored-by: Claude Opus 5.5 --- wrolpi/files/lib.py | 136 +++++++++++++++++++++++++++------- wrolpi/files/test/test_lib.py | 114 ++++++++++++++++++++++++++++ 2 files changed, 223 insertions(+), 27 deletions(-) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 312f69d8..8133a540 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -843,6 +843,98 @@ def _file_names_sql(table: str) -> str: ORDER BY name))''' +def _rank_file_groups(curs, file_group_ids: Iterable[int]) -> List[int]: + """The ids of these FileGroups, the one most worth keeping first: modeled, then tagged, then with files, then + the newest.""" + curs.execute(''' + SELECT fg.id, + CASE + WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0 + ELSE json_array_length(fg.files) + END AS file_count, + CASE + WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1 + ELSE 0 + END AS has_model, + (SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count + FROM file_group fg + WHERE fg.id IN (SELECT value FROM json_each(?)) + ''', (json.dumps(list(file_group_ids)),)) + members = list(curs.fetchall()) + members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True) + return [m['id'] for m in members] + + +def _count_primary_path_change(curs, file_group_id: int): + """tags.yaml, playlists.yaml and recently_viewed.yaml save a FileGroup by its primary path; count a change of + that path for each config that holds this FileGroup.""" + curs.execute(''' + SELECT EXISTS (SELECT 1 FROM tag_file WHERE file_group_id = :id) AS tagged, + EXISTS (SELECT 1 + FROM collection_item ci + JOIN collection c ON c.id = ci.collection_id + WHERE ci.file_group_id = :id + AND c.kind = 'playlist') AS in_playlist, + (SELECT viewed IS NOT NULL FROM file_group WHERE id = :id) AS viewed + ''', {'id': file_group_id}) + row = curs.fetchone() + names = [name for name, held in (('tags.yaml', row['tagged']), ('playlists.yaml', row['in_playlist']), + ('recently_viewed.yaml', row['viewed'])) if held] + if names: + count_config_change_curs(curs, *names) + + +def _collapse_file_group(curs, loser_id: int, winner_id: int): + """Delete the loser FileGroup after giving its tags, collection membership and viewing state to the winner. + + The winner keeps its own viewing state when it has one.""" + merge_file_group_relations(curs, loser_id, winner_id) + curs.execute('SELECT viewed IS NOT NULL AS viewed FROM file_group WHERE id = ?', (loser_id,)) + if curs.fetchone()['viewed']: + # recently_viewed.yaml saves the loser by its primary path. + count_config_change_curs(curs, 'recently_viewed.yaml') + curs.execute(''' + UPDATE file_group + SET viewed = COALESCE(file_group.viewed, loser.viewed), + progress = COALESCE(file_group.progress, loser.progress), + position = COALESCE(file_group.position, loser.position) + FROM (SELECT viewed, progress, position FROM file_group WHERE id = :loser) AS loser + WHERE file_group.id = :winner + ''', {'loser': loser_id, 'winner': winner_id}) + curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,)) + + +def _give_stem_owners_their_primary_paths(curs, values: List[tuple]): + """Give each incoming primary path to the FileGroup that already owns its (directory, stem). + + `_upsert_files` then updates that FileGroup through ON CONFLICT (primary_path) instead of colliding with it on + the (directory, stem) unique index. The FileGroup keeps its id, so its tags, collection membership, model and + viewing state stay with it. If a different FileGroup owns the primary path, the two collapse into the one + `_rank_file_groups` prefers. + + @param values: `_upsert_files` rows: (primary_path, directory, ..., stem).""" + incoming = json.dumps([[value[1], value[-1], value[0]] for value in values]) + curs.execute(''' + SELECT fg.id, json_extract(v.value, '$[2]') AS primary_path + FROM json_each(:incoming) AS v + JOIN file_group fg ON fg.directory = json_extract(v.value, '$[0]') + AND fg.stem = json_extract(v.value, '$[1]') + WHERE fg.primary_path != json_extract(v.value, '$[2]') + -- A FileGroup whose primary path is incoming is updated by its own group. + AND fg.primary_path NOT IN (SELECT json_extract(value, '$[2]') FROM json_each(:incoming)) + ''', {'incoming': incoming}) + for stem_owner_id, primary_path in curs.fetchall(): + curs.execute('SELECT id FROM file_group WHERE primary_path = ?', (primary_path,)) + if path_owner := curs.fetchone(): + winner_id, loser_id = _rank_file_groups(curs, [stem_owner_id, path_owner['id']]) + _collapse_file_group(curs, loser_id, winner_id) + if winner_id == path_owner['id']: + # The survivor already has the primary path. + continue + _count_primary_path_change(curs, stem_owner_id) + curs.execute('UPDATE file_group SET primary_path = ? WHERE id = ?', (primary_path, stem_owner_id)) + + # About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks. UPSERT_CHUNK_SIZE = 500 @@ -862,12 +954,13 @@ def _upsert_files(files: List[pathlib.Path], total_files = len(files) processed_count = 0 - non_primary_files = set() for chunk in chunks_by_stem(files, UPSERT_CHUNK_SIZE): # Group all files by their shared full-stem. `chunks_by_stem` keeps each stem in one chunk. grouped = group_files_by_stem(chunk) # Convert the groups into file_groups. Extract common information into `file_group.data`. values = dict() + # The primary path of each companion file's group. + companion_owners = dict() for group in grouped: # The primary file is the video/SingleFile/epub, etc. try: @@ -882,7 +975,7 @@ def _upsert_files(files: List[pathlib.Path], stem, suffix = split_path_stem_and_suffix(primary_path) suffix = (suffix or '').lower() or None values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem) - non_primary_files.update(i for i in group if i != primary_path) + companion_owners.update((str(i), str(primary_path)) for i in group if i != primary_path) except FileNotFoundError as e: # A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.). refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e) @@ -900,11 +993,9 @@ def _upsert_files(files: List[pathlib.Path], '(SELECT value FROM json_each(?))', (json.dumps(primary_paths),)) previously_indexed = {i['primary_path'] for i in curs.fetchall()} - if non_primary_files: - # Delete before INSERT so a new primary_path that shares (directory, stem) - # with an old sidecar FileGroup does not hit the unique index. - curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))', - (json.dumps(list(map(str, non_primary_files))),)) + # Before INSERT so a new primary_path does not collide on (directory, stem) with the FileGroup that + # already owns that stem. + _give_stem_owners_their_primary_paths(curs, values) placeholders, params = values_clause(values) # Compare names, not counts: a file can replace another without changing the size or newest time. @@ -953,7 +1044,15 @@ def _upsert_files(files: List[pathlib.Path], else: refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}') - non_primary_files.clear() + # A companion file can still be the primary path of an old FileGroup (one with a stale stem). Its + # tags, collection membership and viewing state belong to its group's FileGroup. + if companion_owners: + ids_by_primary_path = {i['primary_path']: i['id'] for i in results} + curs.execute('SELECT id, primary_path FROM file_group WHERE primary_path IN ' + '(SELECT value FROM json_each(?))', (json.dumps(list(companion_owners)),)) + for old in curs.fetchall(): + owner_id = ids_by_primary_path[companion_owners[old['primary_path']]] + _collapse_file_group(curs, old['id'], owner_id) # Report progress after each chunk processed_count += len(chunk) @@ -1081,25 +1180,8 @@ def dedupe_file_groups_by_directory_stem() -> int: deleted = 0 for directory, stem in keys: with get_db_curs() as curs: - curs.execute(''' - SELECT fg.id, - CASE - WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0 - ELSE json_array_length(fg.files) - END AS file_count, - CASE - WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1 - ELSE 0 - END AS has_model, - (SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count - FROM file_group fg - WHERE fg.directory = ? - AND fg.stem = ? - ''', (directory, stem)) - members = list(curs.fetchall()) - members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True) - winner_id = members[0]['id'] - loser_ids = [m['id'] for m in members[1:]] + curs.execute('SELECT id FROM file_group WHERE directory = ? AND stem = ?', (directory, stem)) + winner_id, *loser_ids = _rank_file_groups(curs, [row['id'] for row in curs.fetchall()]) with get_db_curs(commit=True) as curs: for loser_id in loser_ids: merge_file_group_relations(curs, loser_id, winner_id) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 038bbc19..4af40c7d 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -18,6 +18,7 @@ from PIL import Image import wrolpi.common from modules.videos import Video from wrolpi.common import timer, get_wrolpi_config +from wrolpi.config_version import get_config_version, set_config_version from wrolpi.conftest import await_switches, await_file_worker import sqlalchemy from sqlalchemy.exc import IntegrityError @@ -495,6 +496,119 @@ async def test__upsert_files_interleaved_stems(test_session, make_files_structur ]) +def _file_group_by_stem(test_session, stem: str) -> FileGroup: + test_session.expire_all() + return test_session.query(FileGroup).filter_by(stem=stem).one() + + +@pytest.mark.asyncio +async def test__upsert_files_primary_moves_to_new_file(async_client, test_session, make_files_structure, video_bytes, + tag_factory): + """A FileGroup whose primary file becomes a companion keeps its id, tags and viewing state when the new + primary file arrives.""" + jpg, = make_files_structure({'foo.jpg': 'jpg'}) + lib._upsert_files([jpg]) + foo = _file_group_by_stem(test_session, 'foo') + foo_id = foo.id + tag = await tag_factory() + foo.add_tag(test_session, tag.id) + foo.set_viewed() + set_config_version(test_session, 'tags.yaml', 1) + test_session.commit() + + mp4, = make_files_structure({'foo.mp4': video_bytes}) + lib._upsert_files([jpg, mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.id == foo_id + assert foo.primary_path == mp4 + assert foo.tag_names == [tag.name] + assert foo.viewed + # tags.yaml saves tagged files by their primary path. + assert get_config_version(test_session, 'tags.yaml') > 1 + + +@pytest.mark.asyncio +async def test__upsert_files_primary_file_replaced(async_client, test_session, make_files_structure, video_bytes, tag_factory): + """A FileGroup whose primary file was replaced by a file with the same stem is renamed, not duplicated. + + The old primary file is not in the refresh, so it is not a companion of the new group.""" + txt, = make_files_structure({'foo.txt': 'text'}) + lib._upsert_files([txt]) + foo = _file_group_by_stem(test_session, 'foo') + foo_id = foo.id + tag = await tag_factory() + foo.add_tag(test_session, tag.id) + test_session.commit() + + txt.unlink() + mp4, = make_files_structure({'foo.mp4': video_bytes}) + lib._upsert_files([mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.id == foo_id + assert foo.primary_path == mp4 + assert foo.tag_names == [tag.name] + + +@pytest.mark.asyncio +async def test__upsert_files_primary_path_and_stem_owned_by_different_rows(async_client, test_session, make_files_structure, + video_bytes, tag_factory): + """When one FileGroup owns the incoming primary path and another owns its (directory, stem), the modeled one + survives with the tags and viewing state of both.""" + jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes}) + # A row with a stale stem owns foo.mp4. + lib._upsert_files([mp4]) + stale = _file_group_by_stem(test_session, 'foo') + stale.stem = 'stale' + test_session.commit() + # The modeled row owns stem `foo`. + lib._upsert_files([jpg]) + modeled = _file_group_by_stem(test_session, 'foo') + modeled.model = 'video' + modeled_id = modeled.id + one, two = await tag_factory(), await tag_factory() + stale.add_tag(test_session, one.id) + stale.set_viewed() + modeled.add_tag(test_session, two.id) + test_session.commit() + + lib._upsert_files([jpg, mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.id == modeled_id + assert foo.primary_path == mp4 + assert sorted(foo.tag_names) == sorted([one.name, two.name]) + assert foo.viewed + + +@pytest.mark.asyncio +async def test__upsert_files_companion_with_stale_stem(async_client, test_session, make_files_structure, video_bytes, + tag_factory): + """A companion file's old FileGroup gives its tags and viewing state to the group's FileGroup before it is + deleted.""" + jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes}) + lib._upsert_files([jpg]) + old = _file_group_by_stem(test_session, 'foo') + old.stem = 'stale' + tag = await tag_factory() + old.add_tag(test_session, tag.id) + old.set_viewed() + test_session.commit() + + lib._upsert_files([jpg, mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.primary_path == mp4 + assert foo.tag_names == [tag.name] + assert foo.viewed + assert jpg.is_file() + + @pytest.mark.asyncio async def test__upsert_files_same_count_different_members(test_session, make_files_structure, video_bytes): """A FileGroup whose files changed is re-indexed even when the count, total size and newest modification time