diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index 312f69d8..8133a540 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -843,6 +843,98 @@ def _file_names_sql(table: str) -> str: ORDER BY name))''' +def _rank_file_groups(curs, file_group_ids: Iterable[int]) -> List[int]: + """The ids of these FileGroups, the one most worth keeping first: modeled, then tagged, then with files, then + the newest.""" + curs.execute(''' + SELECT fg.id, + CASE + WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0 + ELSE json_array_length(fg.files) + END AS file_count, + CASE + WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1 + ELSE 0 + END AS has_model, + (SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count + FROM file_group fg + WHERE fg.id IN (SELECT value FROM json_each(?)) + ''', (json.dumps(list(file_group_ids)),)) + members = list(curs.fetchall()) + members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True) + return [m['id'] for m in members] + + +def _count_primary_path_change(curs, file_group_id: int): + """tags.yaml, playlists.yaml and recently_viewed.yaml save a FileGroup by its primary path; count a change of + that path for each config that holds this FileGroup.""" + curs.execute(''' + SELECT EXISTS (SELECT 1 FROM tag_file WHERE file_group_id = :id) AS tagged, + EXISTS (SELECT 1 + FROM collection_item ci + JOIN collection c ON c.id = ci.collection_id + WHERE ci.file_group_id = :id + AND c.kind = 'playlist') AS in_playlist, + (SELECT viewed IS NOT NULL FROM file_group WHERE id = :id) AS viewed + ''', {'id': file_group_id}) + row = curs.fetchone() + names = [name for name, held in (('tags.yaml', row['tagged']), ('playlists.yaml', row['in_playlist']), + ('recently_viewed.yaml', row['viewed'])) if held] + if names: + count_config_change_curs(curs, *names) + + +def _collapse_file_group(curs, loser_id: int, winner_id: int): + """Delete the loser FileGroup after giving its tags, collection membership and viewing state to the winner. + + The winner keeps its own viewing state when it has one.""" + merge_file_group_relations(curs, loser_id, winner_id) + curs.execute('SELECT viewed IS NOT NULL AS viewed FROM file_group WHERE id = ?', (loser_id,)) + if curs.fetchone()['viewed']: + # recently_viewed.yaml saves the loser by its primary path. + count_config_change_curs(curs, 'recently_viewed.yaml') + curs.execute(''' + UPDATE file_group + SET viewed = COALESCE(file_group.viewed, loser.viewed), + progress = COALESCE(file_group.progress, loser.progress), + position = COALESCE(file_group.position, loser.position) + FROM (SELECT viewed, progress, position FROM file_group WHERE id = :loser) AS loser + WHERE file_group.id = :winner + ''', {'loser': loser_id, 'winner': winner_id}) + curs.execute('DELETE FROM file_group WHERE id = ?', (loser_id,)) + + +def _give_stem_owners_their_primary_paths(curs, values: List[tuple]): + """Give each incoming primary path to the FileGroup that already owns its (directory, stem). + + `_upsert_files` then updates that FileGroup through ON CONFLICT (primary_path) instead of colliding with it on + the (directory, stem) unique index. The FileGroup keeps its id, so its tags, collection membership, model and + viewing state stay with it. If a different FileGroup owns the primary path, the two collapse into the one + `_rank_file_groups` prefers. + + @param values: `_upsert_files` rows: (primary_path, directory, ..., stem).""" + incoming = json.dumps([[value[1], value[-1], value[0]] for value in values]) + curs.execute(''' + SELECT fg.id, json_extract(v.value, '$[2]') AS primary_path + FROM json_each(:incoming) AS v + JOIN file_group fg ON fg.directory = json_extract(v.value, '$[0]') + AND fg.stem = json_extract(v.value, '$[1]') + WHERE fg.primary_path != json_extract(v.value, '$[2]') + -- A FileGroup whose primary path is incoming is updated by its own group. + AND fg.primary_path NOT IN (SELECT json_extract(value, '$[2]') FROM json_each(:incoming)) + ''', {'incoming': incoming}) + for stem_owner_id, primary_path in curs.fetchall(): + curs.execute('SELECT id FROM file_group WHERE primary_path = ?', (primary_path,)) + if path_owner := curs.fetchone(): + winner_id, loser_id = _rank_file_groups(curs, [stem_owner_id, path_owner['id']]) + _collapse_file_group(curs, loser_id, winner_id) + if winner_id == path_owner['id']: + # The survivor already has the primary path. + continue + _count_primary_path_change(curs, stem_owner_id) + curs.execute('UPDATE file_group SET primary_path = ? WHERE id = ?', (primary_path, stem_owner_id)) + + # About how many files `_upsert_files` writes per transaction. A stem's files are never split across chunks. UPSERT_CHUNK_SIZE = 500 @@ -862,12 +954,13 @@ def _upsert_files(files: List[pathlib.Path], total_files = len(files) processed_count = 0 - non_primary_files = set() for chunk in chunks_by_stem(files, UPSERT_CHUNK_SIZE): # Group all files by their shared full-stem. `chunks_by_stem` keeps each stem in one chunk. grouped = group_files_by_stem(chunk) # Convert the groups into file_groups. Extract common information into `file_group.data`. values = dict() + # The primary path of each companion file's group. + companion_owners = dict() for group in grouped: # The primary file is the video/SingleFile/epub, etc. try: @@ -882,7 +975,7 @@ def _upsert_files(files: List[pathlib.Path], stem, suffix = split_path_stem_and_suffix(primary_path) suffix = (suffix or '').lower() or None values[primary_path] = (modification_datetime, mimetype, size, files_json, suffix, stem) - non_primary_files.update(i for i in group if i != primary_path) + companion_owners.update((str(i), str(primary_path)) for i in group if i != primary_path) except FileNotFoundError as e: # A file was deleted between discovery and upsert (temp files, WAL sidecars, etc.). refresh_logger.warning(f'File vanished during refresh, skipping group near {group[0]}', exc_info=e) @@ -900,11 +993,9 @@ def _upsert_files(files: List[pathlib.Path], '(SELECT value FROM json_each(?))', (json.dumps(primary_paths),)) previously_indexed = {i['primary_path'] for i in curs.fetchall()} - if non_primary_files: - # Delete before INSERT so a new primary_path that shares (directory, stem) - # with an old sidecar FileGroup does not hit the unique index. - curs.execute('DELETE FROM file_group WHERE primary_path IN (SELECT value FROM json_each(?))', - (json.dumps(list(map(str, non_primary_files))),)) + # Before INSERT so a new primary_path does not collide on (directory, stem) with the FileGroup that + # already owns that stem. + _give_stem_owners_their_primary_paths(curs, values) placeholders, params = values_clause(values) # Compare names, not counts: a file can replace another without changing the size or newest time. @@ -953,7 +1044,15 @@ def _upsert_files(files: List[pathlib.Path], else: refresh_logger.debug(f'Upserted {len(chunk)} files near {chunk[0]}') - non_primary_files.clear() + # A companion file can still be the primary path of an old FileGroup (one with a stale stem). Its + # tags, collection membership and viewing state belong to its group's FileGroup. + if companion_owners: + ids_by_primary_path = {i['primary_path']: i['id'] for i in results} + curs.execute('SELECT id, primary_path FROM file_group WHERE primary_path IN ' + '(SELECT value FROM json_each(?))', (json.dumps(list(companion_owners)),)) + for old in curs.fetchall(): + owner_id = ids_by_primary_path[companion_owners[old['primary_path']]] + _collapse_file_group(curs, old['id'], owner_id) # Report progress after each chunk processed_count += len(chunk) @@ -1081,25 +1180,8 @@ def dedupe_file_groups_by_directory_stem() -> int: deleted = 0 for directory, stem in keys: with get_db_curs() as curs: - curs.execute(''' - SELECT fg.id, - CASE - WHEN fg.files IS NULL OR json_type(fg.files) != 'array' THEN 0 - ELSE json_array_length(fg.files) - END AS file_count, - CASE - WHEN fg.model IS NOT NULL AND fg.model != '' THEN 1 - ELSE 0 - END AS has_model, - (SELECT COUNT(*) FROM tag_file tf WHERE tf.file_group_id = fg.id) AS tag_count - FROM file_group fg - WHERE fg.directory = ? - AND fg.stem = ? - ''', (directory, stem)) - members = list(curs.fetchall()) - members.sort(key=lambda m: (m['has_model'], m['tag_count'], m['file_count'] > 0, m['id']), reverse=True) - winner_id = members[0]['id'] - loser_ids = [m['id'] for m in members[1:]] + curs.execute('SELECT id FROM file_group WHERE directory = ? AND stem = ?', (directory, stem)) + winner_id, *loser_ids = _rank_file_groups(curs, [row['id'] for row in curs.fetchall()]) with get_db_curs(commit=True) as curs: for loser_id in loser_ids: merge_file_group_relations(curs, loser_id, winner_id) diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index 038bbc19..4af40c7d 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -18,6 +18,7 @@ from PIL import Image import wrolpi.common from modules.videos import Video from wrolpi.common import timer, get_wrolpi_config +from wrolpi.config_version import get_config_version, set_config_version from wrolpi.conftest import await_switches, await_file_worker import sqlalchemy from sqlalchemy.exc import IntegrityError @@ -495,6 +496,119 @@ async def test__upsert_files_interleaved_stems(test_session, make_files_structur ]) +def _file_group_by_stem(test_session, stem: str) -> FileGroup: + test_session.expire_all() + return test_session.query(FileGroup).filter_by(stem=stem).one() + + +@pytest.mark.asyncio +async def test__upsert_files_primary_moves_to_new_file(async_client, test_session, make_files_structure, video_bytes, + tag_factory): + """A FileGroup whose primary file becomes a companion keeps its id, tags and viewing state when the new + primary file arrives.""" + jpg, = make_files_structure({'foo.jpg': 'jpg'}) + lib._upsert_files([jpg]) + foo = _file_group_by_stem(test_session, 'foo') + foo_id = foo.id + tag = await tag_factory() + foo.add_tag(test_session, tag.id) + foo.set_viewed() + set_config_version(test_session, 'tags.yaml', 1) + test_session.commit() + + mp4, = make_files_structure({'foo.mp4': video_bytes}) + lib._upsert_files([jpg, mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.id == foo_id + assert foo.primary_path == mp4 + assert foo.tag_names == [tag.name] + assert foo.viewed + # tags.yaml saves tagged files by their primary path. + assert get_config_version(test_session, 'tags.yaml') > 1 + + +@pytest.mark.asyncio +async def test__upsert_files_primary_file_replaced(async_client, test_session, make_files_structure, video_bytes, tag_factory): + """A FileGroup whose primary file was replaced by a file with the same stem is renamed, not duplicated. + + The old primary file is not in the refresh, so it is not a companion of the new group.""" + txt, = make_files_structure({'foo.txt': 'text'}) + lib._upsert_files([txt]) + foo = _file_group_by_stem(test_session, 'foo') + foo_id = foo.id + tag = await tag_factory() + foo.add_tag(test_session, tag.id) + test_session.commit() + + txt.unlink() + mp4, = make_files_structure({'foo.mp4': video_bytes}) + lib._upsert_files([mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.id == foo_id + assert foo.primary_path == mp4 + assert foo.tag_names == [tag.name] + + +@pytest.mark.asyncio +async def test__upsert_files_primary_path_and_stem_owned_by_different_rows(async_client, test_session, make_files_structure, + video_bytes, tag_factory): + """When one FileGroup owns the incoming primary path and another owns its (directory, stem), the modeled one + survives with the tags and viewing state of both.""" + jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes}) + # A row with a stale stem owns foo.mp4. + lib._upsert_files([mp4]) + stale = _file_group_by_stem(test_session, 'foo') + stale.stem = 'stale' + test_session.commit() + # The modeled row owns stem `foo`. + lib._upsert_files([jpg]) + modeled = _file_group_by_stem(test_session, 'foo') + modeled.model = 'video' + modeled_id = modeled.id + one, two = await tag_factory(), await tag_factory() + stale.add_tag(test_session, one.id) + stale.set_viewed() + modeled.add_tag(test_session, two.id) + test_session.commit() + + lib._upsert_files([jpg, mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.id == modeled_id + assert foo.primary_path == mp4 + assert sorted(foo.tag_names) == sorted([one.name, two.name]) + assert foo.viewed + + +@pytest.mark.asyncio +async def test__upsert_files_companion_with_stale_stem(async_client, test_session, make_files_structure, video_bytes, + tag_factory): + """A companion file's old FileGroup gives its tags and viewing state to the group's FileGroup before it is + deleted.""" + jpg, mp4 = make_files_structure({'foo.jpg': 'jpg', 'foo.mp4': video_bytes}) + lib._upsert_files([jpg]) + old = _file_group_by_stem(test_session, 'foo') + old.stem = 'stale' + tag = await tag_factory() + old.add_tag(test_session, tag.id) + old.set_viewed() + test_session.commit() + + lib._upsert_files([jpg, mp4]) + + foo = _file_group_by_stem(test_session, 'foo') + assert test_session.query(FileGroup).count() == 1 + assert foo.primary_path == mp4 + assert foo.tag_names == [tag.name] + assert foo.viewed + assert jpg.is_file() + + @pytest.mark.asyncio async def test__upsert_files_same_count_different_members(test_session, make_files_structure, video_bytes): """A FileGroup whose files changed is re-indexed even when the count, total size and newest modification time