From be3e8375a943cf1d42e7bf990b6355ca8928a53c Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 3 May 2023 08:28:53 -0600 Subject: [PATCH] Creating FileGroup.merge to be used in FileGroup.from_paths. --- modules/videos/models.py | 6 +---- wrolpi/files/models.py | 50 ++++++++++++++++++++++++++++++----- wrolpi/files/test/test_lib.py | 43 +++++++++++++++++++++++------- 3 files changed, 79 insertions(+), 20 deletions(-) diff --git a/modules/videos/models.py b/modules/videos/models.py index 6c9eceb2..e827045b 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -204,11 +204,7 @@ class Video(ModelHelper, Base): @staticmethod def from_paths(session: Session, *paths: pathlib.Path) -> 'Video': - str_paths = list(map(str, paths)) - file_group = session.query(FileGroup).filter(FileGroup.primary_path == any_(str_paths)).one_or_none() - if not file_group: - # This video has not been downloaded previously. - file_group = FileGroup.from_paths(session, *paths) + file_group = FileGroup.from_paths(session, *paths) # Video may have been downloaded previously. video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none() diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index d3ff5a66..1269eac3 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -226,19 +226,34 @@ class FileGroup(ModelHelper, Base): def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup': """Create a new FileGroup which contains the provided file paths.""" from wrolpi.files.lib import get_primary_file, get_mimetype - file_group = FileGroup() - primary_path = get_primary_file(paths) + existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all() + if len(existing_groups) == 0: + # These paths have not been used previously, create a new FileGroup. + file_group = FileGroup() + primary_path = get_primary_file(paths) + file_group.primary_path = primary_path + file_group.append_files(*paths) + file_group.mimetype = get_mimetype(file_group.primary_path) + session.add(file_group) + elif len(existing_groups) == 1: + # Found one FileGroup with these paths, no need to create a new FileGroup. + file_group = existing_groups[0] + primary_path = file_group.primary_path + else: + # Multiple FileGroups contain these paths as primary. + primary_path = get_primary_file(paths) + file_group: FileGroup = next(filter(lambda i: i.primary_path == primary_path, existing_groups), None) + if not file_group: + file_group = FileGroup.from_paths(session, primary_path) + file_group.merge(existing_groups) + if not isinstance(primary_path, pathlib.Path): raise ValueError('Cannot create FileGroup without a primary path.') - file_group.append_files(*paths) - file_group.primary_path = primary_path - file_group.mimetype = get_mimetype(file_group.primary_path) file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths)) file_group.size = sum(i.stat().st_size for i in paths) - session.add(file_group) return file_group @staticmethod @@ -251,6 +266,29 @@ class FileGroup(ModelHelper, Base): def tag_names(self) -> List[str]: return [i.tag.name for i in self.tag_files] + def merge(self, file_groups: List['FileGroup']): + """Consume the files and Tags of the provided FileGroups and attach them to this FileGroup. Delete the provided + FileGroups.""" + session = Session.object_session(self) + + collected_files = self.files.copy() + for file_group in file_groups: + if file_group.primary_path == self.primary_path: + # Don't merge myself. + continue + + for file in file_group.files: + if file['path'] not in self.my_paths(): + collected_files.append(file) + # Move any applied Tags. + for tag_file in file_group.tag_files: + if tag_file.tag.name not in self.tag_names: + # Preserve the created at. + self.add_tag(tag_file.tag).created_at = tag_file.created_at + session.delete(file_group) + + self.files = collected_files + class Directory(ModelHelper, Base): """A representation of a file directory in the media directory.""" diff --git a/wrolpi/files/test/test_lib.py b/wrolpi/files/test/test_lib.py index b4800c1f..6433e2cc 100644 --- a/wrolpi/files/test/test_lib.py +++ b/wrolpi/files/test/test_lib.py @@ -1,5 +1,6 @@ import asyncio import json +import pathlib import shutil import zipfile from datetime import datetime @@ -515,7 +516,8 @@ def test_group_files_by_stem(make_files_structure, test_directory): assert list(lib.group_files_by_stem(files)) == [ [test_directory / 'bar.txt'], [test_directory / 'baz.txt'], - [test_directory / 'foo.info.json', test_directory / 'foo.live_chat.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'], + [test_directory / 'foo.info.json', test_directory / 'foo.live_chat.json', test_directory / 'foo.mp4', + test_directory / 'foo.txt'], ] @@ -636,12 +638,35 @@ async def test_refresh_directories(test_session, test_directory, assert_director assert_directories({'foo', 'baz'}) -def test_file_group_primary_path_is_unique(test_session, test_directory): - """You cannot create a FileGroup that shares a path""" - path = test_directory / 'file.txt' - path.touch() +def test_file_group_merge(test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3): + """A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be + migrated.""" + vid, srt = make_files_structure({ + 'vid.mp4': video_bytes, + 'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(), + }) + one, two = tag_factory(), tag_factory() + vid_group = FileGroup.from_paths(test_session, vid) + srt_group = FileGroup.from_paths(test_session, srt) + test_session.add_all([vid_group, srt_group]) + test_session.flush([vid_group, srt_group]) + vid_tag_file = vid_group.add_tag(one, test_session) + srt_tag_file = srt_group.add_tag(two, test_session) + test_session.flush([vid_tag_file, srt_tag_file]) + tag_file_created_at = vid_tag_file.created_at + srt_file_created_at = srt_tag_file.created_at + test_session.commit() - one = FileGroup.from_paths(test_session, path) - two = FileGroup.from_paths(test_session, path) - with pytest.raises(sqlalchemy.exc.IntegrityError): - test_session.commit() + # Both FileGroups are merged. + vid = FileGroup.from_paths(test_session, vid, srt) + test_session.commit() + assert {i['path'].name for i in vid.files} == {'vid.mp4', 'vid.srt'} + + assert test_session.query(FileGroup).count() == 1 + assert set(vid.tag_names) == {'one', 'two'} + assert {i['path'].name for i in vid.files} == {'vid.mp4', 'vid.srt'} + # TagFile.created_at is preserved. + assert [i for i in vid.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at + assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at + # Size is combined + assert vid.size > len(video_bytes)