Creating FileGroup.merge to be used in FileGroup.from_paths.

This commit is contained in:
Roland 2023-05-03 08:28:53 -06:00
parent 77635b7926
commit be3e8375a9
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
3 changed files with 79 additions and 20 deletions

View file

@ -204,11 +204,7 @@ class Video(ModelHelper, Base):
@staticmethod
def from_paths(session: Session, *paths: pathlib.Path) -> 'Video':
str_paths = list(map(str, paths))
file_group = session.query(FileGroup).filter(FileGroup.primary_path == any_(str_paths)).one_or_none()
if not file_group:
# This video has not been downloaded previously.
file_group = FileGroup.from_paths(session, *paths)
file_group = FileGroup.from_paths(session, *paths)
# Video may have been downloaded previously.
video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none()

View file

@ -226,19 +226,34 @@ class FileGroup(ModelHelper, Base):
def from_paths(cls, session: Session, *paths: pathlib.Path) -> 'FileGroup':
"""Create a new FileGroup which contains the provided file paths."""
from wrolpi.files.lib import get_primary_file, get_mimetype
file_group = FileGroup()
primary_path = get_primary_file(paths)
existing_groups = session.query(FileGroup).filter(FileGroup.primary_path.in_(list(map(str, paths)))).all()
if len(existing_groups) == 0:
# These paths have not been used previously, create a new FileGroup.
file_group = FileGroup()
primary_path = get_primary_file(paths)
file_group.primary_path = primary_path
file_group.append_files(*paths)
file_group.mimetype = get_mimetype(file_group.primary_path)
session.add(file_group)
elif len(existing_groups) == 1:
# Found one FileGroup with these paths, no need to create a new FileGroup.
file_group = existing_groups[0]
primary_path = file_group.primary_path
else:
# Multiple FileGroups contain these paths as primary.
primary_path = get_primary_file(paths)
file_group: FileGroup = next(filter(lambda i: i.primary_path == primary_path, existing_groups), None)
if not file_group:
file_group = FileGroup.from_paths(session, primary_path)
file_group.merge(existing_groups)
if not isinstance(primary_path, pathlib.Path):
raise ValueError('Cannot create FileGroup without a primary path.')
file_group.append_files(*paths)
file_group.primary_path = primary_path
file_group.mimetype = get_mimetype(file_group.primary_path)
file_group.modification_datetime = from_timestamp(max(i.stat().st_mtime for i in paths))
file_group.size = sum(i.stat().st_size for i in paths)
session.add(file_group)
return file_group
@staticmethod
@ -251,6 +266,29 @@ class FileGroup(ModelHelper, Base):
def tag_names(self) -> List[str]:
return [i.tag.name for i in self.tag_files]
def merge(self, file_groups: List['FileGroup']):
"""Consume the files and Tags of the provided FileGroups and attach them to this FileGroup. Delete the provided
FileGroups."""
session = Session.object_session(self)
collected_files = self.files.copy()
for file_group in file_groups:
if file_group.primary_path == self.primary_path:
# Don't merge myself.
continue
for file in file_group.files:
if file['path'] not in self.my_paths():
collected_files.append(file)
# Move any applied Tags.
for tag_file in file_group.tag_files:
if tag_file.tag.name not in self.tag_names:
# Preserve the created at.
self.add_tag(tag_file.tag).created_at = tag_file.created_at
session.delete(file_group)
self.files = collected_files
class Directory(ModelHelper, Base):
"""A representation of a file directory in the media directory."""

View file

@ -1,5 +1,6 @@
import asyncio
import json
import pathlib
import shutil
import zipfile
from datetime import datetime
@ -515,7 +516,8 @@ def test_group_files_by_stem(make_files_structure, test_directory):
assert list(lib.group_files_by_stem(files)) == [
[test_directory / 'bar.txt'],
[test_directory / 'baz.txt'],
[test_directory / 'foo.info.json', test_directory / 'foo.live_chat.json', test_directory / 'foo.mp4', test_directory / 'foo.txt'],
[test_directory / 'foo.info.json', test_directory / 'foo.live_chat.json', test_directory / 'foo.mp4',
test_directory / 'foo.txt'],
]
@ -636,12 +638,35 @@ async def test_refresh_directories(test_session, test_directory, assert_director
assert_directories({'foo', 'baz'})
def test_file_group_primary_path_is_unique(test_session, test_directory):
"""You cannot create a FileGroup that shares a path"""
path = test_directory / 'file.txt'
path.touch()
def test_file_group_merge(test_session, test_directory, make_files_structure, tag_factory, video_bytes, srt_file3):
"""A FileGroup can be created from multiple existing FileGroups. Any Tags applied to the existing groups will be
migrated."""
vid, srt = make_files_structure({
'vid.mp4': video_bytes,
'vid.srt': (PROJECT_DIR / 'test/example3.en.srt').read_text(),
})
one, two = tag_factory(), tag_factory()
vid_group = FileGroup.from_paths(test_session, vid)
srt_group = FileGroup.from_paths(test_session, srt)
test_session.add_all([vid_group, srt_group])
test_session.flush([vid_group, srt_group])
vid_tag_file = vid_group.add_tag(one, test_session)
srt_tag_file = srt_group.add_tag(two, test_session)
test_session.flush([vid_tag_file, srt_tag_file])
tag_file_created_at = vid_tag_file.created_at
srt_file_created_at = srt_tag_file.created_at
test_session.commit()
one = FileGroup.from_paths(test_session, path)
two = FileGroup.from_paths(test_session, path)
with pytest.raises(sqlalchemy.exc.IntegrityError):
test_session.commit()
# Both FileGroups are merged.
vid = FileGroup.from_paths(test_session, vid, srt)
test_session.commit()
assert {i['path'].name for i in vid.files} == {'vid.mp4', 'vid.srt'}
assert test_session.query(FileGroup).count() == 1
assert set(vid.tag_names) == {'one', 'two'}
assert {i['path'].name for i in vid.files} == {'vid.mp4', 'vid.srt'}
# TagFile.created_at is preserved.
assert [i for i in vid.tag_files if i.tag.name == 'one'][0].created_at == tag_file_created_at
assert [i for i in vid.tag_files if i.tag.name == 'two'][0].created_at == srt_file_created_at
# Size is combined
assert vid.size > len(video_bytes)