diff --git a/modules/videos/models.py b/modules/videos/models.py index 0b030ca4..eb0e640b 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -104,10 +104,10 @@ class Video(ModelHelper, Base): ) return d - def delete(self, add_to_skip_list: bool = True, force: bool = False): + def delete(self, add_to_skip_list: bool = True, force: bool = False, keep_download: bool = False): """Remove all files and File records related to this video. Delete this Video record. - Add it to it's Channel's skip list.""" - self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force) + Add it to it's Channel's skip list. See `FileGroup.delete` for `keep_download`.""" + self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force, keep_download=keep_download) session = Session.object_session(self) session.delete(self) @@ -497,12 +497,14 @@ class Video(ModelHelper, Base): tag_names = {i for j in duplicate_videos for i in j.file_group.tag_names} def delete_video(video_: Video): - """Delete a Video with any of its tags.""" + """Delete a Video with any of its tags. A copy at `url` shares it with the kept video, so that URL's + Download (the one that is finishing, and reads its row afterwards) and the skip list are left alone. A + copy found by `source_id` under another URL is deleted as any video is, with that URL's Download.""" # Untagging forgets which links are this Video's, so remove them first. video_.file_group.unlink_tag_directory_links() for tag_name_ in video_.file_group.tag_names: video_.file_group.untag(session, tag_name_) - video_.delete() + video_.delete(keep_download=video_.file_group.url == url) changes = False # Delete any Videos without real files. diff --git a/modules/videos/test/test_video.py b/modules/videos/test/test_video.py index 5ccbee9e..aa999eb7 100644 --- a/modules/videos/test/test_video.py +++ b/modules/videos/test/test_video.py @@ -193,6 +193,61 @@ async def test_delete_duplicate_video(async_client, test_session, channel_factor assert video.file_group.tag_names == [tag.name, ] +@pytest.mark.asyncio +async def test_delete_duplicate_video_keeps_the_download(async_client, test_session, channel_factory, video_factory): + """The duplicates share the URL of the Download that is finishing. Deleting the losing copy must not delete + that Download (the worker reads its row again to record the result) nor put the URL on the skip list.""" + from wrolpi.downloader import Download, download_manager + channel = channel_factory(name='Channel Name') + video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4' + url = 'https://example.com/video' + + for title in (f'{channel.name}_20000101_ABC123456_The video title', + f'{channel.name}_20000101_ABC123456_The video title (renamed)'): + video = video_factory(channel_id=channel.id, title=title, upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), + source_id='ABC123456', with_video_file=True, with_info_json=True) + video.file_group.url = url + download = Download(url=url, status='pending', attempts=1) + test_session.add(download) + test_session.commit() + + assert await Video.delete_duplicate_videos(test_session, url, 'ABC123456', video_path) + test_session.commit() + + assert test_session.query(Video).count() == 1 + assert test_session.query(Download).filter_by(url=url).count() == 1, 'The running Download was deleted' + assert not download_manager.is_skipped(url), 'The URL of the kept video must not be skipped' + + +@pytest.mark.asyncio +async def test_delete_duplicate_video_under_another_url_deletes_its_download(async_client, test_session, + channel_factory, video_factory): + """A copy found by `source_id` under a different URL is an ordinary delete: its own Download goes with it, while + the Download of the URL being downloaded is kept.""" + from wrolpi.downloader import Download, download_manager + channel = channel_factory(name='Channel Name') + video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4' + url, other_url = 'https://example.com/video', 'https://example.com/short/video' + + new = video_factory(channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title', + upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), source_id='ABC123456', + with_video_file=True, with_info_json=True) + new.file_group.url = url + old = video_factory(channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title (old)', + upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), source_id='ABC123456', + with_video_file=True, with_info_json=True) + old.file_group.url = other_url + test_session.add_all([Download(url=url, status='pending', attempts=1), Download(url=other_url, status='complete')]) + test_session.commit() + + assert await Video.delete_duplicate_videos(test_session, url, 'ABC123456', video_path) + test_session.commit() + + assert [i.file_group.url for i in test_session.query(Video)] == [url] + assert [i.url for i in test_session.query(Download)] == [url], 'Only the finishing Download remains' + assert not download_manager.is_skipped(url) + + @pytest.mark.asyncio async def test_delete_duplicate_video_tagged_duplicate(async_client, test_session, channel_factory, video_factory, tag_factory): diff --git a/wrolpi/files/models.py b/wrolpi/files/models.py index adc88099..4f71f16a 100644 --- a/wrolpi/files/models.py +++ b/wrolpi/files/models.py @@ -475,9 +475,11 @@ class FileGroup(ModelHelper, Base): else: self.replace_info_json(contents) - def delete(self, add_to_skip_list: bool = True, force: bool = False): + def delete(self, add_to_skip_list: bool = True, force: bool = False, keep_download: bool = False): """Delete this FileGroup record, and all of its files. + @param keep_download: Leave the Download of my URL and the skip list alone: the URL still belongs to another + FileGroup (a duplicate is being removed while its Download finishes). @raise FileGroupIsTagged: If any tags are related to me and force is False.""" if self.tag_files and not force: raise FileGroupIsTagged(f'Cannot delete {self} because it has tags') @@ -488,10 +490,11 @@ class FileGroup(ModelHelper, Base): if session := Session.object_session(self): from wrolpi.downloader import download_manager - if self.url and (download := Download.get_by_url(session, self.url)): - download.delete(add_to_skip_list) - elif self.url and add_to_skip_list is True: - download_manager.add_to_skip_list(self.url) + if not keep_download: + if self.url and (download := Download.get_by_url(session, self.url)): + download.delete(add_to_skip_list) + elif self.url and add_to_skip_list is True: + download_manager.add_to_skip_list(self.url) session.delete(self) @property