mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 14:36:22 -05:00
Removing a duplicate video keeps the Download that is finishing (#942)
* A refresh that finishes before tags.yaml is imported no longer reports a failed save (#941) Every refresh fires save_tags_config at its cleanup. In the first seconds after a start, a refresh can finish before the startup import of tags.yaml, which on a rebuilt database happens while the first refresh runs. The dump then asked ConfigFile.save to write a database that held no tags over the file; the save guard refused, as it must, but it did so with an ERROR traceback and a "Failed to save config: config/tags.yaml" event for the user, although nothing was lost: the import that followed read the file and recreated the tags. save_tags_config now skips the dump while the config has not been imported and says so at WARNING. The file stays the source of truth until the import. Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com> * Removing a duplicate video keeps the Download that is finishing After a video download, Video.delete_duplicate_videos removes the other copies of the same video, which happens when the file name changed (a renamed file, a new title). Each copy was deleted through FileGroup.delete, which also deletes the Download of the copy's URL and, without one, adds the URL to the skip list. The duplicates share that URL with the video just downloaded, so the Download still running was deleted from under its worker: the worker read its row again to record the result and failed with NoResultFound ("Unexpected error"), and the download's record was gone. FileGroup.delete and Video.delete take keep_download; the duplicate cleanup passes it, so the URL's Download and the skip list are left alone. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> * Only a duplicate at the downloaded URL keeps its Download delete_duplicate_videos also finds copies by source_id, and those can sit under another URL. Such a copy is deleted as any video is, with that URL's Download or a skip-list entry; only a copy that shares the URL being downloaded leaves the finishing Download alone. Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com> --------- Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
parent
0aa4019f86
commit
9797af157a
3 changed files with 70 additions and 10 deletions
|
|
@ -104,10 +104,10 @@ class Video(ModelHelper, Base):
|
|||
)
|
||||
return d
|
||||
|
||||
def delete(self, add_to_skip_list: bool = True, force: bool = False):
|
||||
def delete(self, add_to_skip_list: bool = True, force: bool = False, keep_download: bool = False):
|
||||
"""Remove all files and File records related to this video. Delete this Video record.
|
||||
Add it to it's Channel's skip list."""
|
||||
self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force)
|
||||
Add it to it's Channel's skip list. See `FileGroup.delete` for `keep_download`."""
|
||||
self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force, keep_download=keep_download)
|
||||
session = Session.object_session(self)
|
||||
session.delete(self)
|
||||
|
||||
|
|
@ -497,12 +497,14 @@ class Video(ModelHelper, Base):
|
|||
tag_names = {i for j in duplicate_videos for i in j.file_group.tag_names}
|
||||
|
||||
def delete_video(video_: Video):
|
||||
"""Delete a Video with any of its tags."""
|
||||
"""Delete a Video with any of its tags. A copy at `url` shares it with the kept video, so that URL's
|
||||
Download (the one that is finishing, and reads its row afterwards) and the skip list are left alone. A
|
||||
copy found by `source_id` under another URL is deleted as any video is, with that URL's Download."""
|
||||
# Untagging forgets which links are this Video's, so remove them first.
|
||||
video_.file_group.unlink_tag_directory_links()
|
||||
for tag_name_ in video_.file_group.tag_names:
|
||||
video_.file_group.untag(session, tag_name_)
|
||||
video_.delete()
|
||||
video_.delete(keep_download=video_.file_group.url == url)
|
||||
|
||||
changes = False
|
||||
# Delete any Videos without real files.
|
||||
|
|
|
|||
|
|
@ -193,6 +193,61 @@ async def test_delete_duplicate_video(async_client, test_session, channel_factor
|
|||
assert video.file_group.tag_names == [tag.name, ]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_duplicate_video_keeps_the_download(async_client, test_session, channel_factory, video_factory):
|
||||
"""The duplicates share the URL of the Download that is finishing. Deleting the losing copy must not delete
|
||||
that Download (the worker reads its row again to record the result) nor put the URL on the skip list."""
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
channel = channel_factory(name='Channel Name')
|
||||
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
|
||||
url = 'https://example.com/video'
|
||||
|
||||
for title in (f'{channel.name}_20000101_ABC123456_The video title',
|
||||
f'{channel.name}_20000101_ABC123456_The video title (renamed)'):
|
||||
video = video_factory(channel_id=channel.id, title=title, upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
|
||||
source_id='ABC123456', with_video_file=True, with_info_json=True)
|
||||
video.file_group.url = url
|
||||
download = Download(url=url, status='pending', attempts=1)
|
||||
test_session.add(download)
|
||||
test_session.commit()
|
||||
|
||||
assert await Video.delete_duplicate_videos(test_session, url, 'ABC123456', video_path)
|
||||
test_session.commit()
|
||||
|
||||
assert test_session.query(Video).count() == 1
|
||||
assert test_session.query(Download).filter_by(url=url).count() == 1, 'The running Download was deleted'
|
||||
assert not download_manager.is_skipped(url), 'The URL of the kept video must not be skipped'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_duplicate_video_under_another_url_deletes_its_download(async_client, test_session,
|
||||
channel_factory, video_factory):
|
||||
"""A copy found by `source_id` under a different URL is an ordinary delete: its own Download goes with it, while
|
||||
the Download of the URL being downloaded is kept."""
|
||||
from wrolpi.downloader import Download, download_manager
|
||||
channel = channel_factory(name='Channel Name')
|
||||
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
|
||||
url, other_url = 'https://example.com/video', 'https://example.com/short/video'
|
||||
|
||||
new = video_factory(channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title',
|
||||
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), source_id='ABC123456',
|
||||
with_video_file=True, with_info_json=True)
|
||||
new.file_group.url = url
|
||||
old = video_factory(channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title (old)',
|
||||
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), source_id='ABC123456',
|
||||
with_video_file=True, with_info_json=True)
|
||||
old.file_group.url = other_url
|
||||
test_session.add_all([Download(url=url, status='pending', attempts=1), Download(url=other_url, status='complete')])
|
||||
test_session.commit()
|
||||
|
||||
assert await Video.delete_duplicate_videos(test_session, url, 'ABC123456', video_path)
|
||||
test_session.commit()
|
||||
|
||||
assert [i.file_group.url for i in test_session.query(Video)] == [url]
|
||||
assert [i.url for i in test_session.query(Download)] == [url], 'Only the finishing Download remains'
|
||||
assert not download_manager.is_skipped(url)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_duplicate_video_tagged_duplicate(async_client, test_session, channel_factory, video_factory,
|
||||
tag_factory):
|
||||
|
|
|
|||
|
|
@ -475,9 +475,11 @@ class FileGroup(ModelHelper, Base):
|
|||
else:
|
||||
self.replace_info_json(contents)
|
||||
|
||||
def delete(self, add_to_skip_list: bool = True, force: bool = False):
|
||||
def delete(self, add_to_skip_list: bool = True, force: bool = False, keep_download: bool = False):
|
||||
"""Delete this FileGroup record, and all of its files.
|
||||
|
||||
@param keep_download: Leave the Download of my URL and the skip list alone: the URL still belongs to another
|
||||
FileGroup (a duplicate is being removed while its Download finishes).
|
||||
@raise FileGroupIsTagged: If any tags are related to me and force is False."""
|
||||
if self.tag_files and not force:
|
||||
raise FileGroupIsTagged(f'Cannot delete {self} because it has tags')
|
||||
|
|
@ -488,10 +490,11 @@ class FileGroup(ModelHelper, Base):
|
|||
|
||||
if session := Session.object_session(self):
|
||||
from wrolpi.downloader import download_manager
|
||||
if self.url and (download := Download.get_by_url(session, self.url)):
|
||||
download.delete(add_to_skip_list)
|
||||
elif self.url and add_to_skip_list is True:
|
||||
download_manager.add_to_skip_list(self.url)
|
||||
if not keep_download:
|
||||
if self.url and (download := Download.get_by_url(session, self.url)):
|
||||
download.delete(add_to_skip_list)
|
||||
elif self.url and add_to_skip_list is True:
|
||||
download_manager.add_to_skip_list(self.url)
|
||||
session.delete(self)
|
||||
|
||||
@property
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue