Removing a duplicate video keeps the Download that is finishing (#942)

* A refresh that finishes before tags.yaml is imported no longer reports a failed save (#941)

Every refresh fires save_tags_config at its cleanup.  In the first seconds
after a start, a refresh can finish before the startup import of tags.yaml,
which on a rebuilt database happens while the first refresh runs.  The dump
then asked ConfigFile.save to write a database that held no tags over the
file; the save guard refused, as it must, but it did so with an ERROR traceback
and a "Failed to save config: config/tags.yaml" event for the user, although
nothing was lost: the import that followed read the file and recreated the tags.

save_tags_config now skips the dump while the config has not been imported and
says so at WARNING.  The file stays the source of truth until the import.

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

* Removing a duplicate video keeps the Download that is finishing

After a video download, Video.delete_duplicate_videos removes the other copies
of the same video, which happens when the file name changed (a renamed file, a
new title).  Each copy was deleted through FileGroup.delete, which also deletes
the Download of the copy's URL and, without one, adds the URL to the skip list.
The duplicates share that URL with the video just downloaded, so the Download
still running was deleted from under its worker: the worker read its row again
to record the result and failed with NoResultFound ("Unexpected error"), and the
download's record was gone.

FileGroup.delete and Video.delete take keep_download; the duplicate cleanup
passes it, so the URL's Download and the skip list are left alone.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

* Only a duplicate at the downloaded URL keeps its Download

delete_duplicate_videos also finds copies by source_id, and those can sit under
another URL.  Such a copy is deleted as any video is, with that URL's Download
or a skip-list entry; only a copy that shares the URL being downloaded leaves
the finishing Download alone.

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-11-02 19:13:25 -07:00 • committed by Roland
parent 0aa4019f86
commit 9797af157a
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 70 additions and 10 deletions

View file

@ -104,10 +104,10 @@ class Video(ModelHelper, Base):
)
return d
def delete(self, add_to_skip_list: bool = True, force: bool = False):
def delete(self, add_to_skip_list: bool = True, force: bool = False, keep_download: bool = False):
"""Remove all files and File records related to this video. Delete this Video record.
Add it to it's Channel's skip list."""
self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force)
Add it to it's Channel's skip list. See `FileGroup.delete` for `keep_download`."""
self.file_group.delete(add_to_skip_list=add_to_skip_list, force=force, keep_download=keep_download)
session = Session.object_session(self)
session.delete(self)
@ -497,12 +497,14 @@ class Video(ModelHelper, Base):
tag_names = {i for j in duplicate_videos for i in j.file_group.tag_names}
def delete_video(video_: Video):
"""Delete a Video with any of its tags."""
"""Delete a Video with any of its tags. A copy at `url` shares it with the kept video, so that URL's
Download (the one that is finishing, and reads its row afterwards) and the skip list are left alone. A
copy found by `source_id` under another URL is deleted as any video is, with that URL's Download."""
# Untagging forgets which links are this Video's, so remove them first.
video_.file_group.unlink_tag_directory_links()
for tag_name_ in video_.file_group.tag_names:
video_.file_group.untag(session, tag_name_)
video_.delete()
video_.delete(keep_download=video_.file_group.url == url)
changes = False
# Delete any Videos without real files.

View file

@ -193,6 +193,61 @@ async def test_delete_duplicate_video(async_client, test_session, channel_factor
assert video.file_group.tag_names == [tag.name, ]
@pytest.mark.asyncio
async def test_delete_duplicate_video_keeps_the_download(async_client, test_session, channel_factory, video_factory):
"""The duplicates share the URL of the Download that is finishing. Deleting the losing copy must not delete
that Download (the worker reads its row again to record the result) nor put the URL on the skip list."""
from wrolpi.downloader import Download, download_manager
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
url = 'https://example.com/video'
for title in (f'{channel.name}_20000101_ABC123456_The video title',
f'{channel.name}_20000101_ABC123456_The video title (renamed)'):
video = video_factory(channel_id=channel.id, title=title, upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456', with_video_file=True, with_info_json=True)
video.file_group.url = url
download = Download(url=url, status='pending', attempts=1)
test_session.add(download)
test_session.commit()
assert await Video.delete_duplicate_videos(test_session, url, 'ABC123456', video_path)
test_session.commit()
assert test_session.query(Video).count() == 1
assert test_session.query(Download).filter_by(url=url).count() == 1, 'The running Download was deleted'
assert not download_manager.is_skipped(url), 'The URL of the kept video must not be skipped'
@pytest.mark.asyncio
async def test_delete_duplicate_video_under_another_url_deletes_its_download(async_client, test_session,
channel_factory, video_factory):
"""A copy found by `source_id` under a different URL is an ordinary delete: its own Download goes with it, while
the Download of the URL being downloaded is kept."""
from wrolpi.downloader import Download, download_manager
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
url, other_url = 'https://example.com/video', 'https://example.com/short/video'
new = video_factory(channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), source_id='ABC123456',
with_video_file=True, with_info_json=True)
new.file_group.url = url
old = video_factory(channel_id=channel.id, title=f'{channel.name}_20000101_ABC123456_The video title (old)',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), source_id='ABC123456',
with_video_file=True, with_info_json=True)
old.file_group.url = other_url
test_session.add_all([Download(url=url, status='pending', attempts=1), Download(url=other_url, status='complete')])
test_session.commit()
assert await Video.delete_duplicate_videos(test_session, url, 'ABC123456', video_path)
test_session.commit()
assert [i.file_group.url for i in test_session.query(Video)] == [url]
assert [i.url for i in test_session.query(Download)] == [url], 'Only the finishing Download remains'
assert not download_manager.is_skipped(url)
@pytest.mark.asyncio
async def test_delete_duplicate_video_tagged_duplicate(async_client, test_session, channel_factory, video_factory,
tag_factory):

View file

@ -475,9 +475,11 @@ class FileGroup(ModelHelper, Base):
else:
self.replace_info_json(contents)
def delete(self, add_to_skip_list: bool = True, force: bool = False):
def delete(self, add_to_skip_list: bool = True, force: bool = False, keep_download: bool = False):
"""Delete this FileGroup record, and all of its files.
@param keep_download: Leave the Download of my URL and the skip list alone: the URL still belongs to another
FileGroup (a duplicate is being removed while its Download finishes).
@raise FileGroupIsTagged: If any tags are related to me and force is False."""
if self.tag_files and not force:
raise FileGroupIsTagged(f'Cannot delete {self} because it has tags')
@ -488,10 +490,11 @@ class FileGroup(ModelHelper, Base):
if session := Session.object_session(self):
from wrolpi.downloader import download_manager
if self.url and (download := Download.get_by_url(session, self.url)):
download.delete(add_to_skip_list)
elif self.url and add_to_skip_list is True:
download_manager.add_to_skip_list(self.url)
if not keep_download:
if self.url and (download := Download.get_by_url(session, self.url)):
download.delete(add_to_skip_list)
elif self.url and add_to_skip_list is True:
download_manager.add_to_skip_list(self.url)
session.delete(self)
@property