Duplicate videos are deleted when a video is downloaded.

This commit is contained in:
Roland 2024-02-21 12:39:13 -07:00
parent 26ddd27a77
commit 0d6edc05d0
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 223 additions and 21 deletions

View file

@ -207,18 +207,17 @@ class VideoDownloader(Downloader, ABC):
raise UnrecoverableDownloadError('Max download attempts reached')
url = normalize_video_url(download.url)
info = download.info_json or extract_info(url)
download.info_json = download.info_json or extract_info(url)
if not info:
if not download.info_json:
raise ValueError(f'Cannot download video with no info_json.')
found_channel = None
channel_name = info.get('channel')
source_channel_id = info.get('channel_id')
channel_url = info.get('channel_url')
channel_name = download.info_json.get('channel')
source_channel_id = download.info_json.get('channel_id')
channel_url = download.info_json.get('channel_url')
channel = None
logger.debug(f'{channel_name=} {source_channel_id=} {channel_url=}')
if channel_name or source_channel_id:
# Try to find the channel via info_json from yt-dlp.
try:
@ -284,11 +283,12 @@ class VideoDownloader(Downloader, ABC):
logs = None # noqa
try:
video_path, entry = self.prepare_filename(url, out_dir)
if settings.get('do_not_download'):
# User has requested refresh of video metadata, skip the rest of the video downloader.
return await self.download_info_json(download)
return await self.download_info_json(download, video_path)
video_path, entry = self.prepare_filename(url, out_dir)
# Do the real download.
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
cmd = (
@ -354,18 +354,24 @@ class VideoDownloader(Downloader, ABC):
video = Video.from_paths(session, *video_paths)
video.source_id = entry['id']
video.channel_id = channel_id
video_id = video.id
video_id, video_info_json_path = video.id, video.info_json_path
if video.info_json_path:
format_json_file(video.info_json_path)
session.commit()
if video_info_json_path:
format_json_file(video_info_json_path)
with get_db_session() as session:
# Second session is started because SQLAlchemy will forget what we have done.
need_commit = False
video = Video.get_by_id(video_id, session)
if download.settings and (tag_names := download.settings.get('tag_names')):
existing_names = video.file_group.tag_names
for name in tag_names:
if name not in existing_names:
video.add_tag(name)
need_commit = True
if need_commit:
session.commit()
# Check that video has both audio and video streams.
await video.get_ffprobe_json()
@ -379,7 +385,9 @@ class VideoDownloader(Downloader, ABC):
success=False,
error='Video was downloaded but did not contain audio stream',
)
session.commit()
with get_db_session(commit=True) as session:
await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path)
except UnrecoverableDownloadError:
raise
@ -390,7 +398,7 @@ class VideoDownloader(Downloader, ABC):
if _skip_download(e):
# The video failed to download, and the error will never be fixed. Skip it forever.
try:
source_id = info.get('id')
source_id = download.info_json.get('id')
logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
f'attempt to download it again.')
download.add_to_skip_list()
@ -446,12 +454,19 @@ class VideoDownloader(Downloader, ABC):
logger.debug(f'Downloading {url} to {out_dir}')
return final_filename, entry
async def download_info_json(self, download: Download) -> DownloadResult:
async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult:
"""User has requested to refresh Video metadata. Download only metadata files to temporary directory.
Replace existing metadata files only after successful download."""
url = download.url
source_id = download.info_json['id']
with get_db_session(commit=True) as session:
with get_db_session() as session:
# Delete any duplicate videos before attempting to get info json.
deleted = await Video.delete_duplicate_videos(session, download.url, source_id, video_path)
if deleted:
session.commit()
with get_db_session() as session:
# Find the existing video, replace its info json.
video = Video.get_by_url(url, session)

View file

@ -239,10 +239,11 @@ class Video(ModelHelper, Base):
video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none()
if not video:
video = Video(file_group=file_group)
video.validate()
session.add(video)
session.flush([video, file_group])
video.validate()
session.flush([video, ])
return video
@property
@ -333,9 +334,9 @@ class Video(ModelHelper, Base):
raise UnknownVideo(f'Cannot find Video with id {id_}')
return video
def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile:
def add_tag(self, tag_or_tag_name: Union[Tag, str], session: Session = None) -> TagFile:
tag = Tag.find_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
return self.file_group.add_tag(tag)
return self.file_group.add_tag(tag, session=session)
async def get_ffprobe_json(self) -> dict:
"""Return the ffprobe json object if previously stored.
@ -386,6 +387,86 @@ class Video(ModelHelper, Base):
if self.channel and self.source_id:
return self.channel.get_video_entry_by_id(self.source_id)
@staticmethod
async def delete_duplicate_videos(session: Session, url: str, source_id: str, video_path: pathlib.Path) -> bool:
"""Searches for Videos that share a `source_id` or `url`. Attempts to keep the best video at `video_path`, then
deletes the rest."""
# Get all Videos that share the URL or the source_id.
matching_videos_with_url = session.query(Video) \
.join(FileGroup, FileGroup.id == Video.file_group_id) \
.filter(FileGroup.url == url) \
.all()
matching_videos_with_source_id = session.query(Video) \
.join(FileGroup, FileGroup.id == Video.file_group_id) \
.filter(FileGroup.model == 'video', Video.source_id == source_id) \
.all()
duplicate_videos = list()
for video in matching_videos_with_url:
if video.id not in [i.id for i in duplicate_videos]:
duplicate_videos.append(video)
for video in matching_videos_with_source_id:
if video.id not in [i.id for i in duplicate_videos]:
duplicate_videos.append(video)
# All the tags of all the files.
tag_names = {i for j in duplicate_videos for i in j.file_group.tag_names}
def delete_video(video_: Video):
"""Delete a Video with any of its tags."""
for tag_name_ in video_.file_group.tag_names:
tag_ = Tag.find_by_name(tag_name_)
video_.file_group.remove_tag(tag_, session)
video_.delete()
changes = False
# Delete any Videos without real files.
missing_file_indexes = []
for idx, video in enumerate(duplicate_videos):
if not video.video_path.is_file():
logger.debug(f'Deleting {video} because the video file does not exist')
missing_file_indexes.append(idx)
delete_video(video)
changes = True
duplicate_videos = [i for idx, i in enumerate(duplicate_videos) if idx not in missing_file_indexes]
if len(duplicate_videos) < 2:
# No duplicates.
logger.debug(f'No duplicates to delete: {url=} {source_id=}')
return changes
# Order videos by oldest first, we should keep the oldest if all files are the same size.
duplicate_videos = sorted(duplicate_videos, key=lambda i: i.file_group.primary_path.stat().st_mtime)
# Find the largest video, if possible. If all videos are the same size then rename the oldest.
largest_video_idx = 0
for idx, video in enumerate(duplicate_videos):
if video.file_group.size > duplicate_videos[largest_video_idx].file_group.size:
largest_video_idx = idx
largest_video = duplicate_videos.pop(largest_video_idx)
# Delete the video at the destination, if it is not the largest.
existing_video = Video.get_by_path(video_path, session=session)
if existing_video and largest_video != existing_video:
delete_video(existing_video)
for idx, video in enumerate(duplicate_videos):
if video == existing_video:
duplicate_videos.pop(idx)
# Rename the video, add any missing tag names.
if largest_video.video_path != video_path:
largest_video.file_group.move(video_path)
for tag_name in tag_names:
if tag_name not in largest_video.file_group.tag_names:
largest_video.add_tag(tag_name, session)
logger.warning(f'Deleting duplicate videos: {duplicate_videos}')
# Remove tags from all videos, delete all videos.
for video in duplicate_videos:
delete_video(video)
return True
class Channel(ModelHelper, Base):
__tablename__ = 'channel'

View file

@ -1,3 +1,5 @@
from datetime import datetime, timezone
import pytest
from modules.videos.models import Video
@ -87,3 +89,106 @@ async def test_video_channel_refresh(test_session, test_directory, channel_facto
video_channel = video1.__json__()['video']['channel']
assert video_channel['id'] == channel.id
assert video_channel['name'] == channel.name
@pytest.mark.asyncio
async def test_delete_duplicate_video(test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, and everything about the files matches, delete a random one."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
entry = channel.info_json['entries'][0]
tag = tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
tag_names=[tag.name, ]
)
vid2 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed.
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
)
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
assert test_session.query(Video).count() == 2
test_session.commit()
assert await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path == video_path, 'Video path does not match'
assert video.video_path.is_file()
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
assert video.file_group.tag_names == [tag.name, ]
# Running again has no effect.
assert not await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path.is_file()
assert video.video_path == video_path, 'Video path does not match'
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
assert video.file_group.tag_names == [tag.name, ]
@pytest.mark.asyncio
async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory):
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
channel = channel_factory(name='Channel Name')
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]}
entry = channel.info_json['entries'][0]
tag = tag_factory()
vid1 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title',
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
tag_names=[tag.name, ]
)
vid2 = video_factory(
channel_id=channel.id,
title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed.
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
source_id='ABC123456',
with_video_file=True,
with_info_json=True,
with_caption_file=True,
)
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
assert test_session.query(Video).count() == 2
test_session.commit()
await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
test_session.commit()
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
video = test_session.query(Video).one()
assert video.video_path == video_path, 'Video path does not match'
assert video.video_path.is_file()
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
assert video.file_group.tag_names == [tag.name, ]

View file

@ -813,6 +813,7 @@ def get_files_and_directories(directory: Path):
# These characters are invalid in Windows or Linux.
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%!\n\r]')
SPACE_FILE_CHARS = re.compile(r'( +)|(\t+)')