mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Duplicate videos are deleted when a video is downloaded.
This commit is contained in:
parent
26ddd27a77
commit
0d6edc05d0
4 changed files with 223 additions and 21 deletions
|
|
@ -207,18 +207,17 @@ class VideoDownloader(Downloader, ABC):
|
|||
raise UnrecoverableDownloadError('Max download attempts reached')
|
||||
|
||||
url = normalize_video_url(download.url)
|
||||
info = download.info_json or extract_info(url)
|
||||
download.info_json = download.info_json or extract_info(url)
|
||||
|
||||
if not info:
|
||||
if not download.info_json:
|
||||
raise ValueError(f'Cannot download video with no info_json.')
|
||||
|
||||
found_channel = None
|
||||
|
||||
channel_name = info.get('channel')
|
||||
source_channel_id = info.get('channel_id')
|
||||
channel_url = info.get('channel_url')
|
||||
channel_name = download.info_json.get('channel')
|
||||
source_channel_id = download.info_json.get('channel_id')
|
||||
channel_url = download.info_json.get('channel_url')
|
||||
channel = None
|
||||
logger.debug(f'{channel_name=} {source_channel_id=} {channel_url=}')
|
||||
if channel_name or source_channel_id:
|
||||
# Try to find the channel via info_json from yt-dlp.
|
||||
try:
|
||||
|
|
@ -284,11 +283,12 @@ class VideoDownloader(Downloader, ABC):
|
|||
|
||||
logs = None # noqa
|
||||
try:
|
||||
video_path, entry = self.prepare_filename(url, out_dir)
|
||||
|
||||
if settings.get('do_not_download'):
|
||||
# User has requested refresh of video metadata, skip the rest of the video downloader.
|
||||
return await self.download_info_json(download)
|
||||
return await self.download_info_json(download, video_path)
|
||||
|
||||
video_path, entry = self.prepare_filename(url, out_dir)
|
||||
# Do the real download.
|
||||
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
||||
cmd = (
|
||||
|
|
@ -354,18 +354,24 @@ class VideoDownloader(Downloader, ABC):
|
|||
video = Video.from_paths(session, *video_paths)
|
||||
video.source_id = entry['id']
|
||||
video.channel_id = channel_id
|
||||
video_id = video.id
|
||||
video_id, video_info_json_path = video.id, video.info_json_path
|
||||
|
||||
if video.info_json_path:
|
||||
format_json_file(video.info_json_path)
|
||||
|
||||
session.commit()
|
||||
if video_info_json_path:
|
||||
format_json_file(video_info_json_path)
|
||||
|
||||
with get_db_session() as session:
|
||||
# Second session is started because SQLAlchemy will forget what we have done.
|
||||
need_commit = False
|
||||
video = Video.get_by_id(video_id, session)
|
||||
if download.settings and (tag_names := download.settings.get('tag_names')):
|
||||
existing_names = video.file_group.tag_names
|
||||
for name in tag_names:
|
||||
if name not in existing_names:
|
||||
video.add_tag(name)
|
||||
need_commit = True
|
||||
|
||||
if need_commit:
|
||||
session.commit()
|
||||
|
||||
# Check that video has both audio and video streams.
|
||||
await video.get_ffprobe_json()
|
||||
|
|
@ -379,7 +385,9 @@ class VideoDownloader(Downloader, ABC):
|
|||
success=False,
|
||||
error='Video was downloaded but did not contain audio stream',
|
||||
)
|
||||
session.commit()
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path)
|
||||
|
||||
except UnrecoverableDownloadError:
|
||||
raise
|
||||
|
|
@ -390,7 +398,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
if _skip_download(e):
|
||||
# The video failed to download, and the error will never be fixed. Skip it forever.
|
||||
try:
|
||||
source_id = info.get('id')
|
||||
source_id = download.info_json.get('id')
|
||||
logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
|
||||
f'attempt to download it again.')
|
||||
download.add_to_skip_list()
|
||||
|
|
@ -446,12 +454,19 @@ class VideoDownloader(Downloader, ABC):
|
|||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
return final_filename, entry
|
||||
|
||||
async def download_info_json(self, download: Download) -> DownloadResult:
|
||||
async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult:
|
||||
"""User has requested to refresh Video metadata. Download only metadata files to temporary directory.
|
||||
Replace existing metadata files only after successful download."""
|
||||
url = download.url
|
||||
source_id = download.info_json['id']
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
with get_db_session() as session:
|
||||
# Delete any duplicate videos before attempting to get info json.
|
||||
deleted = await Video.delete_duplicate_videos(session, download.url, source_id, video_path)
|
||||
if deleted:
|
||||
session.commit()
|
||||
|
||||
with get_db_session() as session:
|
||||
# Find the existing video, replace its info json.
|
||||
video = Video.get_by_url(url, session)
|
||||
|
||||
|
|
|
|||
|
|
@ -239,10 +239,11 @@ class Video(ModelHelper, Base):
|
|||
video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none()
|
||||
if not video:
|
||||
video = Video(file_group=file_group)
|
||||
|
||||
video.validate()
|
||||
session.add(video)
|
||||
session.flush([video, file_group])
|
||||
|
||||
video.validate()
|
||||
session.flush([video, ])
|
||||
return video
|
||||
|
||||
@property
|
||||
|
|
@ -333,9 +334,9 @@ class Video(ModelHelper, Base):
|
|||
raise UnknownVideo(f'Cannot find Video with id {id_}')
|
||||
return video
|
||||
|
||||
def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile:
|
||||
def add_tag(self, tag_or_tag_name: Union[Tag, str], session: Session = None) -> TagFile:
|
||||
tag = Tag.find_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name
|
||||
return self.file_group.add_tag(tag)
|
||||
return self.file_group.add_tag(tag, session=session)
|
||||
|
||||
async def get_ffprobe_json(self) -> dict:
|
||||
"""Return the ffprobe json object if previously stored.
|
||||
|
|
@ -386,6 +387,86 @@ class Video(ModelHelper, Base):
|
|||
if self.channel and self.source_id:
|
||||
return self.channel.get_video_entry_by_id(self.source_id)
|
||||
|
||||
@staticmethod
|
||||
async def delete_duplicate_videos(session: Session, url: str, source_id: str, video_path: pathlib.Path) -> bool:
|
||||
"""Searches for Videos that share a `source_id` or `url`. Attempts to keep the best video at `video_path`, then
|
||||
deletes the rest."""
|
||||
# Get all Videos that share the URL or the source_id.
|
||||
matching_videos_with_url = session.query(Video) \
|
||||
.join(FileGroup, FileGroup.id == Video.file_group_id) \
|
||||
.filter(FileGroup.url == url) \
|
||||
.all()
|
||||
matching_videos_with_source_id = session.query(Video) \
|
||||
.join(FileGroup, FileGroup.id == Video.file_group_id) \
|
||||
.filter(FileGroup.model == 'video', Video.source_id == source_id) \
|
||||
.all()
|
||||
duplicate_videos = list()
|
||||
for video in matching_videos_with_url:
|
||||
if video.id not in [i.id for i in duplicate_videos]:
|
||||
duplicate_videos.append(video)
|
||||
for video in matching_videos_with_source_id:
|
||||
if video.id not in [i.id for i in duplicate_videos]:
|
||||
duplicate_videos.append(video)
|
||||
|
||||
# All the tags of all the files.
|
||||
tag_names = {i for j in duplicate_videos for i in j.file_group.tag_names}
|
||||
|
||||
def delete_video(video_: Video):
|
||||
"""Delete a Video with any of its tags."""
|
||||
for tag_name_ in video_.file_group.tag_names:
|
||||
tag_ = Tag.find_by_name(tag_name_)
|
||||
video_.file_group.remove_tag(tag_, session)
|
||||
video_.delete()
|
||||
|
||||
changes = False
|
||||
# Delete any Videos without real files.
|
||||
missing_file_indexes = []
|
||||
for idx, video in enumerate(duplicate_videos):
|
||||
if not video.video_path.is_file():
|
||||
logger.debug(f'Deleting {video} because the video file does not exist')
|
||||
missing_file_indexes.append(idx)
|
||||
delete_video(video)
|
||||
changes = True
|
||||
duplicate_videos = [i for idx, i in enumerate(duplicate_videos) if idx not in missing_file_indexes]
|
||||
|
||||
if len(duplicate_videos) < 2:
|
||||
# No duplicates.
|
||||
logger.debug(f'No duplicates to delete: {url=} {source_id=}')
|
||||
return changes
|
||||
|
||||
# Order videos by oldest first, we should keep the oldest if all files are the same size.
|
||||
duplicate_videos = sorted(duplicate_videos, key=lambda i: i.file_group.primary_path.stat().st_mtime)
|
||||
|
||||
# Find the largest video, if possible. If all videos are the same size then rename the oldest.
|
||||
largest_video_idx = 0
|
||||
for idx, video in enumerate(duplicate_videos):
|
||||
if video.file_group.size > duplicate_videos[largest_video_idx].file_group.size:
|
||||
largest_video_idx = idx
|
||||
largest_video = duplicate_videos.pop(largest_video_idx)
|
||||
|
||||
# Delete the video at the destination, if it is not the largest.
|
||||
existing_video = Video.get_by_path(video_path, session=session)
|
||||
if existing_video and largest_video != existing_video:
|
||||
delete_video(existing_video)
|
||||
for idx, video in enumerate(duplicate_videos):
|
||||
if video == existing_video:
|
||||
duplicate_videos.pop(idx)
|
||||
|
||||
# Rename the video, add any missing tag names.
|
||||
if largest_video.video_path != video_path:
|
||||
largest_video.file_group.move(video_path)
|
||||
for tag_name in tag_names:
|
||||
if tag_name not in largest_video.file_group.tag_names:
|
||||
largest_video.add_tag(tag_name, session)
|
||||
|
||||
logger.warning(f'Deleting duplicate videos: {duplicate_videos}')
|
||||
|
||||
# Remove tags from all videos, delete all videos.
|
||||
for video in duplicate_videos:
|
||||
delete_video(video)
|
||||
|
||||
return True
|
||||
|
||||
|
||||
class Channel(ModelHelper, Base):
|
||||
__tablename__ = 'channel'
|
||||
|
|
|
|||
|
|
@ -1,3 +1,5 @@
|
|||
from datetime import datetime, timezone
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.videos.models import Video
|
||||
|
|
@ -87,3 +89,106 @@ async def test_video_channel_refresh(test_session, test_directory, channel_facto
|
|||
video_channel = video1.__json__()['video']['channel']
|
||||
assert video_channel['id'] == channel.id
|
||||
assert video_channel['name'] == channel.name
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_duplicate_video(test_session, channel_factory, video_factory, tag_factory):
|
||||
"""If duplicate Video's exist, and everything about the files matches, delete a random one."""
|
||||
channel = channel_factory(name='Channel Name')
|
||||
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4'
|
||||
|
||||
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]}
|
||||
entry = channel.info_json['entries'][0]
|
||||
|
||||
tag = tag_factory()
|
||||
vid1 = video_factory(
|
||||
channel_id=channel.id,
|
||||
title=f'{channel.name}_20000101_ABC123456_The video title',
|
||||
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
|
||||
source_id='ABC123456',
|
||||
with_video_file=True,
|
||||
with_info_json=True,
|
||||
with_caption_file=True,
|
||||
tag_names=[tag.name, ]
|
||||
)
|
||||
vid2 = video_factory(
|
||||
channel_id=channel.id,
|
||||
title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed.
|
||||
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
|
||||
source_id='ABC123456',
|
||||
with_video_file=True,
|
||||
with_info_json=True,
|
||||
with_caption_file=True,
|
||||
)
|
||||
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
|
||||
|
||||
assert test_session.query(Video).count() == 2
|
||||
|
||||
test_session.commit()
|
||||
|
||||
assert await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
|
||||
test_session.commit()
|
||||
|
||||
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
|
||||
video = test_session.query(Video).one()
|
||||
assert video.video_path == video_path, 'Video path does not match'
|
||||
assert video.video_path.is_file()
|
||||
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
|
||||
assert video.file_group.tag_names == [tag.name, ]
|
||||
|
||||
# Running again has no effect.
|
||||
assert not await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
|
||||
test_session.commit()
|
||||
|
||||
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
|
||||
video = test_session.query(Video).one()
|
||||
assert video.video_path.is_file()
|
||||
assert video.video_path == video_path, 'Video path does not match'
|
||||
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
|
||||
assert video.file_group.tag_names == [tag.name, ]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory):
|
||||
"""If duplicate Video's exist, delete all the videos that do not have the new title."""
|
||||
channel = channel_factory(name='Channel Name')
|
||||
video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4'
|
||||
|
||||
channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]}
|
||||
entry = channel.info_json['entries'][0]
|
||||
|
||||
tag = tag_factory()
|
||||
vid1 = video_factory(
|
||||
channel_id=channel.id,
|
||||
title=f'{channel.name}_20000101_ABC123456_The video title',
|
||||
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
|
||||
source_id='ABC123456',
|
||||
with_video_file=True,
|
||||
with_info_json=True,
|
||||
with_caption_file=True,
|
||||
tag_names=[tag.name, ]
|
||||
)
|
||||
vid2 = video_factory(
|
||||
channel_id=channel.id,
|
||||
title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed.
|
||||
upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc),
|
||||
source_id='ABC123456',
|
||||
with_video_file=True,
|
||||
with_info_json=True,
|
||||
with_caption_file=True,
|
||||
)
|
||||
vid1.file_group.url = vid2.file_group.url = 'https://example.com/video'
|
||||
|
||||
assert test_session.query(Video).count() == 2
|
||||
|
||||
test_session.commit()
|
||||
|
||||
await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path)
|
||||
test_session.commit()
|
||||
|
||||
assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.'
|
||||
video = test_session.query(Video).one()
|
||||
assert video.video_path == video_path, 'Video path does not match'
|
||||
assert video.video_path.is_file()
|
||||
assert set(channel.directory.iterdir()) == set(video.file_group.my_paths())
|
||||
assert video.file_group.tag_names == [tag.name, ]
|
||||
|
|
|
|||
|
|
@ -813,6 +813,7 @@ def get_files_and_directories(directory: Path):
|
|||
|
||||
# These characters are invalid in Windows or Linux.
|
||||
INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%!\n\r]')
|
||||
|
||||
SPACE_FILE_CHARS = re.compile(r'( +)|(\t+)')
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue