diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 8ea699b3..ecf016ef 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -207,18 +207,17 @@ class VideoDownloader(Downloader, ABC): raise UnrecoverableDownloadError('Max download attempts reached') url = normalize_video_url(download.url) - info = download.info_json or extract_info(url) + download.info_json = download.info_json or extract_info(url) - if not info: + if not download.info_json: raise ValueError(f'Cannot download video with no info_json.') found_channel = None - channel_name = info.get('channel') - source_channel_id = info.get('channel_id') - channel_url = info.get('channel_url') + channel_name = download.info_json.get('channel') + source_channel_id = download.info_json.get('channel_id') + channel_url = download.info_json.get('channel_url') channel = None - logger.debug(f'{channel_name=} {source_channel_id=} {channel_url=}') if channel_name or source_channel_id: # Try to find the channel via info_json from yt-dlp. try: @@ -284,11 +283,12 @@ class VideoDownloader(Downloader, ABC): logs = None # noqa try: + video_path, entry = self.prepare_filename(url, out_dir) + if settings.get('do_not_download'): # User has requested refresh of video metadata, skip the rest of the video downloader. - return await self.download_info_json(download) + return await self.download_info_json(download, video_path) - video_path, entry = self.prepare_filename(url, out_dir) # Do the real download. file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s' cmd = ( @@ -354,18 +354,24 @@ class VideoDownloader(Downloader, ABC): video = Video.from_paths(session, *video_paths) video.source_id = entry['id'] video.channel_id = channel_id - video_id = video.id + video_id, video_info_json_path = video.id, video.info_json_path - if video.info_json_path: - format_json_file(video.info_json_path) - - session.commit() + if video_info_json_path: + format_json_file(video_info_json_path) + with get_db_session() as session: + # Second session is started because SQLAlchemy will forget what we have done. + need_commit = False + video = Video.get_by_id(video_id, session) if download.settings and (tag_names := download.settings.get('tag_names')): existing_names = video.file_group.tag_names for name in tag_names: if name not in existing_names: video.add_tag(name) + need_commit = True + + if need_commit: + session.commit() # Check that video has both audio and video streams. await video.get_ffprobe_json() @@ -379,7 +385,9 @@ class VideoDownloader(Downloader, ABC): success=False, error='Video was downloaded but did not contain audio stream', ) - session.commit() + + with get_db_session(commit=True) as session: + await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path) except UnrecoverableDownloadError: raise @@ -390,7 +398,7 @@ class VideoDownloader(Downloader, ABC): if _skip_download(e): # The video failed to download, and the error will never be fixed. Skip it forever. try: - source_id = info.get('id') + source_id = download.info_json.get('id') logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not ' f'attempt to download it again.') download.add_to_skip_list() @@ -446,12 +454,19 @@ class VideoDownloader(Downloader, ABC): logger.debug(f'Downloading {url} to {out_dir}') return final_filename, entry - async def download_info_json(self, download: Download) -> DownloadResult: + async def download_info_json(self, download: Download, video_path: pathlib.Path) -> DownloadResult: """User has requested to refresh Video metadata. Download only metadata files to temporary directory. Replace existing metadata files only after successful download.""" url = download.url + source_id = download.info_json['id'] - with get_db_session(commit=True) as session: + with get_db_session() as session: + # Delete any duplicate videos before attempting to get info json. + deleted = await Video.delete_duplicate_videos(session, download.url, source_id, video_path) + if deleted: + session.commit() + + with get_db_session() as session: # Find the existing video, replace its info json. video = Video.get_by_url(url, session) diff --git a/modules/videos/models.py b/modules/videos/models.py index 370bc0a9..1b708d21 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -239,10 +239,11 @@ class Video(ModelHelper, Base): video = session.query(Video).filter(Video.file_group_id == file_group.id).one_or_none() if not video: video = Video(file_group=file_group) - - video.validate() session.add(video) session.flush([video, file_group]) + + video.validate() + session.flush([video, ]) return video @property @@ -333,9 +334,9 @@ class Video(ModelHelper, Base): raise UnknownVideo(f'Cannot find Video with id {id_}') return video - def add_tag(self, tag_or_tag_name: Union[Tag, str]) -> TagFile: + def add_tag(self, tag_or_tag_name: Union[Tag, str], session: Session = None) -> TagFile: tag = Tag.find_by_name(tag_or_tag_name) if isinstance(tag_or_tag_name, str) else tag_or_tag_name - return self.file_group.add_tag(tag) + return self.file_group.add_tag(tag, session=session) async def get_ffprobe_json(self) -> dict: """Return the ffprobe json object if previously stored. @@ -386,6 +387,86 @@ class Video(ModelHelper, Base): if self.channel and self.source_id: return self.channel.get_video_entry_by_id(self.source_id) + @staticmethod + async def delete_duplicate_videos(session: Session, url: str, source_id: str, video_path: pathlib.Path) -> bool: + """Searches for Videos that share a `source_id` or `url`. Attempts to keep the best video at `video_path`, then + deletes the rest.""" + # Get all Videos that share the URL or the source_id. + matching_videos_with_url = session.query(Video) \ + .join(FileGroup, FileGroup.id == Video.file_group_id) \ + .filter(FileGroup.url == url) \ + .all() + matching_videos_with_source_id = session.query(Video) \ + .join(FileGroup, FileGroup.id == Video.file_group_id) \ + .filter(FileGroup.model == 'video', Video.source_id == source_id) \ + .all() + duplicate_videos = list() + for video in matching_videos_with_url: + if video.id not in [i.id for i in duplicate_videos]: + duplicate_videos.append(video) + for video in matching_videos_with_source_id: + if video.id not in [i.id for i in duplicate_videos]: + duplicate_videos.append(video) + + # All the tags of all the files. + tag_names = {i for j in duplicate_videos for i in j.file_group.tag_names} + + def delete_video(video_: Video): + """Delete a Video with any of its tags.""" + for tag_name_ in video_.file_group.tag_names: + tag_ = Tag.find_by_name(tag_name_) + video_.file_group.remove_tag(tag_, session) + video_.delete() + + changes = False + # Delete any Videos without real files. + missing_file_indexes = [] + for idx, video in enumerate(duplicate_videos): + if not video.video_path.is_file(): + logger.debug(f'Deleting {video} because the video file does not exist') + missing_file_indexes.append(idx) + delete_video(video) + changes = True + duplicate_videos = [i for idx, i in enumerate(duplicate_videos) if idx not in missing_file_indexes] + + if len(duplicate_videos) < 2: + # No duplicates. + logger.debug(f'No duplicates to delete: {url=} {source_id=}') + return changes + + # Order videos by oldest first, we should keep the oldest if all files are the same size. + duplicate_videos = sorted(duplicate_videos, key=lambda i: i.file_group.primary_path.stat().st_mtime) + + # Find the largest video, if possible. If all videos are the same size then rename the oldest. + largest_video_idx = 0 + for idx, video in enumerate(duplicate_videos): + if video.file_group.size > duplicate_videos[largest_video_idx].file_group.size: + largest_video_idx = idx + largest_video = duplicate_videos.pop(largest_video_idx) + + # Delete the video at the destination, if it is not the largest. + existing_video = Video.get_by_path(video_path, session=session) + if existing_video and largest_video != existing_video: + delete_video(existing_video) + for idx, video in enumerate(duplicate_videos): + if video == existing_video: + duplicate_videos.pop(idx) + + # Rename the video, add any missing tag names. + if largest_video.video_path != video_path: + largest_video.file_group.move(video_path) + for tag_name in tag_names: + if tag_name not in largest_video.file_group.tag_names: + largest_video.add_tag(tag_name, session) + + logger.warning(f'Deleting duplicate videos: {duplicate_videos}') + + # Remove tags from all videos, delete all videos. + for video in duplicate_videos: + delete_video(video) + + return True + class Channel(ModelHelper, Base): __tablename__ = 'channel' diff --git a/modules/videos/test/test_video.py b/modules/videos/test/test_video.py index 545cd974..bcdc9704 100644 --- a/modules/videos/test/test_video.py +++ b/modules/videos/test/test_video.py @@ -1,3 +1,5 @@ +from datetime import datetime, timezone + import pytest from modules.videos.models import Video @@ -87,3 +89,106 @@ async def test_video_channel_refresh(test_session, test_directory, channel_facto video_channel = video1.__json__()['video']['channel'] assert video_channel['id'] == channel.id assert video_channel['name'] == channel.name + + +@pytest.mark.asyncio +async def test_delete_duplicate_video(test_session, channel_factory, video_factory, tag_factory): + """If duplicate Video's exist, and everything about the files matches, delete a random one.""" + channel = channel_factory(name='Channel Name') + video_path = channel.directory / f'{channel.name}_20000101_ABC123456_The video title.mp4' + + channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'The video title'}]} + entry = channel.info_json['entries'][0] + + tag = tag_factory() + vid1 = video_factory( + channel_id=channel.id, + title=f'{channel.name}_20000101_ABC123456_The video title', + upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), + source_id='ABC123456', + with_video_file=True, + with_info_json=True, + with_caption_file=True, + tag_names=[tag.name, ] + ) + vid2 = video_factory( + channel_id=channel.id, + title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed. + upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), + source_id='ABC123456', + with_video_file=True, + with_info_json=True, + with_caption_file=True, + ) + vid1.file_group.url = vid2.file_group.url = 'https://example.com/video' + + assert test_session.query(Video).count() == 2 + + test_session.commit() + + assert await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path) + test_session.commit() + + assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.' + video = test_session.query(Video).one() + assert video.video_path == video_path, 'Video path does not match' + assert video.video_path.is_file() + assert set(channel.directory.iterdir()) == set(video.file_group.my_paths()) + assert video.file_group.tag_names == [tag.name, ] + + # Running again has no effect. + assert not await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path) + test_session.commit() + + assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.' + video = test_session.query(Video).one() + assert video.video_path.is_file() + assert video.video_path == video_path, 'Video path does not match' + assert set(channel.directory.iterdir()) == set(video.file_group.my_paths()) + assert video.file_group.tag_names == [tag.name, ] + + +@pytest.mark.asyncio +async def test_delete_renamed_video(test_session, channel_factory, video_factory, tag_factory): + """If duplicate Video's exist, delete all the videos that do not have the new title.""" + channel = channel_factory(name='Channel Name') + video_path = channel.directory / f'{channel.name}_20000101_ABC123456_Some new title.mp4' + + channel.info_json = {'entries': [{'id': 'ABC123456', 'title': 'Some new title'}]} + entry = channel.info_json['entries'][0] + + tag = tag_factory() + vid1 = video_factory( + channel_id=channel.id, + title=f'{channel.name}_20000101_ABC123456_The video title', + upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), + source_id='ABC123456', + with_video_file=True, + with_info_json=True, + with_caption_file=True, + tag_names=[tag.name, ] + ) + vid2 = video_factory( + channel_id=channel.id, + title=f'{channel.name}_20000101_ABC123456_The video title ', # Video was renamed, trailing space was removed. + upload_date=datetime(2000, 1, 1, tzinfo=timezone.utc), + source_id='ABC123456', + with_video_file=True, + with_info_json=True, + with_caption_file=True, + ) + vid1.file_group.url = vid2.file_group.url = 'https://example.com/video' + + assert test_session.query(Video).count() == 2 + + test_session.commit() + + await Video.delete_duplicate_videos(test_session, 'https://example.com/video', entry['id'], video_path) + test_session.commit() + + assert test_session.query(Video).count() == 1, 'Duplicate video was not deleted.' + video = test_session.query(Video).one() + assert video.video_path == video_path, 'Video path does not match' + assert video.video_path.is_file() + assert set(channel.directory.iterdir()) == set(video.file_group.my_paths()) + assert video.file_group.tag_names == [tag.name, ] diff --git a/wrolpi/common.py b/wrolpi/common.py index 7aed499a..0734b750 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -813,6 +813,7 @@ def get_files_and_directories(directory: Path): # These characters are invalid in Windows or Linux. INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*%!\n\r]') + SPACE_FILE_CHARS = re.compile(r'( +)|(\t+)')