mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Simplifying scripts to find similarlly named videos.
This commit is contained in:
parent
e1621fb57b
commit
08f8454540
1 changed files with 41 additions and 16 deletions
|
|
@ -10,9 +10,10 @@ import difflib
|
|||
import functools
|
||||
import re
|
||||
from difflib import SequenceMatcher
|
||||
from typing import List, Generator, Optional
|
||||
from typing import List, Generator, Optional, Dict
|
||||
|
||||
from modules.videos import Video, Channel
|
||||
from modules.videos.lib import parse_video_file_name
|
||||
from wrolpi.common import get_relative_to_media_directory, chain
|
||||
from wrolpi.db import get_db_curs, get_db_session
|
||||
from wrolpi.files.models import FileGroup
|
||||
|
|
@ -67,7 +68,7 @@ def find_similar_videos(channel_id: int, minimum_rank: float = MINIMUM_RANK, dir
|
|||
|
||||
vid1, *vids = link
|
||||
vid1_title = vid1.file_group.title
|
||||
similar_videos = [i for i in vids if similar(i.file_group.title, vid1_title) > minimum_rank]
|
||||
similar_videos = [i for i in vids if similar(i.file_group.title, vid1_title) >= minimum_rank]
|
||||
|
||||
if not similar_videos:
|
||||
continue
|
||||
|
|
@ -80,7 +81,7 @@ def find_similar_videos(channel_id: int, minimum_rank: float = MINIMUM_RANK, dir
|
|||
yield similar_videos
|
||||
|
||||
|
||||
async def rank_video_quality(video: Video) -> int:
|
||||
async def rank_video_quality(video: Video, sizes_by_source_id: Dict[str, int]) -> int:
|
||||
"""Returns a higher integer the more quality the Video record is. A video with info_json is more valuable than
|
||||
one without, etc."""
|
||||
rank = 0
|
||||
|
|
@ -90,18 +91,27 @@ async def rank_video_quality(video: Video) -> int:
|
|||
rank += 2
|
||||
if video.source_id:
|
||||
rank += 2
|
||||
if video.info_json_path:
|
||||
if video.info_json_path and video.info_json_path.is_file() and video.info_json_path.stat().st_size:
|
||||
rank += 2
|
||||
if video_file_matches_channel_info_json(video):
|
||||
rank += 2
|
||||
# These are more common.
|
||||
if video.channel_id:
|
||||
rank += 1
|
||||
if video.caption_paths:
|
||||
rank += 1
|
||||
if video.poster_path:
|
||||
if video.poster_path and video.poster_path.is_file() and video.poster_path.stat().st_size:
|
||||
rank += 1
|
||||
if video.ffprobe_json or await video.get_ffprobe_json():
|
||||
rank += 1
|
||||
if video_file_matches_channel_info_json(video):
|
||||
largest_size = sizes_by_source_id.get(video.source_id)
|
||||
if largest_size and video.file_group.size == largest_size:
|
||||
rank += 1
|
||||
|
||||
_, _, _, video_title = parse_video_file_name(video.video_path)
|
||||
video_title = video_title.strip()
|
||||
if video.file_group.title == video_title:
|
||||
# Video title from JSON matches the video file name.
|
||||
rank += 1
|
||||
|
||||
channel_name = get_channel_name(video.channel_id)
|
||||
|
|
@ -145,6 +155,16 @@ def remove_part_from_title(title: str) -> str:
|
|||
return title
|
||||
|
||||
|
||||
def largest_size_by_source_id(similar_videos: List[Video]) -> Dict[str, int]:
|
||||
"""Videos that are larger are probably better quality, prefer to preserve larger videos."""
|
||||
sizes = dict()
|
||||
for video in similar_videos:
|
||||
source_id = video.source_id
|
||||
old_size = sizes.get(source_id, 0)
|
||||
sizes[source_id] = max(old_size, video.file_group.size)
|
||||
return sizes
|
||||
|
||||
|
||||
async def handle_user_delete_duplicates(similar_videos: List[Video], video_url: str = None,
|
||||
delete_lower_ranked_automatically: bool = False):
|
||||
similar_videos = similar_videos.copy()
|
||||
|
|
@ -160,13 +180,15 @@ async def handle_user_delete_duplicates(similar_videos: List[Video], video_url:
|
|||
all_changes += get_different_characters(a, b)
|
||||
# Ignore space changes.
|
||||
all_changes = ''.join(i for i in all_changes if i != ' ' and i != '.')
|
||||
if all_changes.isdigit() or not all_changes:
|
||||
if all_changes.isdigit():
|
||||
print(f'\nSkipping {similar_videos[0]} and similar because they are parts')
|
||||
return
|
||||
|
||||
print()
|
||||
print(f'{all_changes=}')
|
||||
ranked_videos = [(await rank_video_quality(i), i) for i in similar_videos]
|
||||
# Preserve the largest duplicate video.
|
||||
sizes_by_source_id = largest_size_by_source_id(similar_videos)
|
||||
|
||||
print('\n')
|
||||
ranked_videos = [(await rank_video_quality(i, sizes_by_source_id), i) for i in similar_videos]
|
||||
ranked_videos = sorted(ranked_videos, key=lambda i: (i[0], i[1].file_group.size), reverse=True)
|
||||
for rank, video in ranked_videos:
|
||||
fg = video.file_group
|
||||
|
|
@ -174,9 +196,12 @@ async def handle_user_delete_duplicates(similar_videos: List[Video], video_url:
|
|||
size = f'{size // 1_048_576} MB'
|
||||
path = get_relative_to_media_directory(path)
|
||||
if video_url:
|
||||
print(f'{rank} {id_} {repr(title)} {repr(str(path))} {source_id} {size} {video_url + str(id_)}')
|
||||
print(f'{rank} {repr(title)} {size} {repr(str(path.name))} {source_id} {id_} {video_url + str(id_)}')
|
||||
else:
|
||||
print(f'{rank} {id_} {repr(title)} {repr(str(path))} {source_id} {size}')
|
||||
print(f'{rank} {repr(title)} {size} {repr(str(path.name))} {source_id} {id_}')
|
||||
|
||||
# Detect re-uploaded videos.
|
||||
# If video with exact title is uploaded day(s) later, then suggest deleting the oldest video.
|
||||
|
||||
unique_ranks = {i[0] for i in ranked_videos}
|
||||
if len(unique_ranks) > 1:
|
||||
|
|
@ -189,10 +214,10 @@ async def handle_user_delete_duplicates(similar_videos: List[Video], video_url:
|
|||
continue
|
||||
response = input('Delete the lowest ranked video? (y/N)').lower()
|
||||
if response == 'y':
|
||||
if input(f'\nConfirm delete: {lowest_video} ').lower() == 'y':
|
||||
delete_video(lowest_video.id)
|
||||
index = similar_videos.index(lowest_video)
|
||||
del similar_videos[index]
|
||||
# Don't need to confirm when the video has been ranked low.
|
||||
delete_video(lowest_video.id)
|
||||
index = similar_videos.index(lowest_video)
|
||||
del similar_videos[index]
|
||||
elif response == 'n':
|
||||
break
|
||||
elif response == 'c':
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue