From 61abff1c5ecf1f29c97261ec5f0aca4700e71a0e Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 4 Jan 2022 14:30:45 -0700 Subject: [PATCH] Video titles have HTML entities replaced. --- modules/videos/common.py | 12 ++++++++++- modules/videos/downloader.py | 2 +- modules/videos/lib.py | 6 +++++- modules/videos/test/test_common.py | 32 +++++++++++++++++++++++++++++- requirements.txt | 1 + 5 files changed, 49 insertions(+), 4 deletions(-) diff --git a/modules/videos/common.py b/modules/videos/common.py index 645d8f08..ef34c144 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -1,3 +1,4 @@ +import html import json import os import pathlib @@ -453,9 +454,18 @@ async def get_bulk_video_info_json(video_ids: List[int]): if not video.duration: # Get duration from info_json without reading video file. video.duration = info_json.get('duration') - elif not video.duration: + # Trust the info_json title before the video path title. + video.title = info_json.get('title', video.title) + if not video.duration: # As a last resort, get duration from the video file. video.duration = get_video_duration(video.video_path.path) + if not video.title: + # Use the video path name without the extension. + video_path = video.video_path.path.with_suffix('') + video.title = video_path.name + + # Unescape any HTML entities. + video.title = html.unescape(video.title) except Exception: logger.warning(f'Unable to get meta data of {video}', exc_info=True) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 9d846df6..9e23d196 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -96,7 +96,7 @@ class ChannelDownloader(Downloader, ABC): class VideoDownloader(Downloader, ABC): """ - Download a single video. Store the video in it's channel's directory, otherwise store it in `videos/NO CHANNEL`. + Download a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`. """ name = 'video' pretty_name = 'Videos' diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 94d1484e..f2b76a78 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -1,4 +1,5 @@ import asyncio +import html import json import pathlib import re @@ -89,7 +90,7 @@ def refresh_channel_info_json() -> bool: v.video_path IS NOT NULL AND v.info_json_path IS NOT NULL AND v.info_json_path != '' - AND (v.duration IS NULL OR v.view_count IS NULL) + AND (v.duration IS NULL OR v.view_count IS NULL OR v.title IS NULL) ''' curs.execute(query) missing_duration = [i for (i,) in curs.fetchall()] @@ -377,10 +378,13 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = json_contents = json.load(fh) duration = json_contents.get('duration') url = json_contents.get('webpage_url') + # Trust the info_json title before the video filename. + title = json_contents.get('title', title) except json.decoder.JSONDecodeError: logger.warning(f'Failed to load JSON file to get duration: {info_json_path}') size = video_path.stat().st_size + title = html.unescape(title) if title else None video_dict = dict( caption_path=str(caption_path) if caption_path else None, diff --git a/modules/videos/test/test_common.py b/modules/videos/test/test_common.py index 17633763..2a143ec6 100644 --- a/modules/videos/test/test_common.py +++ b/modules/videos/test/test_common.py @@ -1,3 +1,4 @@ +import json import pathlib import subprocess import tempfile @@ -14,7 +15,8 @@ from wrolpi.db import get_db_session from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI from wrolpi.vars import PROJECT_DIR from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \ - apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster + apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \ + get_bulk_video_info_json class TestCommon(TestAPI): @@ -375,3 +377,31 @@ def test_update_censored_videos(test_session, video_factory, simple_channel): test_session.commit() apply_info_json(simple_channel.id) check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)]) + + +@pytest.mark.asyncio +async def test_missing_title(test_session, simple_video): + """ + A Video title will be filled out by `get_bulk_video_info_json`. + """ + video_path: pathlib.Path = simple_video.video_path.path + simple_video.duration = 10 # don't update the duration during get_bulk_video_info_json() + + info_json_path = video_path.with_suffix('.info.json') + info_json_path.write_text(json.dumps({'title': 'info json title &'})) + simple_video.info_json_path = info_json_path + test_session.commit() + + # simple_video does not come with a title. + assert not simple_video.title + + # Title is fetched from the info_json. + await get_bulk_video_info_json([simple_video.id]) + assert simple_video.title == 'info json title &' + + # No info_json means the filename is used. + simple_video.title = None + simple_video.info_json_path = None + test_session.commit() + await get_bulk_video_info_json([simple_video.id]) + assert simple_video.title == 'simple_video' diff --git a/requirements.txt b/requirements.txt index 8b983ca4..b07e20c4 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,6 +5,7 @@ pdfkit==0.6.1 Pillow==8.3.2 Pint==0.16 psycopg2-binary==2.8.6 +pytest-asyncio==0.16.0 pytz==2021.1 PyYAML==5.4 requests==2.26.0