diff --git a/help.sh b/help.sh index c0e582f0..976968e5 100755 --- a/help.sh +++ b/help.sh @@ -351,8 +351,8 @@ else echo "FAILED: Cannot ping 1.1.1.1. Is internet working?" fi -if ping -c1 wrolpi.org -W 5 &>/dev/null; then - echo "OK: Can ping wrolpi.org" +if ping -c1 one.one.one.one -W 5 &>/dev/null; then + echo "OK: Can ping one.one.one.one" else - echo "FAILED: Cannot ping wrolpi.org Is internet working?" + echo "FAILED: Cannot ping one.one.one.one Is internet working?" fi diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index 76d54b0d..95346e89 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -407,6 +407,9 @@ class VideoDownloader(Downloader, ABC): video.source_id = entry['id'] video.channel_id = channel_id video_id, video_info_json_path = video.id, video.info_json_path + if video_info_json_path: + if new_info_json := video.clean_info_json(): + video.replace_info_json(new_info_json) session.commit() if video_info_json_path: diff --git a/modules/videos/models.py b/modules/videos/models.py index ac061adf..e03661b3 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -16,7 +16,7 @@ from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix from wrolpi.files.models import FileGroup from wrolpi.media_path import MediaPathType from wrolpi.tags import Tag, TagFile -from wrolpi.vars import PYTEST +from wrolpi.vars import PYTEST, DELETED_VIDEO_KEYS logger = logger.getChild(__name__) @@ -483,12 +483,21 @@ class Video(ModelHelper, Base): return f'/videos/channel/{self.channel_id}/video/{self.id}' return f'/videos/video/{self.id}' - def get_comments(self): + def get_comments(self) -> list | None: return (self.get_info_json() or dict()).get('comments') + def clean_info_json(self, info_json_contents: dict = None) -> dict | None: + """Remove large and mostly useless data in the info_json.""" + info_json_contents = info_json_contents or self.get_info_json() + if info_json_contents: + for key in DELETED_VIDEO_KEYS: + info_json_contents.pop(key, None) + return info_json_contents + def replace_info_json(self, info_json: dict): """Replace the info json file with the new json dict. Handles adding new info_json file, if necessary.""" info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json') + info_json = self.clean_info_json(info_json) info_json = json.dumps(info_json, indent=2) replace_file(info_json_path, info_json, missing_ok=True) diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index e284c0f2..863835ad 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,4 +1,5 @@ import asyncio +import json import shutil from copy import copy from unittest import mock @@ -255,6 +256,9 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i # Create a poster file which was downloaded. poster_path = video_path.with_suffix('.png') image_file.rename(poster_path) + # `formats` should be cleaned from info json. + info_json_path = video_path.with_suffix('.info.json') + info_json_path.write_text(json.dumps({'duration': 5, 'formats': [1, 2, 3]})) url = 'https://www.youtube.com/watch?v=31jPEBiAC3c' with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename: @@ -274,6 +278,10 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i video: Video = test_session.query(Video).one() assert video.video_path.is_absolute() and video.video_path == video_path, 'Video path is not absolute' assert video.video_path.is_absolute() and video.poster_path == poster_path, 'Video poster was not discovered' + # Some info json data is deleted during download. + assert video.get_info_json(), 'Download should keep info json file' + assert 'duration' in video.get_info_json(), 'Duration should stay in info json' + assert 'formats' not in video.get_info_json(), 'formats should be deleted from info json' @pytest.mark.asyncio diff --git a/wrolpi/vars.py b/wrolpi/vars.py index 328d2204..291fcd1c 100644 --- a/wrolpi/vars.py +++ b/wrolpi/vars.py @@ -3,6 +3,7 @@ import os import pathlib import sys from pathlib import Path +from typing import List from dotenv import load_dotenv @@ -39,6 +40,10 @@ def truthy_arg(value: str) -> bool: return v == 'true' or v == 't' or v == '1' or v == 'yes' or v == 'y' +def list_var(name: str, default='') -> List[str]: + return [i for i in os.environ.get(name, default).split(',')] + + # Special environment variable set in the docker/api/Dockerfile. DOCKERIZED = truthy_arg(os.environ.get('DOCKER', '')) # tests are running @@ -83,6 +88,7 @@ FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000)) VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 80)) YTDLP_CACHE_DIR = os.environ.get('YTDLP_CACHE_DIR', '/tmp/ytdlp_cache') +DELETED_VIDEO_KEYS = list_var('DELETED_VIDEO_KEYS', 'automatic_captions,formats') USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.366'