Cleaning up some extra data from video info json.

help.sh

@@ -351,8 +351,8 @@ else
   echo "FAILED: Cannot ping 1.1.1.1.  Is internet working?"
 fi
 
-if ping -c1 wrolpi.org -W 5 &>/dev/null; then
-  echo "OK: Can ping wrolpi.org"
+if ping -c1 one.one.one.one -W 5 &>/dev/null; then
+  echo "OK: Can ping one.one.one.one"
 else
-  echo "FAILED: Cannot ping wrolpi.org  Is internet working?"
+  echo "FAILED: Cannot ping one.one.one.one  Is internet working?"
 fi

modules/videos/downloader.py

@@ -407,6 +407,9 @@ class VideoDownloader(Downloader, ABC):
                 video.source_id = entry['id']
                 video.channel_id = channel_id
                 video_id, video_info_json_path = video.id, video.info_json_path
+                if video_info_json_path:
+                    if new_info_json := video.clean_info_json():
+                        video.replace_info_json(new_info_json)
                 session.commit()
 
             if video_info_json_path:

modules/videos/models.py

@@ -16,7 +16,7 @@ from wrolpi.files.lib import refresh_files, split_path_stem_and_suffix
 from wrolpi.files.models import FileGroup
 from wrolpi.media_path import MediaPathType
 from wrolpi.tags import Tag, TagFile
-from wrolpi.vars import PYTEST
+from wrolpi.vars import PYTEST, DELETED_VIDEO_KEYS
 
 logger = logger.getChild(__name__)
 
@@ -483,12 +483,21 @@ class Video(ModelHelper, Base):
             return f'/videos/channel/{self.channel_id}/video/{self.id}'
         return f'/videos/video/{self.id}'
 
-    def get_comments(self):
+    def get_comments(self) -> list | None:
         return (self.get_info_json() or dict()).get('comments')
 
+    def clean_info_json(self, info_json_contents: dict = None) -> dict | None:
+        """Remove large and mostly useless data in the info_json."""
+        info_json_contents = info_json_contents or self.get_info_json()
+        if info_json_contents:
+            for key in DELETED_VIDEO_KEYS:
+                info_json_contents.pop(key, None)
+        return info_json_contents
+
     def replace_info_json(self, info_json: dict):
         """Replace the info json file with the new json dict.  Handles adding new info_json file, if necessary."""
         info_json_path = self.info_json_path or self.video_path.with_suffix('.info.json')
+        info_json = self.clean_info_json(info_json)
         info_json = json.dumps(info_json, indent=2)
         replace_file(info_json_path, info_json, missing_ok=True)
 

modules/videos/test/test_downloader.py

@@ -1,4 +1,5 @@
 import asyncio
+import json
 import shutil
 from copy import copy
 from unittest import mock
@@ -255,6 +256,9 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i
     # Create a poster file which was downloaded.
     poster_path = video_path.with_suffix('.png')
     image_file.rename(poster_path)
+    # `formats` should be cleaned from info json.
+    info_json_path = video_path.with_suffix('.info.json')
+    info_json_path.write_text(json.dumps({'duration': 5, 'formats': [1, 2, 3]}))
 
     url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
     with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
@@ -274,6 +278,10 @@ async def test_video_download(test_session, test_directory, mock_video_extract_i
     video: Video = test_session.query(Video).one()
     assert video.video_path.is_absolute() and video.video_path == video_path, 'Video path is not absolute'
     assert video.video_path.is_absolute() and video.poster_path == poster_path, 'Video poster was not discovered'
+    # Some info json data is deleted during download.
+    assert video.get_info_json(), 'Download should keep info json file'
+    assert 'duration' in video.get_info_json(), 'Duration should stay in info json'
+    assert 'formats' not in video.get_info_json(), 'formats should be deleted from info json'
 
 
 @pytest.mark.asyncio

wrolpi/vars.py

@@ -3,6 +3,7 @@ import os
 import pathlib
 import sys
 from pathlib import Path
+from typing import List
 
 from dotenv import load_dotenv
 
@@ -39,6 +40,10 @@ def truthy_arg(value: str) -> bool:
     return v == 'true' or v == 't' or v == '1' or v == 'yes' or v == 'y'
 
 
+def list_var(name: str, default='') -> List[str]:
+    return [i for i in os.environ.get(name, default).split(',')]
+
+
 # Special environment variable set in the docker/api/Dockerfile.
 DOCKERIZED = truthy_arg(os.environ.get('DOCKER', ''))
 # tests are running
@@ -83,6 +88,7 @@ FILE_MAX_TEXT_SIZE = int(os.environ.get('FILE_MAX_TEXT_SIZE', 100_000))
 
 VIDEO_COMMENTS_FETCH_COUNT = int(os.environ.get('VIDEO_COMMENTS_FETCH_COUNT', 80))
 YTDLP_CACHE_DIR = os.environ.get('YTDLP_CACHE_DIR', '/tmp/ytdlp_cache')
+DELETED_VIDEO_KEYS = list_var('DELETED_VIDEO_KEYS', 'automatic_captions,formats')
 
 USER_AGENT = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.366'