Trimming video file name to avoid error when file name is too long.

modules/archive/lib.py

@@ -65,6 +65,7 @@ def get_domain_directory(url: str) -> pathlib.Path:
     return directory
 
 
+# File names include domain and datetime.
 MAXIMUM_ARCHIVE_FILE_CHARACTER_LENGTH = 200
 
 

modules/archive/test/test_lib.py

@@ -359,7 +359,7 @@ async def test_get_title_from_html(test_directory, test_session, fake_now):
 
 
 @skip_circleci
-def test_get_new_archive_files_length(fake_now):
+def test_get_new_archive_files_length(test_directory, fake_now):
     """Archive titles are truncated to fit file system length limit.  (255-character limit for most file systems)"""
     fake_now(datetime(2001, 1, 1))
     archive_files = get_new_archive_files('https://example.com', 'a' * 500)
@@ -372,7 +372,7 @@ def test_get_new_archive_files_length(fake_now):
 
 @skip_circleci
 def test_get_new_archive_files(fake_now):
-    """Archive files have a specific format so they are sorted by datetime, and are near each other."""
+    """Archive files have a specific format, so they are sorted by datetime, and are near each other."""
     fake_now(datetime(2001, 1, 1))
     archive_files = get_new_archive_files('https://example.com/two', None)
     assert str(archive_files.singlefile).endswith('archive/example.com/2001-01-01-00-00-00_NA.html')

modules/videos/downloader.py

@@ -1,5 +1,6 @@
 #! /usr/bin/env python3
 import json
+import os.path
 import pathlib
 import traceback
 from abc import ABC
@@ -12,7 +13,7 @@ from yt_dlp.extractor import YoutubeTabIE  # noqa
 
 from wrolpi.cmd import YT_DLP_BIN
 from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
-    format_json_file
+    format_json_file, trim_file_name
 from wrolpi.dates import now
 from wrolpi.db import get_db_session
 from wrolpi.db import optional_session
@@ -62,7 +63,9 @@ def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict:
 
 def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
     """Get filename from YoutubeDL.  Separated for testing."""
-    return ydl.prepare_filename(entry)
+    dir_name, file_name = os.path.split(ydl.prepare_filename(entry))
+    file_name = trim_file_name(file_name)
+    return f'{dir_name}/{file_name}'
 
 
 class ChannelDownloader(Downloader, ABC):
@@ -332,7 +335,6 @@ class VideoDownloader(Downloader, ABC):
                 return await self.download_info_json(download, video_path)
 
             # Do the real download.
-            file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
             cmd = (
                 str(YT_DLP_BIN),
                 '-c',  # Continue downloads
@@ -347,7 +349,7 @@ class VideoDownloader(Downloader, ABC):
                 '--write-info-json',
                 '--merge-output-format', PREFERRED_VIDEO_EXTENSION,
                 '--remux-video', PREFERRED_VIDEO_EXTENSION,
-                '-o', file_name_format,
+                '-o', video_path,
                 '--no-cache-dir',
                 '--compat-options', 'no-live-chat',
                 # Get top 20 comments, 10 replies per parent.
@@ -496,7 +498,7 @@ class VideoDownloader(Downloader, ABC):
         entry = extract_info(url, ydl=ydl, process=True)
         final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
 
-        logger.debug(f'Downloading {url} to {out_dir}')
+        logger.debug(f'Downloading {url} to {repr(str(final_filename))}')
         return final_filename, entry
 
     @staticmethod

modules/videos/lib.py

@@ -10,7 +10,7 @@ from sqlalchemy import or_
 from sqlalchemy.orm import Session
 from yt_dlp import YoutubeDL
 
-from wrolpi import before_startup, dates
+from wrolpi import before_startup, dates, flags
 from wrolpi.captions import extract_captions
 from wrolpi.common import ConfigFile, get_media_directory, register_refresh_cleanup, limit_concurrent
 from wrolpi.dates import Seconds, from_timestamp
@@ -427,14 +427,17 @@ def import_channels_config():
                 full_data.update(data)
                 channel.update(full_data)
 
-                if not channel.source_id and channel.url:
+                if not channel.source_id and channel.url and flags.have_internet.is_set():
                     # If we can download from a channel, we must have its source_id.
-                    channel.source_id = get_channel_source_id(channel.url)
-                    save_config = True
-                    if not channel.source_id:
-                        channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
-                    else:
-                        session.commit()
+                    try:
+                        channel.source_id = get_channel_source_id(channel.url)
+                        save_config = True
+                        if not channel.source_id:
+                            channel_import_logger.warning(f'Unable to fetch source_id for {channel.url}')
+                        else:
+                            session.commit()
+                    except Exception as e:
+                        logger.error(f'Failed to get Channel source id of {channel}', exc_info=e)
 
                 channel_import_logger.debug(f'Updated {repr(channel.name)}'
                                             f' url={channel.url}'

modules/videos/video/lib.py

@@ -283,6 +283,9 @@ async def get_missing_videos_comments(limit: int = VIDEO_COMMENTS_FETCH_COUNT):
             elif 'not a valid URL' in str(e):
                 logger.error(f'URL is not valid: {url=}', exc_info=e)
                 add_video_to_skip_list(url, censored=True)
+            elif ' Sign in ' in str(e):
+                logger.error(f'Video requires sign-in: {url=}', exc_info=e)
+                add_video_to_skip_list(url, censored=True)
             else:
                 logger.error(f'Got error when attempting to download video comments: {url=}', exc_info=e)
             continue

modules/zim/downloader.py

@@ -2,7 +2,8 @@ from http import HTTPStatus
 from typing import List
 
 from modules.zim import lib
-from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task, get_html_soup
+from wrolpi.common import logger, aiohttp_get, get_download_info, download_file, background_task, get_html_soup, \
+    trim_file_name
 from wrolpi.downloader import Downloader, Download, DownloadResult
 
 __all__ = ['KiwixCatalogDownloader', 'KiwixZimDownloader', 'kiwix_zim_downloader', 'kiwix_catalog_downloader']
@@ -72,7 +73,8 @@ class KiwixZimDownloader(Downloader):
 
         zim_directory = lib.get_zim_directory()
         zim_directory.mkdir(parents=True, exist_ok=True)
-        output_path = zim_directory / f'{download_info.name}'
+        name = trim_file_name(download_info.name)
+        output_path = zim_directory / name
 
         if output_path.is_file():
             output_size = output_path.stat().st_size