Checking for video corruption. Processing video when preparing filname to avoid corrupt videos.

This commit is contained in:
Roland 2022-05-18 17:24:50 -06:00
parent 0c98187276
commit dd07236834
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
9 changed files with 87 additions and 12 deletions

View file

@ -3,6 +3,7 @@ import os
import pathlib
import subprocess
import tempfile
from decimal import Decimal
from functools import partial
from pathlib import Path
from typing import Union, Tuple, List, Set, Iterable, Optional
@ -254,10 +255,38 @@ def get_video_duration(video_path: Path) -> Optional[int]:
logger.warning(f'FFPROBE failed to get duration with stderr: {e.stderr.decode()}')
raise
stdout = proc.stdout.decode()
duration = int(float(stdout.strip()))
duration = int(Decimal(stdout.strip()))
return duration
def check_for_video_corruption(video_path: Path) -> bool:
"""Uses ffprobe to check for specific ways a video file can be corrupt."""
if not isinstance(video_path, Path):
video_path = Path(video_path)
if not video_path.is_file():
raise FileNotFoundError(f'{video_path} does not exist!')
cmd = [FFPROBE_BIN, str(video_path)]
try:
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
except subprocess.CalledProcessError as e:
logger.warning(f'FFPROBE failed to check for corruption with stdout: {e.stdout.decode()}')
logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}')
return True # video is corrupt.
messages = (
'Invalid NAL unit size',
'Error splitting the input into NAL units',
)
stderr = proc.stderr.decode()
corrupt = False
for error in messages:
if error in stderr:
logger.warning(f'Possible video corruption ({error}): {video_path}')
corrupt = True
return corrupt
def apply_info_json(channel_id: int):
"""Update view_count for all Videos in a channel using its info_json file.

View file

@ -45,7 +45,7 @@ PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18
def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict:
"""Get info about a video. Separated for testing."""
return ydl.extract_info(url, download=False, process=False)
return ydl.extract_info(url, download=False, process=True)
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
@ -254,6 +254,7 @@ class VideoDownloader(Downloader, ABC):
options = get_downloader_config().dict()
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
logger.debug(f'Download video with options: {options}')
logger.debug(f'Downloading {url} to {out_dir}')
@ -265,10 +266,8 @@ class VideoDownloader(Downloader, ABC):
# Get the path where the video will be saved.
entry = extract_info(url, ydl=ydl)
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
if final_filename.suffix == '.NA':
# yt-dlp has odd behavior sometimes and ignores our preferred video extension.
final_filename = final_filename.with_suffix('.mp4')
logger.warning(f'Had to replace video extension: {final_filename}')
if final_filename.suffix.lower() != f'.{PREFERRED_VIDEO_EXTENSION}':
raise DownloadError(f'Cannot download video {url} because yt-dlp filename is invalid.')
return final_filename, entry

View file

@ -18,7 +18,7 @@ from wrolpi.vars import PYTEST
from .captions import get_captions
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
get_no_channel_directory
get_no_channel_directory, check_for_video_corruption
from .models import Channel, Video
logger = logger.getChild(__name__)
@ -181,6 +181,9 @@ def validate_video(video: Video, channel_generate_poster: bool):
video_path = video.video_path.path if isinstance(video.video_path, MediaPath) else video.video_path
# Check for corruption, but don't throw away the video even if it is corrupt. Errors will be logged in this call.
check_for_video_corruption(video_path)
if not video.title or not video.upload_date or not video.source_id:
# Video is missing things that can be extracted from the video file name.
# These are the least trusted, so anything already on the video should be trusted.

View file

@ -14,6 +14,7 @@ from wrolpi.dates import local_timezone, now
from wrolpi.downloader import Download, DownloadFrequency
from wrolpi.test.common import build_test_directories, TestAPI
from wrolpi.vars import PROJECT_DIR
from .. import common
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
from ..lib import save_channels_config, get_channels_config, import_channels_config
@ -414,3 +415,29 @@ def test_import_channels_config_outdated(test_session, test_directory, channel_f
# Videos were favorited by their channel "link".
assert vid1.favorite and vid2.favorite and vid3.favorite
assert not vid4.favorite
def test_check_for_video_corruption(video_file, test_directory):
# The test video is not corrupt.
assert common.check_for_video_corruption(video_file) is False
# An empty file is corrupt.
empty_file = test_directory / 'empty file.mp4'
empty_file.touch()
assert common.check_for_video_corruption(empty_file) is True
# A video file must be complete.
truncated_video = test_directory / 'truncated_video.mp4'
with truncated_video.open('wb') as fh:
fh.write(video_file.read_bytes()[:10000])
assert common.check_for_video_corruption(truncated_video) is True
# Check for specific ffprobe errors.
with mock.patch('modules.videos.common.subprocess') as mock_subprocess:
# `video_file` is ignored for these calls.
mock_subprocess.run().stderr.decode.return_value = 'Something\nInvalid NAL unit size'
assert common.check_for_video_corruption(video_file) is True
mock_subprocess.run().stderr.decode.return_value = 'Something\nError splitting the input into NAL units'
assert common.check_for_video_corruption(video_file) is True
mock_subprocess.run().stderr.decode.return_value = 'Some stderr is fine'
assert common.check_for_video_corruption(video_file) is False