mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
Checking for video corruption. Processing video when preparing filname to avoid corrupt videos.
This commit is contained in:
parent
0c98187276
commit
dd07236834
9 changed files with 87 additions and 12 deletions
|
|
@ -3,6 +3,7 @@ import os
|
|||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
from decimal import Decimal
|
||||
from functools import partial
|
||||
from pathlib import Path
|
||||
from typing import Union, Tuple, List, Set, Iterable, Optional
|
||||
|
|
@ -254,10 +255,38 @@ def get_video_duration(video_path: Path) -> Optional[int]:
|
|||
logger.warning(f'FFPROBE failed to get duration with stderr: {e.stderr.decode()}')
|
||||
raise
|
||||
stdout = proc.stdout.decode()
|
||||
duration = int(float(stdout.strip()))
|
||||
duration = int(Decimal(stdout.strip()))
|
||||
return duration
|
||||
|
||||
|
||||
def check_for_video_corruption(video_path: Path) -> bool:
|
||||
"""Uses ffprobe to check for specific ways a video file can be corrupt."""
|
||||
if not isinstance(video_path, Path):
|
||||
video_path = Path(video_path)
|
||||
if not video_path.is_file():
|
||||
raise FileNotFoundError(f'{video_path} does not exist!')
|
||||
|
||||
cmd = [FFPROBE_BIN, str(video_path)]
|
||||
try:
|
||||
proc = subprocess.run(cmd, check=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
except subprocess.CalledProcessError as e:
|
||||
logger.warning(f'FFPROBE failed to check for corruption with stdout: {e.stdout.decode()}')
|
||||
logger.warning(f'FFPROBE failed to check for corruption with stderr: {e.stderr.decode()}')
|
||||
return True # video is corrupt.
|
||||
|
||||
messages = (
|
||||
'Invalid NAL unit size',
|
||||
'Error splitting the input into NAL units',
|
||||
)
|
||||
stderr = proc.stderr.decode()
|
||||
corrupt = False
|
||||
for error in messages:
|
||||
if error in stderr:
|
||||
logger.warning(f'Possible video corruption ({error}): {video_path}')
|
||||
corrupt = True
|
||||
return corrupt
|
||||
|
||||
|
||||
def apply_info_json(channel_id: int):
|
||||
"""Update view_count for all Videos in a channel using its info_json file.
|
||||
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18
|
|||
|
||||
def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict:
|
||||
"""Get info about a video. Separated for testing."""
|
||||
return ydl.extract_info(url, download=False, process=False)
|
||||
return ydl.extract_info(url, download=False, process=True)
|
||||
|
||||
|
||||
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
|
||||
|
|
@ -254,6 +254,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
options = get_downloader_config().dict()
|
||||
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
|
||||
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
||||
logger.debug(f'Download video with options: {options}')
|
||||
|
||||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
|
||||
|
|
@ -265,10 +266,8 @@ class VideoDownloader(Downloader, ABC):
|
|||
# Get the path where the video will be saved.
|
||||
entry = extract_info(url, ydl=ydl)
|
||||
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
|
||||
if final_filename.suffix == '.NA':
|
||||
# yt-dlp has odd behavior sometimes and ignores our preferred video extension.
|
||||
final_filename = final_filename.with_suffix('.mp4')
|
||||
logger.warning(f'Had to replace video extension: {final_filename}')
|
||||
if final_filename.suffix.lower() != f'.{PREFERRED_VIDEO_EXTENSION}':
|
||||
raise DownloadError(f'Cannot download video {url} because yt-dlp filename is invalid.')
|
||||
return final_filename, entry
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -18,7 +18,7 @@ from wrolpi.vars import PYTEST
|
|||
from .captions import get_captions
|
||||
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
|
||||
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
|
||||
get_no_channel_directory
|
||||
get_no_channel_directory, check_for_video_corruption
|
||||
from .models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -181,6 +181,9 @@ def validate_video(video: Video, channel_generate_poster: bool):
|
|||
|
||||
video_path = video.video_path.path if isinstance(video.video_path, MediaPath) else video.video_path
|
||||
|
||||
# Check for corruption, but don't throw away the video even if it is corrupt. Errors will be logged in this call.
|
||||
check_for_video_corruption(video_path)
|
||||
|
||||
if not video.title or not video.upload_date or not video.source_id:
|
||||
# Video is missing things that can be extracted from the video file name.
|
||||
# These are the least trusted, so anything already on the video should be trusted.
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ from wrolpi.dates import local_timezone, now
|
|||
from wrolpi.downloader import Download, DownloadFrequency
|
||||
from wrolpi.test.common import build_test_directories, TestAPI
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from .. import common
|
||||
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
|
||||
apply_info_json, get_video_duration, generate_video_poster, is_valid_poster
|
||||
from ..lib import save_channels_config, get_channels_config, import_channels_config
|
||||
|
|
@ -414,3 +415,29 @@ def test_import_channels_config_outdated(test_session, test_directory, channel_f
|
|||
# Videos were favorited by their channel "link".
|
||||
assert vid1.favorite and vid2.favorite and vid3.favorite
|
||||
assert not vid4.favorite
|
||||
|
||||
|
||||
def test_check_for_video_corruption(video_file, test_directory):
|
||||
# The test video is not corrupt.
|
||||
assert common.check_for_video_corruption(video_file) is False
|
||||
|
||||
# An empty file is corrupt.
|
||||
empty_file = test_directory / 'empty file.mp4'
|
||||
empty_file.touch()
|
||||
assert common.check_for_video_corruption(empty_file) is True
|
||||
|
||||
# A video file must be complete.
|
||||
truncated_video = test_directory / 'truncated_video.mp4'
|
||||
with truncated_video.open('wb') as fh:
|
||||
fh.write(video_file.read_bytes()[:10000])
|
||||
assert common.check_for_video_corruption(truncated_video) is True
|
||||
|
||||
# Check for specific ffprobe errors.
|
||||
with mock.patch('modules.videos.common.subprocess') as mock_subprocess:
|
||||
# `video_file` is ignored for these calls.
|
||||
mock_subprocess.run().stderr.decode.return_value = 'Something\nInvalid NAL unit size'
|
||||
assert common.check_for_video_corruption(video_file) is True
|
||||
mock_subprocess.run().stderr.decode.return_value = 'Something\nError splitting the input into NAL units'
|
||||
assert common.check_for_video_corruption(video_file) is True
|
||||
mock_subprocess.run().stderr.decode.return_value = 'Some stderr is fine'
|
||||
assert common.check_for_video_corruption(video_file) is False
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue