mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 02:48:38 -04:00
Feature/video validate (#74)
* Videos are marked as validated so they will not be tried again. All previous bulk methods are combined into one. * Passing json to fixture factory to avoid writing the files in every test. * Adding log to explain if refresh is slow. * Generating video poster if one cannot be found. * Generating video poster if no poster can be found. * Videos without a video file (yet to be downloaded) are not validated. * Video title is extracted from the video file name as a last resort. * Keeping existing view count because it may be from the channel, which will be more recent. * Downloads should only be downloaded after they are due, not the same day. * Setting manager on new downloads. * Checking if downloading is disabled in loop. * Adding log when refresh is complete. * Not generating posters by default. This is to avoid deleting files. * Not converting posters unless channel has that enabled. * Not logging on valid because all posters will be checked. * Validating videos in upsert. * Using video name parser in upsert. * Using validate_video in upsert so that data is only read once from disk. * Attaching manager in download_manager.get_download. * Adding 22 to preferred video format for odd 720p resolution. Signed-off-by: Roland <roland@learningselfreliance.com>
This commit is contained in:
lrnselfreliance
Roland
parent
6976abcdad
commit
86893ef7f1
14 changed files with 464 additions and 434 deletions
|
|
@ -1,13 +1,12 @@
|
|||
#! /usr/bin/env python3
|
||||
import pathlib
|
||||
from pathlib import Path
|
||||
from typing import Generator, List, Union
|
||||
from typing import Generator, Union
|
||||
|
||||
import srt
|
||||
import webvtt
|
||||
|
||||
from wrolpi.common import logger, chunks
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.common import logger
|
||||
from wrolpi.media_path import MediaPath
|
||||
from .models import Video
|
||||
|
||||
|
|
@ -39,9 +38,9 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator:
|
|||
yield line
|
||||
|
||||
|
||||
def process_captions(video: Video):
|
||||
def get_video_captions(video: Video):
|
||||
"""
|
||||
Parse and insert captions for a video record.
|
||||
Parse video captions from the video's captions file.
|
||||
"""
|
||||
if isinstance(video.caption_path, MediaPath):
|
||||
caption_path = video.caption_path.path
|
||||
|
|
@ -51,8 +50,7 @@ def process_captions(video: Video):
|
|||
try:
|
||||
lines = get_unique_caption_lines(str(caption_path))
|
||||
block = '\n'.join(lines)
|
||||
video.caption = block
|
||||
return
|
||||
return block
|
||||
except UnicodeDecodeError:
|
||||
# Failed to decode the caption file
|
||||
# TODO handle this error
|
||||
|
|
@ -65,12 +63,3 @@ def process_captions(video: Video):
|
|||
pass
|
||||
|
||||
logger.debug(f'Failed to parse caption file {caption_path}')
|
||||
|
||||
|
||||
async def insert_bulk_captions(video_ids: List[int]):
|
||||
for video_ids in chunks(video_ids, 10):
|
||||
with get_db_session(commit=True) as session:
|
||||
for idx, video_id in enumerate(video_ids):
|
||||
video = session.query(Video).filter_by(id=video_id).one()
|
||||
process_captions(video)
|
||||
logger.debug(f'Inserted {len(video_ids)} captions')
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
import html
|
||||
import json
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
|
|
@ -14,7 +14,7 @@ from sqlalchemy.orm import Session
|
|||
from yt_dlp import YoutubeDL
|
||||
|
||||
from wrolpi import before_startup
|
||||
from wrolpi.common import logger, get_config, iterify, chunks, get_media_directory, \
|
||||
from wrolpi.common import logger, get_config, iterify, get_media_directory, \
|
||||
minimize_dict, api_param_limiter
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \
|
||||
|
|
@ -342,28 +342,12 @@ def generate_video_poster(video_path: Path) -> Path:
|
|||
logger.warning(f'FFMPEG poster generation failed with stdout: {e.stdout.decode()}')
|
||||
logger.warning(f'FFMPEG poster generation failed with stdout: {e.stderr.decode()}')
|
||||
raise
|
||||
if not poster_path.exists():
|
||||
raise Exception(f'Failed to find generated poster! {poster_path}')
|
||||
|
||||
return poster_path
|
||||
|
||||
|
||||
async def generate_bulk_posters(video_ids: List[int]):
|
||||
"""
|
||||
Generate all posters for the provided videos. Update the video object with the new jpg file location. Do not
|
||||
clobber existing jpg files.
|
||||
"""
|
||||
logger.info(f'Generating {len(video_ids)} video posters')
|
||||
for video_ids in chunks(video_ids, 10):
|
||||
with get_db_session(commit=True) as session:
|
||||
videos = session.query(Video).filter(Video.id.in_(video_ids))
|
||||
for video in videos:
|
||||
video_path = video.video_path.path
|
||||
|
||||
poster_path = replace_extension(video_path, '.jpg')
|
||||
if not poster_path.exists():
|
||||
generate_video_poster(video_path)
|
||||
video.poster_path = poster_path
|
||||
logger.info('Done generating video posters')
|
||||
|
||||
|
||||
def convert_image(existing_path: Path, destination_path: Path, ext: str = 'jpeg'):
|
||||
"""
|
||||
Convert an image from one format to another. Remove the existing image file. This will safely overwrite an image
|
||||
|
|
@ -391,42 +375,6 @@ def is_valid_poster(poster_path: Path) -> bool:
|
|||
return False
|
||||
|
||||
|
||||
def bulk_validate_posters(video_ids: List[int]):
|
||||
"""
|
||||
Replace all posters for the provided videos if a video's poster is not a JPEG format.
|
||||
"""
|
||||
logger.info(f'Validating {len(video_ids)} video posters')
|
||||
for video_ids in chunks(video_ids, 10):
|
||||
for video_id in video_ids:
|
||||
with get_db_session(commit=True) as session:
|
||||
video: Video = session.query(Video).filter_by(id=video_id).one()
|
||||
|
||||
poster_path: Path = video.poster_path.path
|
||||
new_poster_path = poster_path.with_suffix('.jpg')
|
||||
|
||||
if poster_path != new_poster_path and new_poster_path.exists():
|
||||
# Destination JPEG already exists (it may have the wrong format), lets overwrite it with a valid
|
||||
# JPEG.
|
||||
poster_path.unlink()
|
||||
poster_path = new_poster_path
|
||||
|
||||
if not is_valid_poster(poster_path):
|
||||
# Poster is not valid, convert it and place it in the new location.
|
||||
try:
|
||||
convert_image(poster_path, new_poster_path)
|
||||
logger.info(f'Converted invalid poster {poster_path} to {new_poster_path}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to convert invalid poster {poster_path} to {new_poster_path}', exc_info=e)
|
||||
continue
|
||||
else:
|
||||
logger.debug(f'Poster was already valid: {new_poster_path}')
|
||||
|
||||
# Update the video with the new poster path. Mark it as validated.
|
||||
video.poster_path = new_poster_path
|
||||
video.validated_poster = True
|
||||
logger.info('Done validating video posters.')
|
||||
|
||||
|
||||
def get_video_duration(video_path: Path) -> int:
|
||||
"""
|
||||
Get the duration of a video in seconds. Do this using ffprobe.
|
||||
|
|
@ -450,62 +398,6 @@ def get_video_duration(video_path: Path) -> int:
|
|||
return duration
|
||||
|
||||
|
||||
async def get_bulk_video_info_json(video_ids: List[int]):
|
||||
"""
|
||||
Get and save the info_json data for each video provided.
|
||||
"""
|
||||
logger.info(f'Getting {len(video_ids)} video info_json meta data.')
|
||||
for video_ids in chunks(video_ids, 10):
|
||||
with get_db_session(commit=True) as session:
|
||||
for video_id in video_ids:
|
||||
video = session.query(Video).filter_by(id=video_id).one()
|
||||
logger.debug(f'Getting video info_json data: {video}')
|
||||
|
||||
if not video.video_path:
|
||||
logger.warning(f'Refusing to get info_json for video without a video file: {video}')
|
||||
continue
|
||||
|
||||
try:
|
||||
info_json = video.get_info_json()
|
||||
if info_json:
|
||||
if not video.view_count and info_json.get('view_count'):
|
||||
video.view_count = info_json['view_count']
|
||||
if not video.duration:
|
||||
# Get duration from info_json without reading video file.
|
||||
video.duration = info_json.get('duration')
|
||||
# Trust the info_json title before the video path title.
|
||||
video.title = info_json.get('title')
|
||||
if not video.duration:
|
||||
# As a last resort, get duration from the video file.
|
||||
video.duration = get_video_duration(video.video_path.path)
|
||||
if not video.title:
|
||||
# Use the video path name without the extension.
|
||||
video.title = video.video_path.path.with_suffix('').name
|
||||
|
||||
# Unescape any HTML entities.
|
||||
video.title = html.unescape(video.title)
|
||||
|
||||
except Exception:
|
||||
logger.warning(f'Unable to get meta data of {video}', exc_info=True)
|
||||
logger.info('Done getting video info_json meta data.')
|
||||
|
||||
|
||||
async def get_bulk_video_size(video_ids: List[int]):
|
||||
"""
|
||||
Get and save the size for each video provided.
|
||||
"""
|
||||
logger.info(f'Getting {len(video_ids)} video sizes.')
|
||||
for video_ids in chunks(video_ids, 10):
|
||||
with get_db_session(commit=True) as session:
|
||||
for video_id in video_ids:
|
||||
video = session.query(Video).filter_by(id=video_id).one()
|
||||
logger.debug(f'Getting video size: {video.id} {video.video_path}')
|
||||
|
||||
size = video.video_path.path.stat().st_size
|
||||
video.size = size
|
||||
logger.info('Done getting video sizes')
|
||||
|
||||
|
||||
def apply_info_json(channel_id: int):
|
||||
"""
|
||||
Update view_count for all Videos in a channel using its info_json file. Mark any videos not in the info_json as
|
||||
|
|
|
|||
|
|
@ -1,9 +1,11 @@
|
|||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
from uuid import uuid4
|
||||
|
||||
import mock
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
||||
from modules.videos.channel.lib import spread_channel_downloads
|
||||
from modules.videos.downloader import VideoDownloader, ChannelDownloader
|
||||
|
|
@ -99,13 +101,38 @@ def video_file(test_directory) -> pathlib.Path:
|
|||
|
||||
@pytest.fixture
|
||||
def video_factory(test_session, test_directory):
|
||||
def _(channel_id: int = None):
|
||||
"""
|
||||
Creates Videos for testing.
|
||||
"""
|
||||
|
||||
def _(channel_id: int = None, with_video_file: bool = False, with_info_json: dict = None,
|
||||
with_poster_ext: str = None):
|
||||
title = str(uuid4())
|
||||
if channel_id:
|
||||
path = str(test_directory / f'{title}.mp4')
|
||||
path = test_directory / f'{title}.mp4'
|
||||
else:
|
||||
path = str(test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4')
|
||||
video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title)
|
||||
path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4'
|
||||
|
||||
if with_video_file:
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
|
||||
|
||||
info_json_path = None
|
||||
if with_info_json is True:
|
||||
# User requests a info json file, but does not provide one.
|
||||
info_json_path = path.with_suffix('.info.json')
|
||||
info_json_path.write_text(json.dumps({}))
|
||||
elif with_info_json:
|
||||
# Use the provided object as the json.
|
||||
info_json_path = path.with_suffix('.info.json')
|
||||
info_json_path.write_text(json.dumps(with_info_json))
|
||||
|
||||
poster_path = None
|
||||
if with_poster_ext:
|
||||
poster_path = path.with_suffix(f'.{with_poster_ext}')
|
||||
Image.new('RGB', (25, 25)).save(poster_path)
|
||||
|
||||
video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title,
|
||||
info_json_path=info_json_path, poster_path=poster_path)
|
||||
test_session.add(video)
|
||||
return video
|
||||
|
||||
|
|
|
|||
|
|
@ -37,7 +37,7 @@ ChannelIEs = {
|
|||
}
|
||||
|
||||
PREFERRED_VIDEO_EXTENSION = 'mp4'
|
||||
PREFERRED_VIDEO_FORMAT = 'best[height=720],mp4-480p,mp4-360p,mp4-240p'
|
||||
PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p'
|
||||
|
||||
|
||||
def get_no_channel_directory():
|
||||
|
|
@ -171,7 +171,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
return_code = self.process_runner(url, cmd, out_dir)
|
||||
|
||||
if return_code != 0:
|
||||
raise ValueError(f'youtube-dl exited with {return_code}')
|
||||
raise ValueError(f'video downloader process exited with {return_code}')
|
||||
|
||||
if not video_path.is_file():
|
||||
raise ValueError(f'Video file could not be found! {video_path}')
|
||||
|
|
|
|||
|
|
@ -1,10 +1,8 @@
|
|||
import asyncio
|
||||
import html
|
||||
import json
|
||||
import pathlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Tuple
|
||||
from typing import Tuple, Optional
|
||||
from uuid import uuid1
|
||||
|
||||
from sqlalchemy.orm import Session
|
||||
|
|
@ -12,11 +10,11 @@ from sqlalchemy.orm import Session
|
|||
from wrolpi.common import logger, chunks, get_config
|
||||
from wrolpi.common import save_settings_config
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.media_path import MediaPath
|
||||
from wrolpi.vars import PYTEST
|
||||
from .captions import insert_bulk_captions, process_captions
|
||||
from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \
|
||||
generate_video_paths, remove_duplicate_video_paths, bulk_validate_posters, \
|
||||
apply_info_json, import_videos_config
|
||||
from .captions import get_video_captions
|
||||
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \
|
||||
get_video_duration, is_valid_poster, convert_image, generate_video_poster
|
||||
from .models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -24,115 +22,6 @@ logger = logger.getChild(__name__)
|
|||
DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly
|
||||
|
||||
|
||||
def refresh_channel_video_captions() -> bool:
|
||||
with get_db_curs() as curs:
|
||||
query = 'SELECT id FROM video WHERE caption IS NULL AND caption_path IS NOT NULL'
|
||||
curs.execute(query)
|
||||
missing_captions = [i for (i,) in curs.fetchall()]
|
||||
|
||||
if missing_captions:
|
||||
coro = insert_bulk_captions(missing_captions)
|
||||
asyncio.ensure_future(coro)
|
||||
logger.info('Scheduled insert_bulk_captions()')
|
||||
return True
|
||||
else:
|
||||
logger.info('No missing captions to process.')
|
||||
return False
|
||||
|
||||
|
||||
def refresh_channel_generate_posters() -> bool:
|
||||
with get_db_curs() as curs:
|
||||
query = 'SELECT id FROM video WHERE video_path IS NOT NULL AND poster_path IS NULL'
|
||||
curs.execute(query)
|
||||
missing_posters = [i for (i,) in curs.fetchall()]
|
||||
|
||||
if missing_posters:
|
||||
coro = generate_bulk_posters(missing_posters)
|
||||
asyncio.ensure_future(coro)
|
||||
logger.info('Scheduled generate_bulk_posters()')
|
||||
return True
|
||||
else:
|
||||
logger.info('No missing posters to generate.')
|
||||
return False
|
||||
|
||||
|
||||
def convert_invalid_posters() -> bool:
|
||||
"""
|
||||
Searches the DB for all videos with an invalid poster type (i.e. webp) and converts them to JPEGs. A video with a
|
||||
valid poster will be marked as such in it's column "validated_poster".
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
query = "SELECT id FROM video WHERE poster_path IS NOT NULL AND validated_poster = FALSE"
|
||||
curs.execute(query)
|
||||
invalid_posters = [i for (i,) in curs.fetchall()]
|
||||
|
||||
if invalid_posters:
|
||||
async def _():
|
||||
return bulk_validate_posters(invalid_posters)
|
||||
|
||||
coro = _()
|
||||
asyncio.ensure_future(coro)
|
||||
logger.info('Scheduled bulk_replace_invalid_posters()')
|
||||
return True
|
||||
else:
|
||||
logger.info('No invalid posters to replace.')
|
||||
return False
|
||||
|
||||
|
||||
def refresh_channel_info_json() -> bool:
|
||||
"""
|
||||
Fill in Video columns that are extracted from the info_json.
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
query = '''
|
||||
SELECT v.id
|
||||
FROM video v
|
||||
WHERE
|
||||
v.video_path IS NOT NULL
|
||||
AND v.info_json_path IS NOT NULL
|
||||
AND v.info_json_path != ''
|
||||
AND (v.duration IS NULL OR v.view_count IS NULL OR v.title IS NULL)
|
||||
'''
|
||||
curs.execute(query)
|
||||
missing_duration = [i for (i,) in curs.fetchall()]
|
||||
|
||||
if missing_duration:
|
||||
coro = get_bulk_video_info_json(missing_duration)
|
||||
asyncio.ensure_future(coro)
|
||||
logger.info('Scheduled get_bulk_video_info_json()')
|
||||
return True
|
||||
else:
|
||||
logger.info('No videos need updating from info_json.')
|
||||
return False
|
||||
|
||||
|
||||
def refresh_channel_calculate_size() -> bool:
|
||||
with get_db_curs() as curs:
|
||||
query = 'SELECT id FROM video WHERE video_path IS NOT NULL AND size IS NULL'
|
||||
curs.execute(query)
|
||||
missing_size = [i for (i,) in curs.fetchall()]
|
||||
|
||||
if missing_size:
|
||||
coro = get_bulk_video_size(missing_size)
|
||||
asyncio.ensure_future(coro)
|
||||
logger.info('Scheduled get_bulk_video_size()')
|
||||
return True
|
||||
else:
|
||||
logger.info('No videos missing size.')
|
||||
return False
|
||||
|
||||
|
||||
def process_video_meta_data():
|
||||
"""
|
||||
Search for any videos missing meta data, fill in that data.
|
||||
"""
|
||||
refresh_channel_video_captions()
|
||||
refresh_channel_generate_posters()
|
||||
convert_invalid_posters()
|
||||
refresh_channel_info_json()
|
||||
refresh_channel_calculate_size()
|
||||
|
||||
|
||||
def refresh_channel_videos(channel: Channel):
|
||||
"""
|
||||
Find all video files in a channel's directory. Add any videos not in the DB to the DB.
|
||||
|
|
@ -222,6 +111,136 @@ def refresh_no_channel_videos():
|
|||
logger.info(deleted_status)
|
||||
|
||||
|
||||
def process_video_info_json(video: Video):
|
||||
"""
|
||||
Parse the Video's info json file, return the relevant data.
|
||||
"""
|
||||
title = duration = view_count = url = None
|
||||
if info_json := video.get_info_json():
|
||||
title = info_json.get('fulltitle') or info_json.get('title')
|
||||
title = html.unescape(title) if title else None
|
||||
|
||||
duration = info_json.get('duration')
|
||||
view_count = info_json.get('view_count')
|
||||
url = info_json.get('webpage_url') or info_json.get('url')
|
||||
|
||||
return title, duration, view_count, url
|
||||
|
||||
|
||||
def validate_videos():
|
||||
"""
|
||||
Validate all Videos not yet validated. A Video is validated when we have attempted to find its: title, duration,
|
||||
view_count, url, caption, size. A Video is also valid when it has a JPEG poster, if any. If no poster can be
|
||||
found, it will be generated from the video file.
|
||||
|
||||
This function marks the Video as validated, even if no data can be found so a Video will not be validated multiple
|
||||
times.
|
||||
"""
|
||||
with get_db_curs() as curs:
|
||||
curs.execute('SELECT id FROM video WHERE video_path IS NOT NULL AND validated IS FALSE')
|
||||
video_ids = [i['id'] for i in curs.fetchall()]
|
||||
curs.execute('SELECT id, generate_posters FROM channel')
|
||||
channel_generate_posters = {i['id']: i['generate_posters'] for i in curs.fetchall()}
|
||||
|
||||
logger.info(f'Validating {len(video_ids)} videos.')
|
||||
for chunk in chunks(video_ids, 20):
|
||||
with get_db_session(commit=True) as session:
|
||||
videos = session.query(Video).filter(Video.id.in_(chunk)).all()
|
||||
for video in videos:
|
||||
try:
|
||||
channel_generate_poster = channel_generate_posters.get(video.channel_id)
|
||||
validate_video(video, channel_generate_poster)
|
||||
# All data about the Video has been found, we should not attempt to validate it again.
|
||||
video.validated = True
|
||||
except Exception as e:
|
||||
# This video failed to validate, continue validation for the rest of the videos.
|
||||
logger.warning(f'Failed to validate {video=}', exc_info=e)
|
||||
|
||||
|
||||
def validate_video(video: Video, channel_generate_poster: bool):
|
||||
"""
|
||||
Validate a single video. A Video is validated when we have attempted to find its: title, duration,
|
||||
view_count, url, caption, size. A Video is also valid when it has a JPEG poster, if any. If no poster can be
|
||||
found, it will be generated from the video file.
|
||||
"""
|
||||
if not video.title or not video.duration or not video.view_count or not video.url:
|
||||
# These properties can be found in the info json.
|
||||
title, duration, view_count, url = process_video_info_json(video)
|
||||
video.title = title
|
||||
video.duration = duration
|
||||
video.url = url
|
||||
# View count will probably be overwritten by more recent data when this Video's Channel is
|
||||
# updated.
|
||||
video.view_count = video.view_count or view_count
|
||||
|
||||
video_path = video.video_path.path if isinstance(video.video_path, MediaPath) else video.video_path
|
||||
|
||||
if not video.title or not video.upload_date or not video.source_id:
|
||||
# Video is missing things that can be extracted from the video file name.
|
||||
# These are the least trusted, so anything already on the video should be trusted.
|
||||
_, upload_date, source_id, title = parse_video_file_name(video_path)
|
||||
video.title = video.title or html.unescape(title)
|
||||
video.upload_date = video.upload_date or upload_date
|
||||
video.source_id = video.source_id or source_id
|
||||
if not video.duration:
|
||||
# Video duration was not in the info json, use ffprobe.
|
||||
video.duration = get_video_duration(video_path)
|
||||
if not video.caption and video.caption_path:
|
||||
video.caption = get_video_captions(video)
|
||||
if not video.size:
|
||||
video.size = video_path.stat().st_size
|
||||
if not video.poster_path:
|
||||
# Video poster is not found, lets check near the video file.
|
||||
for ext in ('.jpg', '.jpeg', '.webp', '.png'):
|
||||
if (poster_path := video_path.with_suffix(ext)).is_file():
|
||||
video.poster_path = poster_path
|
||||
break
|
||||
if channel_generate_poster:
|
||||
# Try to convert/generate, but keep the old poster if those fail.
|
||||
video.poster_path = convert_or_generate_poster(video) or video.poster_path
|
||||
|
||||
|
||||
def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]:
|
||||
"""
|
||||
If a Video has a poster, but the poster is invalid, convert it. If a Video has no poster, generate one from the
|
||||
video file.
|
||||
|
||||
Returns None if the poster was not converted, and not generated.
|
||||
"""
|
||||
video_path = video.video_path.path
|
||||
# Modification/generation of poster is enabled for this channel.
|
||||
if video.poster_path:
|
||||
# Check that the poster is a more universally supported JPEG.
|
||||
old: pathlib.Path = video.poster_path.path if \
|
||||
isinstance(video.poster_path, MediaPath) else video.poster_path
|
||||
new = old.with_suffix('.jpg')
|
||||
|
||||
if old != new and new.exists():
|
||||
# Destination JPEG already exists (it may have the wrong format).
|
||||
old.unlink()
|
||||
old = video.poster_path = new
|
||||
|
||||
if not is_valid_poster(old):
|
||||
# Poster is not valid, convert it and place it in the new location.
|
||||
try:
|
||||
convert_image(old, new)
|
||||
old.unlink(missing_ok=True)
|
||||
logger.info(f'Converted invalid poster {old} to {new}')
|
||||
return new
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to convert invalid poster {old} to {new}', exc_info=e)
|
||||
return
|
||||
|
||||
if not video.poster_path:
|
||||
# Video poster was not discovered, or converted. Let's generate it.
|
||||
try:
|
||||
poster_path = generate_video_poster(video_path)
|
||||
logger.debug(f'Generated poster for {video}')
|
||||
return poster_path
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to generate poster for {video}', exc_info=e)
|
||||
|
||||
|
||||
def _refresh_videos(channel_links: list = None):
|
||||
"""
|
||||
Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the
|
||||
|
|
@ -259,7 +278,9 @@ def _refresh_videos(channel_links: list = None):
|
|||
# Fill in any missing data for all videos.
|
||||
if not PYTEST:
|
||||
import_videos_config()
|
||||
process_video_meta_data()
|
||||
validate_videos()
|
||||
|
||||
logger.info('Refresh of video files complete')
|
||||
|
||||
|
||||
def get_channels_config(session: Session) -> dict:
|
||||
|
|
@ -364,12 +385,37 @@ async def get_statistics():
|
|||
return ret
|
||||
|
||||
|
||||
NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.'
|
||||
NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.+?)_)?(.*)\.'
|
||||
r'(jpg|webp|flv|mp4|part|info\.json|description|webm|..\.srt|..\.vtt)')
|
||||
|
||||
|
||||
def parse_video_file_name(video_path: pathlib.Path) -> \
|
||||
Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]:
|
||||
"""
|
||||
A Video's file name can have data in it, this attempts to extract what may be there.
|
||||
|
||||
Example: {channel_name}_{upload_date}_{source_id}_title{ext}
|
||||
"""
|
||||
video_str = str(video_path)
|
||||
if match := NAME_PARSER.match(video_str):
|
||||
channel, date, source_id, title, _ = match.groups()
|
||||
|
||||
channel = None if channel == 'NA' else channel
|
||||
date = None if date == 'NA' else date
|
||||
|
||||
title = title.strip()
|
||||
if date == 'NA':
|
||||
return channel, None, source_id, title
|
||||
if date is None or (len(date) == 8 and date.isdigit()):
|
||||
return channel, date, source_id, title
|
||||
|
||||
# Return the stem as a last resort
|
||||
title = pathlib.Path(video_path).stem.strip()
|
||||
return None, None, None, title
|
||||
|
||||
|
||||
def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = None, idempotency: str = None,
|
||||
skip_captions=False, id_: str = None) -> Video:
|
||||
id_: str = None) -> Video:
|
||||
"""
|
||||
Insert a video into the DB. Also, find any meta-files near the video file and store them on the video row.
|
||||
|
||||
|
|
@ -377,66 +423,37 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
|
|||
"""
|
||||
if not video_path.is_absolute():
|
||||
raise ValueError(f'Video path is not absolute: {video_path}')
|
||||
|
||||
# This function can update or insert a Video.
|
||||
video = session.query(Video).filter_by(id=id_).one() if id_ else Video()
|
||||
|
||||
# Set the file values, all other things can be found using these files.
|
||||
poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
|
||||
video.caption_path = caption_path
|
||||
video.description_path = description_path
|
||||
video.info_json_path = info_json_path
|
||||
video.poster_path = poster_path
|
||||
video.video_path = video_path
|
||||
|
||||
name_match = NAME_PARSER.match(video_path.name)
|
||||
_ = upload_date = source_id = title = ext = None
|
||||
if name_match:
|
||||
_, upload_date, source_id, title, ext = name_match.groups()
|
||||
if channel and not video_path.is_relative_to(channel.directory.path):
|
||||
raise ValueError(f'Video path is not within its channel {video_path=} {channel.directory=}')
|
||||
|
||||
# Make sure the date is a valid date format, if not, leave it blank. Youtube-DL sometimes puts an NA in the date.
|
||||
# We may even get videos that weren't downloaded by WROLPi.
|
||||
if not upload_date or not upload_date.isdigit() or len(upload_date) != 8:
|
||||
logger.debug(f'Could not parse date from filename: {video_path}')
|
||||
upload_date = None
|
||||
video.channel = channel
|
||||
video.idempotency = idempotency
|
||||
|
||||
duration = None
|
||||
url = None
|
||||
if info_json_path:
|
||||
try:
|
||||
with info_json_path.open('rt') as fh:
|
||||
json_contents = json.load(fh)
|
||||
duration = json_contents.get('duration')
|
||||
url = json_contents.get('webpage_url')
|
||||
# Trust the info_json title before the video filename.
|
||||
title = json_contents.get('title', title)
|
||||
except json.decoder.JSONDecodeError:
|
||||
logger.warning(f'Failed to load JSON file to get duration: {info_json_path}')
|
||||
|
||||
size = video_path.stat().st_size
|
||||
title = html.unescape(title) if title else None
|
||||
|
||||
video_dict = dict(
|
||||
caption_path=str(caption_path) if caption_path else None,
|
||||
channel_id=channel.id if channel else None,
|
||||
description_path=str(description_path) if description_path else None,
|
||||
duration=duration,
|
||||
ext=ext,
|
||||
idempotency=idempotency,
|
||||
info_json_path=str(info_json_path) if info_json_path else None,
|
||||
poster_path=str(poster_path) if poster_path else None,
|
||||
size=size,
|
||||
source_id=source_id,
|
||||
title=title,
|
||||
upload_date=upload_date,
|
||||
url=url,
|
||||
video_path=str(video_path),
|
||||
)
|
||||
|
||||
if id_:
|
||||
video = session.query(Video).filter_by(id=id_).one()
|
||||
for key, value in video_dict.items():
|
||||
setattr(video, key, value)
|
||||
else:
|
||||
video = Video(**video_dict)
|
||||
try:
|
||||
# Fill in any missing data. Generate poster if enabled and necessary.
|
||||
validate_video(video, channel.generate_posters if channel else False)
|
||||
# All data about the Video has been found, we should not attempt to validate it again.
|
||||
video.validated = True
|
||||
except Exception:
|
||||
# Could not validate, this could be an issue with a file. This should not prevent the video from being
|
||||
# inserted.
|
||||
pass
|
||||
|
||||
session.add(video)
|
||||
session.flush()
|
||||
|
||||
if skip_captions is False and caption_path:
|
||||
# Process captions only when requested
|
||||
process_captions(video)
|
||||
|
||||
return video
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, Fore
|
|||
from sqlalchemy.orm import relationship, Session, deferred
|
||||
from sqlalchemy.orm.collections import InstrumentedList
|
||||
|
||||
from wrolpi.common import Base, tsvector, ModelHelper, logger
|
||||
from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directory
|
||||
from wrolpi.dates import now, TZDateTime
|
||||
from wrolpi.db import get_db_curs
|
||||
from wrolpi.downloader import Download
|
||||
|
|
@ -23,6 +23,7 @@ class Video(ModelHelper, Base):
|
|||
channel_id = Column(Integer, ForeignKey('channel.id'))
|
||||
channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos')
|
||||
idempotency = Column(String)
|
||||
validated = Column(Boolean, default=False)
|
||||
|
||||
# File paths
|
||||
caption_path = Column(MediaPathType)
|
||||
|
|
@ -33,17 +34,16 @@ class Video(ModelHelper, Base):
|
|||
video_path = Column(MediaPathType)
|
||||
|
||||
caption = deferred(Column(String)) # slow to fetch
|
||||
censored = Column(Boolean, default=False)
|
||||
duration = Column(Integer)
|
||||
favorite = Column(TZDateTime)
|
||||
size = Column(Integer)
|
||||
source_id = Column(String)
|
||||
title = Column(String)
|
||||
upload_date = Column(TZDateTime)
|
||||
validated_poster = Column(Boolean, default=False)
|
||||
viewed = Column(TZDateTime)
|
||||
view_count = Column(Integer)
|
||||
url = Column(String)
|
||||
censored = Column(Boolean, default=False)
|
||||
view_count = Column(Integer)
|
||||
viewed = Column(TZDateTime)
|
||||
|
||||
textsearch = deferred(
|
||||
Column(tsvector, Computed('''to_tsvector('english'::regconfig,
|
||||
|
|
@ -51,7 +51,9 @@ class Video(ModelHelper, Base):
|
|||
COALESCE(caption, ''::text)))''')))
|
||||
|
||||
def __repr__(self):
|
||||
video_path = self.video_path.path if self.video_path else None
|
||||
video_path = self.video_path
|
||||
if self.video_path and isinstance(self.video_path, MediaPath):
|
||||
video_path = self.video_path.path.relative_to(get_media_directory())
|
||||
return f'<Video id={self.id} title={self.title} path={video_path} channel={self.channel_id} ' \
|
||||
f'source_id={self.source_id}>'
|
||||
|
||||
|
|
@ -258,8 +260,8 @@ class Channel(ModelHelper, Base):
|
|||
match_regex = Column(String)
|
||||
directory: MediaPath = Column(MediaPathType)
|
||||
skip_download_videos = Column(ARRAY(String))
|
||||
generate_posters = Column(Boolean)
|
||||
calculate_duration = Column(Boolean)
|
||||
generate_posters = Column(Boolean, default=False) # generating posters may delete files, and can be slow.
|
||||
calculate_duration = Column(Boolean, default=True)
|
||||
download_frequency = Column(Integer)
|
||||
source_id = Column(String)
|
||||
refreshed = Column(Boolean, default=False)
|
||||
|
|
|
|||
|
|
@ -46,11 +46,11 @@ class TestCaption(TestAPI):
|
|||
video1 = Video(title='scream', caption_path=str(self.vtt_path1))
|
||||
session.add(video1)
|
||||
session.commit()
|
||||
captions.process_captions(video1)
|
||||
video1.caption = captions.get_video_captions(video1)
|
||||
video2 = Video(title='bar', caption_path=str(self.vtt_path2))
|
||||
session.add(video2)
|
||||
session.commit()
|
||||
captions.process_captions(video2)
|
||||
video2.caption = captions.get_video_captions(video2)
|
||||
|
||||
session.commit()
|
||||
session.flush()
|
||||
|
|
|
|||
|
|
@ -1,11 +1,10 @@
|
|||
import json
|
||||
import pathlib
|
||||
import subprocess
|
||||
import tempfile
|
||||
from datetime import datetime
|
||||
from typing import List
|
||||
from unittest import mock
|
||||
from unittest.mock import Mock, MagicMock
|
||||
from unittest.mock import MagicMock
|
||||
|
||||
import pytest
|
||||
from PIL import Image
|
||||
|
|
@ -17,9 +16,9 @@ from wrolpi.dates import local_timezone
|
|||
from wrolpi.db import get_db_session
|
||||
from wrolpi.test.common import build_test_directories, wrap_test_db, TestAPI
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
from ..common import get_matching_directories, convert_image, bulk_validate_posters, remove_duplicate_video_paths, \
|
||||
from ..common import get_matching_directories, convert_image, remove_duplicate_video_paths, \
|
||||
apply_info_json, get_video_duration, generate_video_poster, replace_extension, is_valid_poster, \
|
||||
get_bulk_video_info_json, import_videos_config
|
||||
import_videos_config
|
||||
from ..lib import save_channels_config
|
||||
|
||||
|
||||
|
|
@ -186,79 +185,6 @@ def test_create_db_structure(_structure, paths):
|
|||
test_func(test_)
|
||||
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure(
|
||||
{
|
||||
'channel1': ['vid1.mp4', 'vid1.jpg'],
|
||||
'channel2': ['vid2.flv', 'vid2.webp'],
|
||||
}
|
||||
)
|
||||
def test_bulk_replace_invalid_posters(tempdir: pathlib.Path):
|
||||
"""
|
||||
Test that when a video has an invalid poster format, we convert it to JPEG.
|
||||
"""
|
||||
channel1, channel2 = sorted(tempdir.iterdir())
|
||||
jpg, mp4 = sorted(channel1.iterdir())
|
||||
flv, webp = sorted(channel2.iterdir())
|
||||
|
||||
Image.new('RGB', (25, 25)).save(jpg)
|
||||
Image.new('RGB', (25, 25)).save(webp)
|
||||
|
||||
with open(jpg, 'rb') as jpg_fh, open(webp, 'rb') as webp_fh:
|
||||
# Files are different formats.
|
||||
jpg_fh_contents = jpg_fh.read()
|
||||
webp_fh_contents = webp_fh.read()
|
||||
assert jpg_fh_contents != webp_fh_contents
|
||||
assert Image.open(jpg_fh).format == 'JPEG'
|
||||
assert Image.open(webp_fh).format == 'WEBP'
|
||||
|
||||
with get_db_session() as session:
|
||||
vid1 = session.query(Video).filter_by(poster_path=f'{tempdir}/channel1/vid1.jpg').one()
|
||||
assert vid1.validated_poster is False
|
||||
|
||||
vid2 = session.query(Video).filter_by(poster_path=f'{tempdir}/channel2/vid2.webp').one()
|
||||
assert vid2.validated_poster is False
|
||||
|
||||
# Convert the WEBP image. convert_image() should only be called once.
|
||||
mocked_convert_image = Mock(wraps=convert_image)
|
||||
with mock.patch('modules.videos.common.convert_image', mocked_convert_image):
|
||||
video_ids = [vid1.id, vid2.id]
|
||||
bulk_validate_posters(video_ids)
|
||||
|
||||
mocked_convert_image.assert_called_once_with(webp, tempdir / 'channel2/vid2.jpg')
|
||||
|
||||
with get_db_session() as session:
|
||||
# Get the video by ID because it's poster is now a JPEG.
|
||||
vid2 = session.query(Video).filter_by(id=vid2.id).one()
|
||||
assert str(vid2.poster_path.path).split('/')[-1] == 'vid2.jpg'
|
||||
assert all('webp' not in str(i.poster_path.path) for i in session.query(Video).all())
|
||||
assert vid2.validated_poster is True
|
||||
|
||||
# Vid1's image was validated, but not converted.
|
||||
vid1 = session.query(Video).filter_by(id=vid1.id).one()
|
||||
assert str(vid1.poster_path.path).split('/')[-1] == 'vid1.jpg'
|
||||
assert vid1.validated_poster is True
|
||||
|
||||
# Old webp was removed
|
||||
assert not webp.is_file()
|
||||
new_jpg = tempdir / 'channel2/vid2.jpg'
|
||||
assert new_jpg.is_file()
|
||||
# chmod 644
|
||||
assert new_jpg.stat().st_mode == 0o100644
|
||||
with open(new_jpg, 'rb') as new_jpg_fh:
|
||||
# The converted image is the same as the other JPEG because both are black 25x25 pixel images.
|
||||
assert jpg_fh_contents == new_jpg_fh.read()
|
||||
assert Image.open(new_jpg_fh).format == 'JPEG'
|
||||
|
||||
# Calling convert again has no effect.
|
||||
mocked_convert_image.reset_mock()
|
||||
with mock.patch('modules.videos.common.convert_image', mocked_convert_image):
|
||||
video_ids = [vid1.id, vid2.id]
|
||||
bulk_validate_posters(video_ids)
|
||||
|
||||
mocked_convert_image.assert_not_called()
|
||||
|
||||
|
||||
@wrap_test_db
|
||||
@create_channel_structure(
|
||||
{
|
||||
|
|
@ -383,34 +309,6 @@ def test_update_censored_videos(test_session, video_factory, simple_channel):
|
|||
check_censored([(vid1.id, True), (vid2.id, True), (vid3.id, True), (vid4.id, False)])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_title(test_session, simple_video):
|
||||
"""
|
||||
A Video title will be filled out by `get_bulk_video_info_json`.
|
||||
"""
|
||||
video_path: pathlib.Path = simple_video.video_path.path
|
||||
simple_video.duration = 10 # don't update the duration during get_bulk_video_info_json()
|
||||
|
||||
info_json_path = video_path.with_suffix('.info.json')
|
||||
info_json_path.write_text(json.dumps({'title': 'info json title &'}))
|
||||
simple_video.info_json_path = info_json_path
|
||||
test_session.commit()
|
||||
|
||||
# simple_video does not come with a title.
|
||||
assert not simple_video.title
|
||||
|
||||
# Title is fetched from the info_json.
|
||||
await get_bulk_video_info_json([simple_video.id])
|
||||
assert simple_video.title == 'info json title &'
|
||||
|
||||
# No info_json means the filename is used.
|
||||
simple_video.title = None
|
||||
simple_video.info_json_path = None
|
||||
test_session.commit()
|
||||
await get_bulk_video_info_json([simple_video.id])
|
||||
assert simple_video.title == 'simple_video'
|
||||
|
||||
|
||||
def test_import_favorites(test_session, simple_channel, video_factory):
|
||||
"""
|
||||
A favorited Video will be preserved through everything (channel deletion, DB wipe) and can be imported and will be
|
||||
|
|
|
|||
|
|
@ -1,3 +1,6 @@
|
|||
import json
|
||||
import pathlib
|
||||
import shutil
|
||||
from datetime import datetime
|
||||
from unittest import mock
|
||||
|
||||
|
|
@ -6,10 +9,12 @@ import sqlalchemy
|
|||
|
||||
from modules.videos.channel import lib
|
||||
from modules.videos.common import apply_info_json
|
||||
from modules.videos.lib import validate_videos, parse_video_file_name
|
||||
from modules.videos.models import Channel, Video
|
||||
from modules.videos.video.lib import video_search
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.downloader import DownloadFrequency
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30
|
||||
|
||||
|
|
@ -108,3 +113,157 @@ def test_search_censored_videos(test_session, simple_channel):
|
|||
videos, total = video_search(filters=['censored'], order_by='id')
|
||||
assert [i['source_id'] for i in videos] == [str(i) for i in range(20)]
|
||||
assert total == 25
|
||||
|
||||
|
||||
def test_validate_videos(test_session, simple_channel, video_factory):
|
||||
"""
|
||||
Videos that aren't validated should have their data filled in while being validated.
|
||||
"""
|
||||
simple_channel.generate_posters = True
|
||||
|
||||
vid1_json = {'title': 'info's title'}
|
||||
vid3_json = {'duration': 100}
|
||||
vid5_json = {'view_count': 42}
|
||||
vid6_json = {'webpage_url': 'https://example.com/webpage', 'url': 'https://example.com/url'}
|
||||
|
||||
vid1 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg', with_info_json=vid1_json)
|
||||
vid2 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg', with_info_json=True)
|
||||
vid3 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg', with_info_json=vid3_json)
|
||||
vid4 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg', with_info_json=True)
|
||||
vid5 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg', with_info_json=vid5_json)
|
||||
vid6 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg', with_info_json=vid6_json)
|
||||
vid7 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='jpg')
|
||||
vid8 = video_factory(simple_channel.id, with_video_file=True, with_poster_ext='webp')
|
||||
vid9 = video_factory(simple_channel.id, with_video_file=True)
|
||||
|
||||
# These videos are missing this data, and a refresh should fill them in.
|
||||
vid1.title = None
|
||||
vid2.caption = None
|
||||
vid2.title = None
|
||||
vid3.duration = None
|
||||
vid4.size = None
|
||||
vid5.view_count = None
|
||||
vid6.url = None
|
||||
vid7.poster_path = None
|
||||
test_session.commit()
|
||||
|
||||
# Write the associated files. This data should be processed.
|
||||
vid2.caption_path = vid2.video_path.path.with_suffix('.en.vtt')
|
||||
shutil.copy(PROJECT_DIR / 'test/example1.en.vtt', vid2.caption_path)
|
||||
test_session.commit()
|
||||
|
||||
# All Videos are validated, all data is filled out. The info json data is trusted first.
|
||||
validate_videos()
|
||||
assert all([i.validated for i in test_session.query(Video)])
|
||||
assert vid1.title == "info's title"
|
||||
assert vid2.caption.startswith('okay welcome')
|
||||
assert 'mp4' not in vid2.title
|
||||
assert vid3.duration == 100
|
||||
assert vid4.size == 1055736
|
||||
assert vid5.view_count == 42
|
||||
assert vid6.url == 'https://example.com/webpage'
|
||||
assert str(vid7.poster_path).endswith('.jpg')
|
||||
assert str(vid8.poster_path).endswith('.jpg')
|
||||
assert str(vid9.poster_path).endswith('.jpg') # this was generated from the video file.
|
||||
|
||||
vid1.title = None
|
||||
vid3.duration = None
|
||||
vid5.view_count = None
|
||||
vid6.url = None
|
||||
vid9.poster_path = None
|
||||
test_session.commit()
|
||||
|
||||
# Validation does not happen because all are still validated.
|
||||
validate_videos()
|
||||
|
||||
# All videos need to be validated again.
|
||||
for i in test_session.query(Video):
|
||||
i.validated = False
|
||||
# Use backup methods to fetch data.
|
||||
vid1.info_json_path = None
|
||||
vid3.info_json_path = None
|
||||
vid5.info_json_path = None
|
||||
vid6.info_json_path.path.write_text(json.dumps({'url': 'https://example.com/url'}))
|
||||
test_session.commit()
|
||||
|
||||
validate_videos()
|
||||
assert all([i.validated for i in test_session.query(Video)])
|
||||
assert vid1.title != "info's title" # file name is used.
|
||||
assert vid3.duration == 5 # video file duration is used.
|
||||
assert vid5.view_count is None
|
||||
assert vid6.url == 'https://example.com/url' # url is used as backup url.
|
||||
# Posters are both JPEG format. Webp file is removed.
|
||||
assert vid7.poster_path.path.is_file()
|
||||
assert vid8.poster_path.path.is_file()
|
||||
assert not vid8.poster_path.path.with_suffix('.webp').is_file()
|
||||
assert str(vid9.poster_path).endswith('.jpg') # the generated file was rediscovered.
|
||||
|
||||
|
||||
def test_validate_video_exception(test_session, simple_channel, video_factory):
|
||||
"""
|
||||
Test that even if a Video cannot be validated, the other Videos will still be validated.
|
||||
"""
|
||||
vid1 = video_factory(simple_channel.id, with_video_file=True)
|
||||
vid2 = video_factory(simple_channel.id, with_video_file=True)
|
||||
# The validator should not even check this Video because it does not have a video file.
|
||||
vid3 = video_factory(simple_channel.id)
|
||||
vid3.video_path = None
|
||||
assert not all([i.validated for i in test_session.query(Video)])
|
||||
test_session.commit()
|
||||
|
||||
with mock.patch('modules.videos.lib.process_video_info_json') as mock_process_video_info_json:
|
||||
mock_process_video_info_json.side_effect = [
|
||||
Exception('skip this video'),
|
||||
(None, None, None, None),
|
||||
('should not be set to video 3', None, None, None), # This should not be called!
|
||||
]
|
||||
validate_videos()
|
||||
# Only first two videos are validated.
|
||||
assert mock_process_video_info_json.call_count == 2, mock_process_video_info_json.call_count
|
||||
test_session.commit()
|
||||
|
||||
assert not vid1.validated
|
||||
assert vid2.validated
|
||||
assert not vid3.validated
|
||||
|
||||
|
||||
@pytest.mark.parametrize('file_name,expected', [
|
||||
('channel_20000101_12345678910_ some title.mp4', ('channel', '20000101', '12345678910', 'some title')),
|
||||
('channel name_NA_12345678910_ some title.mp4', ('channel name', None, '12345678910', 'some title')),
|
||||
('20000101 foo.mp4', (None, None, None, '20000101 foo')),
|
||||
('20000101foo.mp4', (None, None, None, '20000101foo')),
|
||||
('something 20000101 foo.mp4', (None, None, None, 'something 20000101 foo')),
|
||||
('something_20000101_foo.mp4', ('something', '20000101', None, 'foo')),
|
||||
('foo .mp4', (None, None, None, 'foo')),
|
||||
('NA_20000303_vp91w5_Bob's Pancakes.mp4', (None, '20000303', 'vp91w5', 'Bob's Pancakes')),
|
||||
('NA_NA_vp91w5_Bob's Pancakes.mp4', (None, None, 'vp91w5', 'Bob's Pancakes')),
|
||||
])
|
||||
def test_parse_video_file_name(file_name, expected):
|
||||
"""
|
||||
A Video's title can be parsed from the video file.
|
||||
"""
|
||||
video_path = pathlib.Path(file_name)
|
||||
assert parse_video_file_name(video_path) == expected, f'{file_name} != {expected}'
|
||||
|
||||
|
||||
def test_validate_does_not_generate_when_disabled(test_session, channel_factory, video_factory):
|
||||
"""
|
||||
A poster should not be generated if the Video's Channel forbids it.
|
||||
"""
|
||||
channel1 = channel_factory() # posters should not be generated by default.
|
||||
channel2 = channel_factory()
|
||||
channel2.generate_posters = True
|
||||
vid1 = video_factory(channel1.id, with_video_file=True)
|
||||
vid2 = video_factory(channel2.id, with_video_file=True)
|
||||
# webp will not be converted unless generate_posters is enabled.
|
||||
vid3 = video_factory(channel1.id, with_video_file=True, with_poster_ext='webp')
|
||||
test_session.commit()
|
||||
assert not vid1.poster_path
|
||||
assert not vid2.poster_path
|
||||
assert str(vid3.poster_path).endswith('webp')
|
||||
|
||||
validate_videos()
|
||||
|
||||
assert not vid1.poster_path
|
||||
assert vid2.poster_path
|
||||
assert str(vid3.poster_path).endswith('webp')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue