From 86893ef7f1c844b92bf54583f0da0262d3a436cb Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Fri, 14 Jan 2022 15:54:33 -0700 Subject: [PATCH] Feature/video validate (#74) * Videos are marked as validated so they will not be tried again. All previous bulk methods are combined into one. * Passing json to fixture factory to avoid writing the files in every test. * Adding log to explain if refresh is slow. * Generating video poster if one cannot be found. * Generating video poster if no poster can be found. * Videos without a video file (yet to be downloaded) are not validated. * Video title is extracted from the video file name as a last resort. * Keeping existing view count because it may be from the channel, which will be more recent. * Downloads should only be downloaded after they are due, not the same day. * Setting manager on new downloads. * Checking if downloading is disabled in loop. * Adding log when refresh is complete. * Not generating posters by default. This is to avoid deleting files. * Not converting posters unless channel has that enabled. * Not logging on valid because all posters will be checked. * Validating videos in upsert. * Using video name parser in upsert. * Using validate_video in upsert so that data is only read once from disk. * Attaching manager in download_manager.get_download. * Adding 22 to preferred video format for odd 720p resolution. Signed-off-by: Roland --- .../versions/3c33b1dae069_video_validated.py | 35 ++ modules/videos/captions.py | 21 +- modules/videos/common.py | 118 +----- modules/videos/conftest.py | 35 +- modules/videos/downloader.py | 4 +- modules/videos/lib.py | 359 +++++++++--------- modules/videos/models.py | 18 +- modules/videos/test/test_captions.py | 4 +- modules/videos/test/test_common.py | 108 +----- modules/videos/test/test_lib.py | 159 ++++++++ wrolpi/downloader.py | 29 +- wrolpi/media_path.py | 3 + wrolpi/root_api.py | 1 - wrolpi/test/test_downloader.py | 4 +- 14 files changed, 464 insertions(+), 434 deletions(-) create mode 100644 alembic/versions/3c33b1dae069_video_validated.py diff --git a/alembic/versions/3c33b1dae069_video_validated.py b/alembic/versions/3c33b1dae069_video_validated.py new file mode 100644 index 00000000..5f6aa7ae --- /dev/null +++ b/alembic/versions/3c33b1dae069_video_validated.py @@ -0,0 +1,35 @@ +"""Video validated. + +Revision ID: 3c33b1dae069 +Revises: f2c6326cad29 +Create Date: 2022-01-13 16:14:23.409242 + +""" +import os + +from alembic import op +from sqlalchemy.orm import Session + +# revision identifiers, used by Alembic. +revision = '3c33b1dae069' +down_revision = 'f2c6326cad29' +branch_labels = None +depends_on = None + +DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False + + +def upgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE video ADD COLUMN validated BOOLEAN DEFAULT FALSE') + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated_poster') + # Generating posters is slow, and may delete files. + session.execute('ALTER TABLE channel ALTER COLUMN generate_posters SET DEFAULT false') + + +def downgrade(): + bind = op.get_bind() + session = Session(bind=bind) + session.execute('ALTER TABLE video DROP COLUMN IF EXISTS validated') + session.execute('ALTER TABLE video ADD COLUMN validated_poster BOOLEAN DEFAULT FALSE') diff --git a/modules/videos/captions.py b/modules/videos/captions.py index f44d8235..5b60d83b 100755 --- a/modules/videos/captions.py +++ b/modules/videos/captions.py @@ -1,13 +1,12 @@ #! /usr/bin/env python3 import pathlib from pathlib import Path -from typing import Generator, List, Union +from typing import Generator, Union import srt import webvtt -from wrolpi.common import logger, chunks -from wrolpi.db import get_db_session +from wrolpi.common import logger from wrolpi.media_path import MediaPath from .models import Video @@ -39,9 +38,9 @@ def get_unique_caption_lines(caption_path: Union[str, Path]) -> Generator: yield line -def process_captions(video: Video): +def get_video_captions(video: Video): """ - Parse and insert captions for a video record. + Parse video captions from the video's captions file. """ if isinstance(video.caption_path, MediaPath): caption_path = video.caption_path.path @@ -51,8 +50,7 @@ def process_captions(video: Video): try: lines = get_unique_caption_lines(str(caption_path)) block = '\n'.join(lines) - video.caption = block - return + return block except UnicodeDecodeError: # Failed to decode the caption file # TODO handle this error @@ -65,12 +63,3 @@ def process_captions(video: Video): pass logger.debug(f'Failed to parse caption file {caption_path}') - - -async def insert_bulk_captions(video_ids: List[int]): - for video_ids in chunks(video_ids, 10): - with get_db_session(commit=True) as session: - for idx, video_id in enumerate(video_ids): - video = session.query(Video).filter_by(id=video_id).one() - process_captions(video) - logger.debug(f'Inserted {len(video_ids)} captions') diff --git a/modules/videos/common.py b/modules/videos/common.py index c0553b66..b558628d 100644 --- a/modules/videos/common.py +++ b/modules/videos/common.py @@ -1,4 +1,4 @@ -import html +import json import json import os import pathlib @@ -14,7 +14,7 @@ from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from wrolpi import before_startup -from wrolpi.common import logger, get_config, iterify, chunks, get_media_directory, \ +from wrolpi.common import logger, get_config, iterify, get_media_directory, \ minimize_dict, api_param_limiter from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import UnknownFile, UnknownDirectory, ChannelNameConflict, ChannelURLConflict, \ @@ -342,28 +342,12 @@ def generate_video_poster(video_path: Path) -> Path: logger.warning(f'FFMPEG poster generation failed with stdout: {e.stdout.decode()}') logger.warning(f'FFMPEG poster generation failed with stdout: {e.stderr.decode()}') raise + if not poster_path.exists(): + raise Exception(f'Failed to find generated poster! {poster_path}') + return poster_path -async def generate_bulk_posters(video_ids: List[int]): - """ - Generate all posters for the provided videos. Update the video object with the new jpg file location. Do not - clobber existing jpg files. - """ - logger.info(f'Generating {len(video_ids)} video posters') - for video_ids in chunks(video_ids, 10): - with get_db_session(commit=True) as session: - videos = session.query(Video).filter(Video.id.in_(video_ids)) - for video in videos: - video_path = video.video_path.path - - poster_path = replace_extension(video_path, '.jpg') - if not poster_path.exists(): - generate_video_poster(video_path) - video.poster_path = poster_path - logger.info('Done generating video posters') - - def convert_image(existing_path: Path, destination_path: Path, ext: str = 'jpeg'): """ Convert an image from one format to another. Remove the existing image file. This will safely overwrite an image @@ -391,42 +375,6 @@ def is_valid_poster(poster_path: Path) -> bool: return False -def bulk_validate_posters(video_ids: List[int]): - """ - Replace all posters for the provided videos if a video's poster is not a JPEG format. - """ - logger.info(f'Validating {len(video_ids)} video posters') - for video_ids in chunks(video_ids, 10): - for video_id in video_ids: - with get_db_session(commit=True) as session: - video: Video = session.query(Video).filter_by(id=video_id).one() - - poster_path: Path = video.poster_path.path - new_poster_path = poster_path.with_suffix('.jpg') - - if poster_path != new_poster_path and new_poster_path.exists(): - # Destination JPEG already exists (it may have the wrong format), lets overwrite it with a valid - # JPEG. - poster_path.unlink() - poster_path = new_poster_path - - if not is_valid_poster(poster_path): - # Poster is not valid, convert it and place it in the new location. - try: - convert_image(poster_path, new_poster_path) - logger.info(f'Converted invalid poster {poster_path} to {new_poster_path}') - except Exception as e: - logger.error(f'Failed to convert invalid poster {poster_path} to {new_poster_path}', exc_info=e) - continue - else: - logger.debug(f'Poster was already valid: {new_poster_path}') - - # Update the video with the new poster path. Mark it as validated. - video.poster_path = new_poster_path - video.validated_poster = True - logger.info('Done validating video posters.') - - def get_video_duration(video_path: Path) -> int: """ Get the duration of a video in seconds. Do this using ffprobe. @@ -450,62 +398,6 @@ def get_video_duration(video_path: Path) -> int: return duration -async def get_bulk_video_info_json(video_ids: List[int]): - """ - Get and save the info_json data for each video provided. - """ - logger.info(f'Getting {len(video_ids)} video info_json meta data.') - for video_ids in chunks(video_ids, 10): - with get_db_session(commit=True) as session: - for video_id in video_ids: - video = session.query(Video).filter_by(id=video_id).one() - logger.debug(f'Getting video info_json data: {video}') - - if not video.video_path: - logger.warning(f'Refusing to get info_json for video without a video file: {video}') - continue - - try: - info_json = video.get_info_json() - if info_json: - if not video.view_count and info_json.get('view_count'): - video.view_count = info_json['view_count'] - if not video.duration: - # Get duration from info_json without reading video file. - video.duration = info_json.get('duration') - # Trust the info_json title before the video path title. - video.title = info_json.get('title') - if not video.duration: - # As a last resort, get duration from the video file. - video.duration = get_video_duration(video.video_path.path) - if not video.title: - # Use the video path name without the extension. - video.title = video.video_path.path.with_suffix('').name - - # Unescape any HTML entities. - video.title = html.unescape(video.title) - - except Exception: - logger.warning(f'Unable to get meta data of {video}', exc_info=True) - logger.info('Done getting video info_json meta data.') - - -async def get_bulk_video_size(video_ids: List[int]): - """ - Get and save the size for each video provided. - """ - logger.info(f'Getting {len(video_ids)} video sizes.') - for video_ids in chunks(video_ids, 10): - with get_db_session(commit=True) as session: - for video_id in video_ids: - video = session.query(Video).filter_by(id=video_id).one() - logger.debug(f'Getting video size: {video.id} {video.video_path}') - - size = video.video_path.path.stat().st_size - video.size = size - logger.info('Done getting video sizes') - - def apply_info_json(channel_id: int): """ Update view_count for all Videos in a channel using its info_json file. Mark any videos not in the info_json as diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py index b3026c33..c4762c1c 100644 --- a/modules/videos/conftest.py +++ b/modules/videos/conftest.py @@ -1,9 +1,11 @@ +import json import pathlib import shutil from uuid import uuid4 import mock import pytest +from PIL import Image from modules.videos.channel.lib import spread_channel_downloads from modules.videos.downloader import VideoDownloader, ChannelDownloader @@ -99,13 +101,38 @@ def video_file(test_directory) -> pathlib.Path: @pytest.fixture def video_factory(test_session, test_directory): - def _(channel_id: int = None): + """ + Creates Videos for testing. + """ + + def _(channel_id: int = None, with_video_file: bool = False, with_info_json: dict = None, + with_poster_ext: str = None): title = str(uuid4()) if channel_id: - path = str(test_directory / f'{title}.mp4') + path = test_directory / f'{title}.mp4' else: - path = str(test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4') - video = Video(video_path=path, title=title, channel_id=channel_id, source_id=title) + path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4' + + if with_video_file: + shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path) + + info_json_path = None + if with_info_json is True: + # User requests a info json file, but does not provide one. + info_json_path = path.with_suffix('.info.json') + info_json_path.write_text(json.dumps({})) + elif with_info_json: + # Use the provided object as the json. + info_json_path = path.with_suffix('.info.json') + info_json_path.write_text(json.dumps(with_info_json)) + + poster_path = None + if with_poster_ext: + poster_path = path.with_suffix(f'.{with_poster_ext}') + Image.new('RGB', (25, 25)).save(poster_path) + + video = Video(video_path=str(path), title=title, channel_id=channel_id, source_id=title, + info_json_path=info_json_path, poster_path=poster_path) test_session.add(video) return video diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index e56da197..56d74c71 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -37,7 +37,7 @@ ChannelIEs = { } PREFERRED_VIDEO_EXTENSION = 'mp4' -PREFERRED_VIDEO_FORMAT = 'best[height=720],mp4-480p,mp4-360p,mp4-240p' +PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p' def get_no_channel_directory(): @@ -171,7 +171,7 @@ class VideoDownloader(Downloader, ABC): return_code = self.process_runner(url, cmd, out_dir) if return_code != 0: - raise ValueError(f'youtube-dl exited with {return_code}') + raise ValueError(f'video downloader process exited with {return_code}') if not video_path.is_file(): raise ValueError(f'Video file could not be found! {video_path}') diff --git a/modules/videos/lib.py b/modules/videos/lib.py index 066805e0..45c860e6 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -1,10 +1,8 @@ -import asyncio import html -import json import pathlib import re from collections import defaultdict -from typing import Tuple +from typing import Tuple, Optional from uuid import uuid1 from sqlalchemy.orm import Session @@ -12,11 +10,11 @@ from sqlalchemy.orm import Session from wrolpi.common import logger, chunks, get_config from wrolpi.common import save_settings_config from wrolpi.db import get_db_curs, get_db_session, optional_session +from wrolpi.media_path import MediaPath from wrolpi.vars import PYTEST -from .captions import insert_bulk_captions, process_captions -from .common import generate_bulk_posters, get_bulk_video_info_json, get_bulk_video_size, \ - generate_video_paths, remove_duplicate_video_paths, bulk_validate_posters, \ - apply_info_json, import_videos_config +from .captions import get_video_captions +from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, import_videos_config, \ + get_video_duration, is_valid_poster, convert_image, generate_video_poster from .models import Channel, Video logger = logger.getChild(__name__) @@ -24,115 +22,6 @@ logger = logger.getChild(__name__) DEFAULT_DOWNLOAD_FREQUENCY = 60 * 60 * 24 * 7 # weekly -def refresh_channel_video_captions() -> bool: - with get_db_curs() as curs: - query = 'SELECT id FROM video WHERE caption IS NULL AND caption_path IS NOT NULL' - curs.execute(query) - missing_captions = [i for (i,) in curs.fetchall()] - - if missing_captions: - coro = insert_bulk_captions(missing_captions) - asyncio.ensure_future(coro) - logger.info('Scheduled insert_bulk_captions()') - return True - else: - logger.info('No missing captions to process.') - return False - - -def refresh_channel_generate_posters() -> bool: - with get_db_curs() as curs: - query = 'SELECT id FROM video WHERE video_path IS NOT NULL AND poster_path IS NULL' - curs.execute(query) - missing_posters = [i for (i,) in curs.fetchall()] - - if missing_posters: - coro = generate_bulk_posters(missing_posters) - asyncio.ensure_future(coro) - logger.info('Scheduled generate_bulk_posters()') - return True - else: - logger.info('No missing posters to generate.') - return False - - -def convert_invalid_posters() -> bool: - """ - Searches the DB for all videos with an invalid poster type (i.e. webp) and converts them to JPEGs. A video with a - valid poster will be marked as such in it's column "validated_poster". - """ - with get_db_curs() as curs: - query = "SELECT id FROM video WHERE poster_path IS NOT NULL AND validated_poster = FALSE" - curs.execute(query) - invalid_posters = [i for (i,) in curs.fetchall()] - - if invalid_posters: - async def _(): - return bulk_validate_posters(invalid_posters) - - coro = _() - asyncio.ensure_future(coro) - logger.info('Scheduled bulk_replace_invalid_posters()') - return True - else: - logger.info('No invalid posters to replace.') - return False - - -def refresh_channel_info_json() -> bool: - """ - Fill in Video columns that are extracted from the info_json. - """ - with get_db_curs() as curs: - query = ''' - SELECT v.id - FROM video v - WHERE - v.video_path IS NOT NULL - AND v.info_json_path IS NOT NULL - AND v.info_json_path != '' - AND (v.duration IS NULL OR v.view_count IS NULL OR v.title IS NULL) - ''' - curs.execute(query) - missing_duration = [i for (i,) in curs.fetchall()] - - if missing_duration: - coro = get_bulk_video_info_json(missing_duration) - asyncio.ensure_future(coro) - logger.info('Scheduled get_bulk_video_info_json()') - return True - else: - logger.info('No videos need updating from info_json.') - return False - - -def refresh_channel_calculate_size() -> bool: - with get_db_curs() as curs: - query = 'SELECT id FROM video WHERE video_path IS NOT NULL AND size IS NULL' - curs.execute(query) - missing_size = [i for (i,) in curs.fetchall()] - - if missing_size: - coro = get_bulk_video_size(missing_size) - asyncio.ensure_future(coro) - logger.info('Scheduled get_bulk_video_size()') - return True - else: - logger.info('No videos missing size.') - return False - - -def process_video_meta_data(): - """ - Search for any videos missing meta data, fill in that data. - """ - refresh_channel_video_captions() - refresh_channel_generate_posters() - convert_invalid_posters() - refresh_channel_info_json() - refresh_channel_calculate_size() - - def refresh_channel_videos(channel: Channel): """ Find all video files in a channel's directory. Add any videos not in the DB to the DB. @@ -222,6 +111,136 @@ def refresh_no_channel_videos(): logger.info(deleted_status) +def process_video_info_json(video: Video): + """ + Parse the Video's info json file, return the relevant data. + """ + title = duration = view_count = url = None + if info_json := video.get_info_json(): + title = info_json.get('fulltitle') or info_json.get('title') + title = html.unescape(title) if title else None + + duration = info_json.get('duration') + view_count = info_json.get('view_count') + url = info_json.get('webpage_url') or info_json.get('url') + + return title, duration, view_count, url + + +def validate_videos(): + """ + Validate all Videos not yet validated. A Video is validated when we have attempted to find its: title, duration, + view_count, url, caption, size. A Video is also valid when it has a JPEG poster, if any. If no poster can be + found, it will be generated from the video file. + + This function marks the Video as validated, even if no data can be found so a Video will not be validated multiple + times. + """ + with get_db_curs() as curs: + curs.execute('SELECT id FROM video WHERE video_path IS NOT NULL AND validated IS FALSE') + video_ids = [i['id'] for i in curs.fetchall()] + curs.execute('SELECT id, generate_posters FROM channel') + channel_generate_posters = {i['id']: i['generate_posters'] for i in curs.fetchall()} + + logger.info(f'Validating {len(video_ids)} videos.') + for chunk in chunks(video_ids, 20): + with get_db_session(commit=True) as session: + videos = session.query(Video).filter(Video.id.in_(chunk)).all() + for video in videos: + try: + channel_generate_poster = channel_generate_posters.get(video.channel_id) + validate_video(video, channel_generate_poster) + # All data about the Video has been found, we should not attempt to validate it again. + video.validated = True + except Exception as e: + # This video failed to validate, continue validation for the rest of the videos. + logger.warning(f'Failed to validate {video=}', exc_info=e) + + +def validate_video(video: Video, channel_generate_poster: bool): + """ + Validate a single video. A Video is validated when we have attempted to find its: title, duration, + view_count, url, caption, size. A Video is also valid when it has a JPEG poster, if any. If no poster can be + found, it will be generated from the video file. + """ + if not video.title or not video.duration or not video.view_count or not video.url: + # These properties can be found in the info json. + title, duration, view_count, url = process_video_info_json(video) + video.title = title + video.duration = duration + video.url = url + # View count will probably be overwritten by more recent data when this Video's Channel is + # updated. + video.view_count = video.view_count or view_count + + video_path = video.video_path.path if isinstance(video.video_path, MediaPath) else video.video_path + + if not video.title or not video.upload_date or not video.source_id: + # Video is missing things that can be extracted from the video file name. + # These are the least trusted, so anything already on the video should be trusted. + _, upload_date, source_id, title = parse_video_file_name(video_path) + video.title = video.title or html.unescape(title) + video.upload_date = video.upload_date or upload_date + video.source_id = video.source_id or source_id + if not video.duration: + # Video duration was not in the info json, use ffprobe. + video.duration = get_video_duration(video_path) + if not video.caption and video.caption_path: + video.caption = get_video_captions(video) + if not video.size: + video.size = video_path.stat().st_size + if not video.poster_path: + # Video poster is not found, lets check near the video file. + for ext in ('.jpg', '.jpeg', '.webp', '.png'): + if (poster_path := video_path.with_suffix(ext)).is_file(): + video.poster_path = poster_path + break + if channel_generate_poster: + # Try to convert/generate, but keep the old poster if those fail. + video.poster_path = convert_or_generate_poster(video) or video.poster_path + + +def convert_or_generate_poster(video: Video) -> Optional[pathlib.Path]: + """ + If a Video has a poster, but the poster is invalid, convert it. If a Video has no poster, generate one from the + video file. + + Returns None if the poster was not converted, and not generated. + """ + video_path = video.video_path.path + # Modification/generation of poster is enabled for this channel. + if video.poster_path: + # Check that the poster is a more universally supported JPEG. + old: pathlib.Path = video.poster_path.path if \ + isinstance(video.poster_path, MediaPath) else video.poster_path + new = old.with_suffix('.jpg') + + if old != new and new.exists(): + # Destination JPEG already exists (it may have the wrong format). + old.unlink() + old = video.poster_path = new + + if not is_valid_poster(old): + # Poster is not valid, convert it and place it in the new location. + try: + convert_image(old, new) + old.unlink(missing_ok=True) + logger.info(f'Converted invalid poster {old} to {new}') + return new + except Exception as e: + logger.error(f'Failed to convert invalid poster {old} to {new}', exc_info=e) + return + + if not video.poster_path: + # Video poster was not discovered, or converted. Let's generate it. + try: + poster_path = generate_video_poster(video_path) + logger.debug(f'Generated poster for {video}') + return poster_path + except Exception as e: + logger.error(f'Failed to generate poster for {video}', exc_info=e) + + def _refresh_videos(channel_links: list = None): """ Find any videos in the channel directories and add them to the DB. Delete DB records of any videos not in the @@ -259,7 +278,9 @@ def _refresh_videos(channel_links: list = None): # Fill in any missing data for all videos. if not PYTEST: import_videos_config() - process_video_meta_data() + validate_videos() + + logger.info('Refresh of video files complete') def get_channels_config(session: Session) -> dict: @@ -364,12 +385,37 @@ async def get_statistics(): return ret -NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{11})_)?(.*)\.' +NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.+?)_)?(.*)\.' r'(jpg|webp|flv|mp4|part|info\.json|description|webm|..\.srt|..\.vtt)') +def parse_video_file_name(video_path: pathlib.Path) -> \ + Tuple[Optional[str], Optional[str], Optional[str], Optional[str]]: + """ + A Video's file name can have data in it, this attempts to extract what may be there. + + Example: {channel_name}_{upload_date}_{source_id}_title{ext} + """ + video_str = str(video_path) + if match := NAME_PARSER.match(video_str): + channel, date, source_id, title, _ = match.groups() + + channel = None if channel == 'NA' else channel + date = None if date == 'NA' else date + + title = title.strip() + if date == 'NA': + return channel, None, source_id, title + if date is None or (len(date) == 8 and date.isdigit()): + return channel, date, source_id, title + + # Return the stem as a last resort + title = pathlib.Path(video_path).stem.strip() + return None, None, None, title + + def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = None, idempotency: str = None, - skip_captions=False, id_: str = None) -> Video: + id_: str = None) -> Video: """ Insert a video into the DB. Also, find any meta-files near the video file and store them on the video row. @@ -377,66 +423,37 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel = """ if not video_path.is_absolute(): raise ValueError(f'Video path is not absolute: {video_path}') + + # This function can update or insert a Video. + video = session.query(Video).filter_by(id=id_).one() if id_ else Video() + + # Set the file values, all other things can be found using these files. poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path) + video.caption_path = caption_path + video.description_path = description_path + video.info_json_path = info_json_path + video.poster_path = poster_path + video.video_path = video_path - name_match = NAME_PARSER.match(video_path.name) - _ = upload_date = source_id = title = ext = None - if name_match: - _, upload_date, source_id, title, ext = name_match.groups() + if channel and not video_path.is_relative_to(channel.directory.path): + raise ValueError(f'Video path is not within its channel {video_path=} {channel.directory=}') - # Make sure the date is a valid date format, if not, leave it blank. Youtube-DL sometimes puts an NA in the date. - # We may even get videos that weren't downloaded by WROLPi. - if not upload_date or not upload_date.isdigit() or len(upload_date) != 8: - logger.debug(f'Could not parse date from filename: {video_path}') - upload_date = None + video.channel = channel + video.idempotency = idempotency - duration = None - url = None - if info_json_path: - try: - with info_json_path.open('rt') as fh: - json_contents = json.load(fh) - duration = json_contents.get('duration') - url = json_contents.get('webpage_url') - # Trust the info_json title before the video filename. - title = json_contents.get('title', title) - except json.decoder.JSONDecodeError: - logger.warning(f'Failed to load JSON file to get duration: {info_json_path}') - - size = video_path.stat().st_size - title = html.unescape(title) if title else None - - video_dict = dict( - caption_path=str(caption_path) if caption_path else None, - channel_id=channel.id if channel else None, - description_path=str(description_path) if description_path else None, - duration=duration, - ext=ext, - idempotency=idempotency, - info_json_path=str(info_json_path) if info_json_path else None, - poster_path=str(poster_path) if poster_path else None, - size=size, - source_id=source_id, - title=title, - upload_date=upload_date, - url=url, - video_path=str(video_path), - ) - - if id_: - video = session.query(Video).filter_by(id=id_).one() - for key, value in video_dict.items(): - setattr(video, key, value) - else: - video = Video(**video_dict) + try: + # Fill in any missing data. Generate poster if enabled and necessary. + validate_video(video, channel.generate_posters if channel else False) + # All data about the Video has been found, we should not attempt to validate it again. + video.validated = True + except Exception: + # Could not validate, this could be an issue with a file. This should not prevent the video from being + # inserted. + pass session.add(video) session.flush() - if skip_captions is False and caption_path: - # Process captions only when requested - process_captions(video) - return video diff --git a/modules/videos/models.py b/modules/videos/models.py index 507aca77..0e70f1d1 100644 --- a/modules/videos/models.py +++ b/modules/videos/models.py @@ -7,7 +7,7 @@ from sqlalchemy import Column, Integer, String, Boolean, JSON, Date, ARRAY, Fore from sqlalchemy.orm import relationship, Session, deferred from sqlalchemy.orm.collections import InstrumentedList -from wrolpi.common import Base, tsvector, ModelHelper, logger +from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directory from wrolpi.dates import now, TZDateTime from wrolpi.db import get_db_curs from wrolpi.downloader import Download @@ -23,6 +23,7 @@ class Video(ModelHelper, Base): channel_id = Column(Integer, ForeignKey('channel.id')) channel = relationship('Channel', primaryjoin='Video.channel_id==Channel.id', back_populates='videos') idempotency = Column(String) + validated = Column(Boolean, default=False) # File paths caption_path = Column(MediaPathType) @@ -33,17 +34,16 @@ class Video(ModelHelper, Base): video_path = Column(MediaPathType) caption = deferred(Column(String)) # slow to fetch + censored = Column(Boolean, default=False) duration = Column(Integer) favorite = Column(TZDateTime) size = Column(Integer) source_id = Column(String) title = Column(String) upload_date = Column(TZDateTime) - validated_poster = Column(Boolean, default=False) - viewed = Column(TZDateTime) - view_count = Column(Integer) url = Column(String) - censored = Column(Boolean, default=False) + view_count = Column(Integer) + viewed = Column(TZDateTime) textsearch = deferred( Column(tsvector, Computed('''to_tsvector('english'::regconfig, @@ -51,7 +51,9 @@ class Video(ModelHelper, Base): COALESCE(caption, ''::text)))'''))) def __repr__(self): - video_path = self.video_path.path if self.video_path else None + video_path = self.video_path + if self.video_path and isinstance(self.video_path, MediaPath): + video_path = self.video_path.path.relative_to(get_media_directory()) return f'