#! /usr/bin/env python3 import json import os.path import pathlib import traceback from abc import ABC from typing import Tuple, List, Dict import yt_dlp.utils from sqlalchemy.orm import Session from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE # noqa from wrolpi.cmd import YT_DLP_BIN from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \ format_json_file, trim_file_name from wrolpi.dates import now from wrolpi.db import get_db_session from wrolpi.db import optional_session from wrolpi.downloader import Downloader, Download, DownloadResult from wrolpi.errors import UnrecoverableDownloadError from wrolpi.files.lib import glob_shared_stem from wrolpi.files.models import FileGroup from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR from .channel.lib import create_channel, get_channel from .common import get_no_channel_directory, get_videos_directory, update_view_counts_and_censored, \ ffmpeg_video_complete from .errors import UnknownChannel from .lib import get_downloader_config, YDL, ydl_logger from .models import Video, Channel from .normalize_video_url import normalize_video_url from .schema import ChannelPostRequest from .video.lib import download_video_info_json logger = logger.getChild(__name__) # Channels are handled differently than a single video. ChannelIEs = { YoutubeTabIE, } PREFERRED_VIDEO_EXTENSION = 'mp4' PREFERRED_VIDEO_FORMAT = ','.join([ 'res:720', # Use the best 720p available first '136+140', # 130=720p video-only, 140=medium quality audio only '22', # 720p video with audio '135+140', # 135=480p video-only, 140=medium quality audio only '135+139', # 135=480p video-only, 139=low quality audio only 'mp4-480p', 'bestvideo*+bestaudio/best', # Download the highest resolution as a last resort (can be quite large). ]) PREFERRED_CAPTION_FORMAT = 'vtt' PREFERRED_POSTER_FORMAT = 'jpg' def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict: """Get info about a video. Separated for testing.""" if PYTEST: raise RuntimeError(f'Refusing to download {url} during testing! {ydl}') return ydl.extract_info(url, download=False, process=process) def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str: """Get filename from YoutubeDL. Separated for testing.""" dir_name, file_name = os.path.split(ydl.prepare_filename(entry)) file_name = trim_file_name(file_name) return f'{dir_name}/{file_name}' class ChannelDownloader(Downloader, ABC): """Handles downloading of videos in a Channel or Playlist.""" name = 'video_channel' pretty_name = 'Video Channel' listable = False def __repr__(self): return f'' @staticmethod def is_a_playlist(info: dict): # A playlist may have an id different from its channel. return info['id'] != info.get('channel_id') async def do_download(self, download: Download) -> DownloadResult: """Update a Channel's catalog, then schedule downloads of every missing video.""" info = extract_info(download.url, process=False) # Resolve the "entries" generator. info: Dict = resolve_generators(info) entries = info.get('entries') if entries and isinstance(entries[0], Dict) and entries[0].get('entries'): # Entries contains more channels/playlist, tell the Maintainer to specify the URL. raise UnrecoverableDownloadError(f'Unable to download {download.url} because the URL is ambiguous.' f' Try requesting the videos only:' f' https://example.com -> https://example.com/videos') download.sub_downloader = video_downloader.name download.info_json = info if session := Session.object_session(download): # May not have a session during testing. session.commit() name = info.get('uploader') or info.get('webpage_url_basename') if not name: raise ValueError(f'Could not find name') channel_source_id = info.get('channel_id') or info.get('id') channel = get_or_create_channel(channel_source_id, download.url, name) channel.dict() # get all attributes while we have the session. location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None # The settings to send to the VideoDownloader. settings = dict() if channel: settings.update(dict(channel_id=channel.id, channel_url=download.url)) # Downloads will inherit the destination, if defined. destination = download.settings.get('destination') if download.settings else None if destination: settings['destination'] = destination # A Download may have applied tags, send any to the VideoDownloader. tag_names = download.settings.get('tag_names') if download.settings else None if tag_names: settings['tag_names'] = tag_names is_a_playlist = self.is_a_playlist(info) try: if not is_a_playlist: await self.prepare_channel_for_downloads(download, channel) else: logger.debug('Not updating channel because this is a playlist') downloads = self.get_missing_videos(download) return DownloadResult( success=True, location=location, downloads=downloads, settings=settings, ) except Exception: if PYTEST: raise kind = 'playlist' if is_a_playlist else 'channel' logger.warning(f'Failed to update catalog of {kind} {download.url}') return DownloadResult( success=False, location=location, error=str(traceback.format_exc()), settings=settings, ) @classmethod async def prepare_channel_for_downloads(cls, download: Download, channel: Channel): """Update the Channel's video catalog. Refresh the Channel's files if necessary.""" logger.debug(f'Preparing {channel} for downloads') channel_id = channel.id update_channel_catalog(channel, download.info_json) with get_db_session() as session: channel = get_channel(session, channel_id=channel_id, return_dict=False) if not channel.videos or not channel.refreshed: logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!') await channel.refresh_files(send_events=False) session.commit() # Commit the refresh. session.refresh(channel) @staticmethod def get_missing_videos(download: Download) -> List[str]: """ Return all URLs of Videos in the `info_json` which need to be downloaded. """ downloads = download.info_json['entries'] total_downloads = len(downloads) title_include = download.settings.get('title_include') if download.settings else None if title_include: # Only download Videos that have title match words in title. title_include = [i.strip() for i in title_include.split(',')] new_downloads = [] for i in downloads: if (title := i.get('title', '')) and any(j for j in title_include if j in title): logger.debug(f'Video with title {title} matches {title_include}') new_downloads.append(i) downloads = new_downloads title_exclude = download.settings.get('title_exclude') if download.settings else None if title_exclude: # Do not download any videos that contain the exclude words. title_exclude = [i.strip() for i in title_exclude.split(',')] new_downloads = list() for i in downloads: if (title := i.get('title', '')) and any(j for j in title_exclude if j in title): logger.debug(f'Video with title {title} matches EXCLUDE {title_exclude}') else: new_downloads.append(i) downloads = new_downloads filtered_entries = len(downloads) if filtered_entries != total_downloads: logger.info(f'Downloaded channel. Total downloaded reduced to {filtered_entries} from {total_downloads}') # Prefer `webpage_url` before `url` for all entries. downloads = [i.get('webpage_url') or i.get('url') for i in downloads] # YouTube Shorts are handled specially. downloads = [normalize_video_url(i) for i in downloads] # Only download those that have not yet been downloaded. already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)] downloads = [i for i in downloads if i not in already_downloaded] return downloads class VideoDownloader(Downloader, ABC): """Downloads a single video. Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`. Videos are validated in a few ways. A video file is valid when it has a complete video stream and an audio stream. A complete video stream is validated by taking a screenshot at the end of the video. """ name = 'video' pretty_name = 'Videos' def __repr__(self): return f'' @optional_session def already_downloaded(self, *urls: str, session: Session = None) -> List: # We only consider a video record with a video file as "downloaded". file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video')) return file_groups async def do_download(self, download: Download) -> DownloadResult: if download.attempts >= 10: raise UnrecoverableDownloadError('Max download attempts reached') download_id = download.id url = normalize_video_url(download.url) # Video may have been downloaded previously, get its location for error reporting. with get_db_session() as session: videos = session.query(Video).join(FileGroup).filter_by(url=url).all() location = videos[0].location if videos else None try: download.info_json = download.info_json or extract_info(url) except yt_dlp.utils.DownloadError as e: # Video may be private. return DownloadResult( success=False, location=location, error='\n'.join(traceback.format_exception(e)), ) if not download.info_json: raise ValueError(f'Cannot download video with no info_json.') found_channel = None channel_name = download.info_json.get('channel') source_channel_id = download.info_json.get('channel_id') channel_url = download.info_json.get('channel_url') channel = None if channel_name or source_channel_id: # Try to find the channel via info_json from yt-dlp. try: channel = get_or_create_channel(source_id=source_channel_id, url=channel_url, name=channel_name) if channel: found_channel = 'yt_dlp' logger.debug(f'Found a channel with source_id {channel=}') except UnknownChannel: # Can't find a channel, use the no channel directory. pass settings = download.settings or dict() destination = settings.get('destination') if not channel and destination: # Destination may override the real Channel. try: channel = get_channel(directory=destination, return_dict=False) found_channel = 'download_settings_directory' logger.debug(f'Found a channel that shares the destination directory {channel=}') except UnknownChannel: # Destination must not be a channel. pass local_channel_id = settings.get('channel_id') channel_url = settings.get('channel_url') if not channel and (local_channel_id or channel_url): # Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download. logger.info(f'Using download.settings to find channel') try: channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False) logger.debug(f'Found channel with channel url {channel=}') except UnknownChannel: # We do not need a Channel if we have a destination directory. if not destination: raise if channel: found_channel = 'download_settings' channel_id = channel.id if channel else None channel_directory = channel.directory if channel else None if found_channel == 'yt_dlp': logger.debug(f'Found {channel} using yt_dlp') elif found_channel == 'download_settings': logger.info(f'Found {channel} using Download.settings') elif found_channel == 'download_settings_directory': logger.info(f'Found {channel} using Download.settings directory') else: logger.warning('Could not find channel') if destination: # Download to the directory specified in the settings. out_dir = pathlib.Path(settings['destination']) logger.debug(f'Downloading {url} to destination from settings') elif channel: out_dir = channel_directory logger.debug(f'Downloading {url} to channel directory') else: # Download to the default directory if this video has no channel. out_dir = get_no_channel_directory() logger.debug(f'Downloading {url} to default directory') out_dir.mkdir(exist_ok=True, parents=True) logs = None # noqa try: video_path, entry = self.prepare_filename(url, out_dir) if settings.get('download_metadata_only'): # User has requested refresh of video metadata, skip the rest of the video downloader. return await self.download_info_json(download, video_path) # Do the real download. cmd = ( str(YT_DLP_BIN), '-c', # Continue downloads '-f', PREFERRED_VIDEO_FORMAT, '--match-filter', '!is_live', # Do not attempt to download Live videos. '--write-subs', '--write-auto-subs', '--sub-format', PREFERRED_CAPTION_FORMAT, '--convert-subs', PREFERRED_CAPTION_FORMAT, '--convert-thumbnails', PREFERRED_POSTER_FORMAT, '--write-thumbnail', '--write-info-json', '--merge-output-format', PREFERRED_VIDEO_EXTENSION, '--remux-video', PREFERRED_VIDEO_EXTENSION, '-o', video_path, '--no-cache-dir', '--compat-options', 'no-live-chat', # Get top 20 comments, 10 replies per parent. '--write-comments', '--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top', # Use experimental feature to merge files. '--ppa', 'Merger+ffmpeg_o1:-strict -2', url, ) return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir) stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout'] stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr'] if return_code != 0: error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {return_code}' return DownloadResult( success=False, error=error, location=location, ) preferred_path = video_path.with_suffix(f'.{PREFERRED_VIDEO_EXTENSION}') if not video_path.is_file() and preferred_path.is_file(): # Prepared filename does not exist, but video with preferred video extension does, it was probably # remuxed by yt-dlp. video_path = preferred_path logger.info(f'Using preferred video file which exists: {preferred_path}') if not video_path.is_file(): error = f'{stdout}\n\n\n{stderr}\n\n' \ f'Video file could not be found! {video_path}' if '!is_live' in stdout: error = f'{stdout}\n\nVideo was live and did not finish downloading.' if ' live event ' in stdout: error = f'{stdout}\n\nVideo will be live and did not finish downloading.' return DownloadResult( success=False, error=error, location=location, ) if not ffmpeg_video_complete(video_path): return DownloadResult( success=False, error='Video was incomplete', location=location, ) with get_db_session(commit=True) as session: # Find any files downloaded with the video (poster, caption, etc.). video_paths = glob_shared_stem(video_path) video = Video.from_paths(session, *video_paths) video.source_id = entry['id'] video.channel_id = channel_id video_id, video_info_json_path = video.id, video.info_json_path session.commit() if video_info_json_path: format_json_file(video_info_json_path) with get_db_session(commit=True) as session: # Second session is started because SQLAlchemy will forget what we have done. video = Video.get_by_id(video_id, session) if download.settings and (tag_names := download.settings.get('tag_names')): existing_names = video.file_group.tag_names for name in tag_names: if name not in existing_names: video.add_tag(name) location = video.location await video.get_ffprobe_json() # Check that video has both audio and video streams. if not video.get_streams_by_codec_type('video'): return DownloadResult( success=False, error='Video was downloaded but did not contain video stream', location=location, ) if not video.get_streams_by_codec_type('audio'): return DownloadResult( success=False, error='Video was downloaded but did not contain audio stream', location=location, ) logger.info(f'Successfully downloaded video {url} {video}') with get_db_session(commit=True) as session: await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path) except UnrecoverableDownloadError: raise except yt_dlp.utils.UnsupportedError as e: raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e except Exception as e: logger.warning(f'VideoDownloader failed to download: {url}', exc_info=e) if _skip_download(e): # The video failed to download, and the error will never be fixed. Skip it forever. try: source_id = download.info_json.get('id') logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not ' f'attempt to download it again.') download.add_to_skip_list() except Exception: # Could not skip this video, it may not have a channel. logger.warning(f'Could not skip video {url}') # Skipped downloads should not be tried again. raise UnrecoverableDownloadError() from e # Download did not succeed, try again later. if logs and (stderr := logs.get('stderr')): error = f'{stderr}\n\n{traceback.format_exc()}' else: error = str(traceback.format_exc()) return DownloadResult(success=False, error=error, location=location) logger.debug(f'Downloaded video {location=}') result = DownloadResult( success=True, location=location, ) return result @staticmethod def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]: """Get the full path of a video file from its URL.""" if not out_dir.is_dir(): raise ValueError(f'Output directory does not exist! {out_dir=}') # YoutubeDL expects specific options, add onto the default options options = get_downloader_config().dict() options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}' options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION options['remuxvideo'] = PREFERRED_VIDEO_EXTENSION options['format'] = PREFERRED_VIDEO_FORMAT options['cachdir'] = YTDLP_CACHE_DIR # Create a new YoutubeDL for the output directory. ydl = YoutubeDL(options) ydl.params['logger'] = ydl_logger ydl.add_default_info_extractors() # Get the path where the video will be saved. entry = extract_info(url, ydl=ydl, process=True) final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute() logger.debug(f'Downloading {url} to {repr(str(final_filename))}') return final_filename, entry @staticmethod async def download_info_json(download: Download, video_path: pathlib.Path) -> DownloadResult: """User has requested to refresh Video metadata. Download only metadata files to temporary directory. Replace existing metadata files only after successful download.""" url = download.url source_id = download.info_json['id'] with get_db_session() as session: # Delete any duplicate videos before attempting to get info json. deleted = await Video.delete_duplicate_videos(session, download.url, source_id, video_path) if deleted: session.commit() with get_db_session() as session: # Find the existing video, replace its info json. video = Video.get_by_url(url, session) location = f'/videos/video/{video.id}' if video.channel_id: location = f'/videos/channel/{video.channel_id}/video/{video.id}' # Download video info json directly using yt-dlp. try: info_json = download_video_info_json(url) except Exception as e: logger.error('Failed to download video info json', exc_info=e) return DownloadResult(success=False, error='Failed to download video info json') with get_db_session(commit=True) as session: # Find the existing video, replace its info json. video = Video.get_by_url(url, session) video.replace_info_json(info_json) if video.get_comments(): video.have_comments = True await video.get_ffprobe_json() logger.debug(f'Downloaded video info json {location=}') logger.info(f'Successfully downloaded video info json {url} {video}') return DownloadResult(success=True, location=location) channel_downloader = ChannelDownloader() # Videos may match the ChannelDownloader, give it a higher priority. video_downloader = VideoDownloader() def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel: """ Attempt to find a Channel using the provided params. The params are in order of reliability. Creates a new Channel if one cannot be found. """ try: channel = get_channel(source_id=source_id, url=url, name=name, return_dict=False) # Get all properties while we have the session. channel.dict() return channel except UnknownChannel: pass if not name: raise UnknownChannel(f'Cannot create channel without a name') # Channel does not exist. Create one in the video directory. channel_directory = get_videos_directory() / escape_file_name(name) if not channel_directory.is_dir(): channel_directory.mkdir(parents=True) data = ChannelPostRequest( source_id=source_id, name=name, url=url, directory=str(channel_directory.relative_to(get_media_directory())), ) channel = create_channel(data=data, return_dict=False) # Create the directory now that the channel is approved. channel_directory.mkdir(exist_ok=True) # Get all properties while we have the session. channel.dict() return channel def update_channel_catalog(channel: Channel, info: dict): """ Connect to the Channel's host website and pull a catalog of all videos. Insert any new videos into the DB. It is expected that any missing videos will be downloaded later. """ logger.info(f'Downloading video list for {channel} at {channel.url}') # Resolve all entries to dictionaries. entries = info['entries'] = list(info['entries']) # yt-dlp may hand back a list of URLs, lets use the "Uploads" URL, if available. try: entries[0]['id'] except Exception: logger.warning('yt-dlp did not return a list of URLs') for entry in entries: if entry['title'] == 'Uploads': logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.') info = extract_info(entry['url']) break with get_db_session(commit=True) as session: # Get the channel in this new context. channel: Channel = session.query(Channel).filter_by(id=channel.id).one() channel.info_json = info channel.info_date = now() channel.source_id = info.get('id') channel_id = channel.id # Write the Channel's info to a JSON file. if channel.directory: info_json_path = channel.directory / f'{channel.name}.info.json' with info_json_path.open('wt') as fh: json.dump(info, fh, indent=2, sort_keys=True) logger.debug(f'Wrote channel info json to {info_json_path}') else: logger.debug(f'Skipping channel info json because it does not have a directory: {channel}') logger.info(f'Finished downloading video list for {channel} found {len(entries)} videos') # Update all view counts using the latest from the Channel's info_json. background_task(update_view_counts_and_censored(channel_id)) UNRECOVERABLE_ERRORS = { '404: Not Found', 'requires payment', 'Content Warning', 'Did not get any data blocks', 'Sign in', 'This live stream recording is not available.', 'members-only content', "You've asked yt-dlp to download the URL", } def _skip_download(error): """Return True if the error is unrecoverable and the video should be skipped in the future.""" error_str = str(error) for msg in UNRECOVERABLE_ERRORS: if msg in error_str: return True return False