mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Python 3.11 is now required, old versions were causing unpredictability in tests. (Sanic does not yet support 3.12) Sanic has been upgraded to 23.6.0, which is the latest version that avoids this bug: https://github.com/sanic-org/sanic/issues/2921 New strategy for multiprocessing is to create all multiprocessing tools in one process, then fork to other processes. The previous strategy was to declare multiprocessing tools at the top of every file, or wherever they were needed at import/creation. Now all multiprocessing tools are attached to the app.shared_ctx. This means `api_app` is imported in many, many places. This forced a change in how the DownloadManager works. Previously, it would continually run download workers which would pull downloads from a multiprocessing.Queue. Now, a single worker checks for new downloads and sends a Sanic signal. Flags have been reworked to use the `api_app`. I removed the `which` flag functionality because the `which` are called at import and needed their own multiprocessing.Event.
616 lines
26 KiB
Python
Executable file
616 lines
26 KiB
Python
Executable file
#! /usr/bin/env python3
|
|
import json
|
|
import pathlib
|
|
import re
|
|
import traceback
|
|
from abc import ABC
|
|
from typing import Tuple, List, Dict
|
|
|
|
import yt_dlp.utils
|
|
from sqlalchemy.orm import Session
|
|
from yt_dlp import YoutubeDL
|
|
from yt_dlp.extractor import YoutubeTabIE # noqa
|
|
|
|
from wrolpi.cmd import YT_DLP_BIN
|
|
from wrolpi.common import logger, get_media_directory, escape_file_name, resolve_generators, background_task, \
|
|
format_json_file
|
|
from wrolpi.dates import now
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.db import optional_session
|
|
from wrolpi.downloader import Downloader, Download, DownloadResult
|
|
from wrolpi.errors import UnrecoverableDownloadError
|
|
from wrolpi.files.lib import glob_shared_stem
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.vars import PYTEST, YTDLP_CACHE_DIR
|
|
from .channel.lib import create_channel, get_channel
|
|
from .common import get_no_channel_directory, get_videos_directory, update_view_counts, ffmpeg_video_complete
|
|
from .errors import UnknownChannel
|
|
from .lib import get_downloader_config, YDL, ydl_logger
|
|
from .models import Video, Channel
|
|
from .normalize_video_url import normalize_video_url
|
|
from .schema import ChannelPostRequest
|
|
from .video.lib import download_video_info_json
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
# Channels are handled differently than a single video.
|
|
ChannelIEs = {
|
|
YoutubeTabIE,
|
|
}
|
|
|
|
PREFERRED_VIDEO_EXTENSION = 'mp4'
|
|
PREFERRED_VIDEO_FORMAT = ','.join([
|
|
'res:720', # Use the best 720p available first
|
|
'136+140', # 130=720p video-only, 140=medium quality audio only
|
|
'22', # 720p video with audio
|
|
'135+140', # 135=480p video-only, 140=medium quality audio only
|
|
'135+139', # 135=480p video-only, 139=low quality audio only
|
|
'mp4-480p',
|
|
'bestvideo*+bestaudio/best', # Download the highest resolution as a last resort (can be quite large).
|
|
])
|
|
PREFERRED_CAPTION_FORMAT = 'vtt'
|
|
PREFERRED_POSTER_FORMAT = 'jpg'
|
|
|
|
|
|
def extract_info(url: str, ydl: YoutubeDL = YDL, process=False) -> dict:
|
|
"""Get info about a video. Separated for testing."""
|
|
return ydl.extract_info(url, download=False, process=process)
|
|
|
|
|
|
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
|
|
"""Get filename from YoutubeDL. Separated for testing."""
|
|
return ydl.prepare_filename(entry)
|
|
|
|
|
|
class ChannelDownloader(Downloader, ABC):
|
|
"""Handles downloading of videos in a Channel or Playlist."""
|
|
name = 'video_channel'
|
|
pretty_name = 'Video Channel'
|
|
listable = False
|
|
|
|
def __repr__(self):
|
|
return f'<ChannelDownloader>'
|
|
|
|
@staticmethod
|
|
def is_a_playlist(info: dict):
|
|
# A playlist may have an id different from its channel.
|
|
return info['id'] != info.get('channel_id')
|
|
|
|
async def do_download(self, download: Download) -> DownloadResult:
|
|
"""Update a Channel's catalog, then schedule downloads of every missing video."""
|
|
info = extract_info(download.url, process=False)
|
|
# Resolve the "entries" generator.
|
|
info: Dict = resolve_generators(info)
|
|
entries = info.get('entries')
|
|
if entries and isinstance(entries[0], Dict) and entries[0].get('entries'):
|
|
# Entries contains more channels/playlist, tell the Maintainer to specify the URL.
|
|
raise UnrecoverableDownloadError(f'Unable to download {download.url} because the URL is ambiguous.'
|
|
f' Try requesting the videos only:'
|
|
f' https://example.com -> https://example.com/videos')
|
|
|
|
download.sub_downloader = video_downloader.name
|
|
download.info_json = info
|
|
if session := Session.object_session(download):
|
|
# May not have a session during testing.
|
|
session.commit()
|
|
|
|
name = info.get('uploader') or info.get('webpage_url_basename')
|
|
if not name:
|
|
raise ValueError(f'Could not find name')
|
|
channel_source_id = info.get('channel_id') or info.get('id')
|
|
channel = get_or_create_channel(channel_source_id, download.url, name)
|
|
channel.dict() # get all attributes while we have the session.
|
|
|
|
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
|
|
|
|
# The settings to send to the VideoDownloader.
|
|
settings = dict()
|
|
if channel:
|
|
settings.update(dict(channel_id=channel.id, channel_url=download.url))
|
|
# Downloads will inherit the destination, if defined.
|
|
destination = download.settings.get('destination') if download.settings else None
|
|
if destination:
|
|
settings['destination'] = destination
|
|
|
|
is_a_playlist = self.is_a_playlist(info)
|
|
try:
|
|
if not is_a_playlist:
|
|
await self.prepare_channel_for_downloads(download, channel)
|
|
else:
|
|
logger.debug('Not updating channel because this is a playlist')
|
|
|
|
downloads = self.get_missing_videos(download, channel)
|
|
return DownloadResult(
|
|
success=True,
|
|
location=location,
|
|
downloads=downloads,
|
|
settings=settings,
|
|
)
|
|
except Exception:
|
|
if PYTEST:
|
|
raise
|
|
kind = 'playlist' if is_a_playlist else 'channel'
|
|
logger.warning(f'Failed to update catalog of {kind} {download.url}')
|
|
return DownloadResult(
|
|
success=False,
|
|
location=location,
|
|
error=str(traceback.format_exc()),
|
|
settings=settings,
|
|
)
|
|
|
|
@classmethod
|
|
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
|
|
"""Update the Channel's video catalog. Refresh the Channel's files if necessary."""
|
|
logger.debug(f'Preparing {channel} for downloads')
|
|
channel_id = channel.id
|
|
|
|
update_channel_catalog(channel, download.info_json)
|
|
|
|
with get_db_session() as session:
|
|
channel = get_channel(session, channel_id=channel_id, return_dict=False)
|
|
if not channel.videos or not channel.refreshed:
|
|
logger.warning(f'Refreshing videos in {channel.directory} because {channel} has no video records!')
|
|
await channel.refresh_files(send_events=False)
|
|
session.commit() # Commit the refresh.
|
|
session.refresh(channel)
|
|
|
|
@staticmethod
|
|
def get_missing_videos(download: Download, channel: Channel) -> List[str]:
|
|
"""
|
|
Return all URLs of Videos in the `info_json` which need to be downloaded.
|
|
"""
|
|
downloads = download.info_json['entries']
|
|
if channel.match_regex:
|
|
# Only download Videos that have matching titles.
|
|
match_regex = re.compile(channel.match_regex)
|
|
downloads = [i for i in downloads if (title := i.get('title')) and match_regex.match(title)]
|
|
|
|
# Prefer `webpage_url` before `url` for all entries.
|
|
downloads = [i.get('webpage_url') or i.get('url') for i in downloads]
|
|
|
|
# YouTube Shorts are handled specially.
|
|
downloads = [normalize_video_url(i) for i in downloads]
|
|
|
|
# Only download those that have not yet been downloaded.
|
|
already_downloaded = [i.url for i in video_downloader.already_downloaded(*downloads)]
|
|
downloads = [i for i in downloads if i not in already_downloaded]
|
|
|
|
return downloads
|
|
|
|
|
|
class VideoDownloader(Downloader, ABC):
|
|
"""Downloads a single video.
|
|
|
|
Store the video in its channel's directory, otherwise store it in `videos/NO CHANNEL`.
|
|
|
|
Videos are validated in a few ways. A video file is valid when it has a complete video stream and an audio stream.
|
|
A complete video stream is validated by taking a screenshot at the end of the video.
|
|
"""
|
|
name = 'video'
|
|
pretty_name = 'Videos'
|
|
|
|
def __repr__(self):
|
|
return f'<VideoDownloader>'
|
|
|
|
@optional_session
|
|
def already_downloaded(self, *urls: str, session: Session = None) -> List:
|
|
# We only consider a video record with a video file as "downloaded".
|
|
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'video'))
|
|
return file_groups
|
|
|
|
async def do_download(self, download: Download) -> DownloadResult:
|
|
if download.attempts >= 10:
|
|
raise UnrecoverableDownloadError('Max download attempts reached')
|
|
|
|
download_id = download.id
|
|
url = normalize_video_url(download.url)
|
|
|
|
# Video may have been downloaded previously, get its location for error reporting.
|
|
with get_db_session() as session:
|
|
videos = session.query(Video).join(FileGroup).filter_by(url=url).all()
|
|
location = videos[0].location if videos else None
|
|
|
|
try:
|
|
download.info_json = download.info_json or extract_info(url)
|
|
except yt_dlp.utils.DownloadError as e:
|
|
# Video may be private.
|
|
return DownloadResult(
|
|
success=False,
|
|
location=location,
|
|
error='\n'.join(traceback.format_exception(e)),
|
|
)
|
|
|
|
if not download.info_json:
|
|
raise ValueError(f'Cannot download video with no info_json.')
|
|
|
|
found_channel = None
|
|
|
|
channel_name = download.info_json.get('channel')
|
|
source_channel_id = download.info_json.get('channel_id')
|
|
channel_url = download.info_json.get('channel_url')
|
|
channel = None
|
|
if channel_name or source_channel_id:
|
|
# Try to find the channel via info_json from yt-dlp.
|
|
try:
|
|
channel = get_or_create_channel(source_id=source_channel_id, url=channel_url, name=channel_name)
|
|
if channel:
|
|
found_channel = 'yt_dlp'
|
|
logger.debug(f'Found a channel with source_id {channel=}')
|
|
except UnknownChannel:
|
|
# Can't find a channel, use the no channel directory.
|
|
pass
|
|
|
|
settings = download.settings or dict()
|
|
destination = settings.get('destination')
|
|
if not channel and destination:
|
|
# Destination may override the real Channel.
|
|
try:
|
|
channel = get_channel(directory=destination, return_dict=False)
|
|
found_channel = 'download_settings_directory'
|
|
logger.debug(f'Found a channel that shares the destination directory {channel=}')
|
|
except UnknownChannel:
|
|
# Destination must not be a channel.
|
|
pass
|
|
|
|
local_channel_id = settings.get('channel_id')
|
|
channel_url = settings.get('channel_url')
|
|
if not channel and (local_channel_id or channel_url):
|
|
# Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
|
logger.info(f'Using download.settings to find channel')
|
|
try:
|
|
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
|
|
logger.debug(f'Found channel with channel url {channel=}')
|
|
except UnknownChannel:
|
|
# We do not need a Channel if we have a destination directory.
|
|
if not destination:
|
|
raise
|
|
if channel:
|
|
found_channel = 'download_settings'
|
|
|
|
channel_id = channel.id if channel else None
|
|
channel_directory = channel.directory if channel else None
|
|
|
|
if found_channel == 'yt_dlp':
|
|
logger.debug(f'Found {channel} using yt_dlp')
|
|
elif found_channel == 'download_settings':
|
|
logger.info(f'Found {channel} using Download.settings')
|
|
elif found_channel == 'download_settings_directory':
|
|
logger.info(f'Found {channel} using Download.settings directory')
|
|
else:
|
|
logger.warning('Could not find channel')
|
|
|
|
if destination:
|
|
# Download to the directory specified in the settings.
|
|
out_dir = pathlib.Path(settings['destination'])
|
|
logger.debug(f'Downloading {url} to destination from settings')
|
|
elif channel:
|
|
out_dir = channel_directory
|
|
logger.debug(f'Downloading {url} to channel directory')
|
|
else:
|
|
# Download to the default directory if this video has no channel.
|
|
out_dir = get_no_channel_directory()
|
|
logger.debug(f'Downloading {url} to default directory')
|
|
out_dir.mkdir(exist_ok=True, parents=True)
|
|
|
|
logs = None # noqa
|
|
try:
|
|
video_path, entry = self.prepare_filename(url, out_dir)
|
|
|
|
if settings.get('do_not_download'):
|
|
# User has requested refresh of video metadata, skip the rest of the video downloader.
|
|
return await self.download_info_json(download, video_path)
|
|
|
|
# Do the real download.
|
|
file_name_format = '%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s'
|
|
cmd = (
|
|
str(YT_DLP_BIN),
|
|
'-cw', # Continue downloads, do not clobber existing files.
|
|
'-f', PREFERRED_VIDEO_FORMAT,
|
|
'--match-filter', '!is_live', # Do not attempt to download Live videos.
|
|
'--write-subs',
|
|
'--write-auto-subs',
|
|
'--sub-format', PREFERRED_CAPTION_FORMAT,
|
|
'--convert-subs', PREFERRED_CAPTION_FORMAT,
|
|
'--convert-thumbnails', PREFERRED_POSTER_FORMAT,
|
|
'--write-thumbnail',
|
|
'--write-info-json',
|
|
'--merge-output-format', PREFERRED_VIDEO_EXTENSION,
|
|
'--remux-video', PREFERRED_VIDEO_EXTENSION,
|
|
'-o', file_name_format,
|
|
'--no-cache-dir',
|
|
'--compat-options', 'no-live-chat',
|
|
# Get top 20 comments, 10 replies per parent.
|
|
'--write-comments',
|
|
'--extractor-args', 'youtube:max_comments=all,20,all,10;comment_sort=top',
|
|
# Use experimental feature to merge files.
|
|
'--ppa', 'Merger+ffmpeg_o1:-strict -2',
|
|
url,
|
|
)
|
|
return_code, logs, _ = await self.process_runner(download_id, url, cmd, out_dir)
|
|
|
|
stdout = logs['stdout'].decode() if hasattr(logs['stdout'], 'decode') else logs['stdout']
|
|
stderr = logs['stderr'].decode() if hasattr(logs['stderr'], 'decode') else logs['stderr']
|
|
|
|
if return_code != 0:
|
|
error = f'{stdout}\n\n\n{stderr}\n\nvideo downloader process exited with {return_code}'
|
|
return DownloadResult(
|
|
success=False,
|
|
error=error,
|
|
location=location,
|
|
)
|
|
|
|
preferred_path = video_path.with_suffix(f'.{PREFERRED_VIDEO_EXTENSION}')
|
|
if not video_path.is_file() and preferred_path.is_file():
|
|
# Prepared filename does not exist, but video with preferred video extension does, it was probably
|
|
# remuxed by yt-dlp.
|
|
video_path = preferred_path
|
|
logger.info(f'Using preferred video file which exists: {preferred_path}')
|
|
|
|
if not video_path.is_file():
|
|
error = f'{stdout}\n\n\n{stderr}\n\n' \
|
|
f'Video file could not be found! {video_path}'
|
|
return DownloadResult(
|
|
success=False,
|
|
error=error,
|
|
location=location,
|
|
)
|
|
|
|
if not ffmpeg_video_complete(video_path):
|
|
return DownloadResult(
|
|
success=False,
|
|
error='Video was incomplete',
|
|
location=location,
|
|
)
|
|
|
|
with get_db_session(commit=True) as session:
|
|
# Find any files downloaded with the video (poster, caption, etc.).
|
|
video_paths = glob_shared_stem(video_path)
|
|
video = Video.from_paths(session, *video_paths)
|
|
video.source_id = entry['id']
|
|
video.channel_id = channel_id
|
|
video_id, video_info_json_path = video.id, video.info_json_path
|
|
session.commit()
|
|
|
|
if video_info_json_path:
|
|
format_json_file(video_info_json_path)
|
|
|
|
with get_db_session(commit=True) as session:
|
|
# Second session is started because SQLAlchemy will forget what we have done.
|
|
video = Video.get_by_id(video_id, session)
|
|
if download.settings and (tag_names := download.settings.get('tag_names')):
|
|
existing_names = video.file_group.tag_names
|
|
for name in tag_names:
|
|
if name not in existing_names:
|
|
video.add_tag(name)
|
|
location = video.location
|
|
|
|
await video.get_ffprobe_json()
|
|
|
|
# Check that video has both audio and video streams.
|
|
if not video.get_streams_by_codec_type('video'):
|
|
return DownloadResult(
|
|
success=False,
|
|
error='Video was downloaded but did not contain video stream',
|
|
location=location,
|
|
)
|
|
if not video.get_streams_by_codec_type('audio'):
|
|
return DownloadResult(
|
|
success=False,
|
|
error='Video was downloaded but did not contain audio stream',
|
|
location=location,
|
|
)
|
|
logger.info(f'Successfully downloaded video {url} {video}')
|
|
|
|
with get_db_session(commit=True) as session:
|
|
await Video.delete_duplicate_videos(session, download.url, entry['id'], video_path)
|
|
|
|
except UnrecoverableDownloadError:
|
|
raise
|
|
except yt_dlp.utils.UnsupportedError as e:
|
|
raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e
|
|
except Exception as e:
|
|
logger.warning(f'VideoDownloader failed to download: {url}', exc_info=e)
|
|
if _skip_download(e):
|
|
# The video failed to download, and the error will never be fixed. Skip it forever.
|
|
try:
|
|
source_id = download.info_json.get('id')
|
|
logger.warning(f'Adding video "{source_id}" to skip list for this channel. WROLPi will not '
|
|
f'attempt to download it again.')
|
|
download.add_to_skip_list()
|
|
except Exception:
|
|
# Could not skip this video, it may not have a channel.
|
|
logger.warning(f'Could not skip video {url}')
|
|
|
|
# Skipped downloads should not be tried again.
|
|
raise UnrecoverableDownloadError() from e
|
|
# Download did not succeed, try again later.
|
|
if logs and (stderr := logs.get('stderr')):
|
|
error = f'{stderr}\n\n{traceback.format_exc()}'
|
|
else:
|
|
error = str(traceback.format_exc())
|
|
return DownloadResult(success=False, error=error, location=location)
|
|
|
|
logger.debug(f'Downloaded video {location=}')
|
|
|
|
result = DownloadResult(
|
|
success=True,
|
|
location=location,
|
|
)
|
|
return result
|
|
|
|
@staticmethod
|
|
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
|
|
"""Get the full path of a video file from its URL."""
|
|
if not out_dir.is_dir():
|
|
raise ValueError(f'Output directory does not exist! {out_dir=}')
|
|
|
|
# YoutubeDL expects specific options, add onto the default options
|
|
options = get_downloader_config().dict()
|
|
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
|
|
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
|
options['remuxvideo'] = PREFERRED_VIDEO_EXTENSION
|
|
options['format'] = PREFERRED_VIDEO_FORMAT
|
|
options['cachdir'] = YTDLP_CACHE_DIR
|
|
|
|
# Create a new YoutubeDL for the output directory.
|
|
ydl = YoutubeDL(options)
|
|
ydl.params['logger'] = ydl_logger
|
|
ydl.add_default_info_extractors()
|
|
|
|
# Get the path where the video will be saved.
|
|
entry = extract_info(url, ydl=ydl, process=True)
|
|
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
|
|
|
|
logger.debug(f'Downloading {url} to {out_dir}')
|
|
return final_filename, entry
|
|
|
|
@staticmethod
|
|
async def download_info_json(download: Download, video_path: pathlib.Path) -> DownloadResult:
|
|
"""User has requested to refresh Video metadata. Download only metadata files to temporary directory.
|
|
Replace existing metadata files only after successful download."""
|
|
url = download.url
|
|
source_id = download.info_json['id']
|
|
|
|
with get_db_session() as session:
|
|
# Delete any duplicate videos before attempting to get info json.
|
|
deleted = await Video.delete_duplicate_videos(session, download.url, source_id, video_path)
|
|
if deleted:
|
|
session.commit()
|
|
|
|
with get_db_session() as session:
|
|
# Find the existing video, replace its info json.
|
|
video = Video.get_by_url(url, session)
|
|
|
|
location = f'/videos/video/{video.id}'
|
|
if video.channel_id:
|
|
location = f'/videos/channel/{video.channel_id}/video/{video.id}'
|
|
|
|
# Download video info json directly using yt-dlp.
|
|
try:
|
|
info_json = download_video_info_json(url)
|
|
except Exception as e:
|
|
logger.error('Failed to download video info json', exc_info=e)
|
|
return DownloadResult(success=False, error='Failed to download video info json')
|
|
|
|
with get_db_session(commit=True) as session:
|
|
# Find the existing video, replace its info json.
|
|
video = Video.get_by_url(url, session)
|
|
video.replace_info_json(info_json)
|
|
|
|
if video.get_comments():
|
|
video.have_comments = True
|
|
|
|
await video.get_ffprobe_json()
|
|
|
|
logger.debug(f'Downloaded video info json {location=}')
|
|
logger.info(f'Successfully downloaded video info json {url} {video}')
|
|
|
|
return DownloadResult(success=True, location=location)
|
|
|
|
|
|
channel_downloader = ChannelDownloader()
|
|
# Videos may match the ChannelDownloader, give it a higher priority.
|
|
video_downloader = VideoDownloader()
|
|
|
|
|
|
def get_or_create_channel(source_id: str = None, url: str = None, name: str = None) -> Channel:
|
|
"""
|
|
Attempt to find a Channel using the provided params. The params are in order of reliability.
|
|
|
|
Creates a new Channel if one cannot be found.
|
|
"""
|
|
try:
|
|
channel = get_channel(source_id=source_id, url=url, name=name, return_dict=False)
|
|
# Get all properties while we have the session.
|
|
channel.dict()
|
|
return channel
|
|
except UnknownChannel:
|
|
pass
|
|
|
|
if not name:
|
|
raise UnknownChannel(f'Cannot create channel without a name')
|
|
|
|
# Channel does not exist. Create one in the video directory.
|
|
channel_directory = get_videos_directory() / escape_file_name(name)
|
|
if not channel_directory.is_dir():
|
|
channel_directory.mkdir(parents=True)
|
|
data = ChannelPostRequest(
|
|
source_id=source_id,
|
|
name=name,
|
|
url=url,
|
|
directory=str(channel_directory.relative_to(get_media_directory())),
|
|
)
|
|
channel = create_channel(data=data, return_dict=False)
|
|
# Create the directory now that the channel is approved.
|
|
channel_directory.mkdir(exist_ok=True)
|
|
# Get all properties while we have the session.
|
|
channel.dict()
|
|
|
|
return channel
|
|
|
|
|
|
def update_channel_catalog(channel: Channel, info: dict):
|
|
"""
|
|
Connect to the Channel's host website and pull a catalog of all videos. Insert any new videos into the DB.
|
|
|
|
It is expected that any missing videos will be downloaded later.
|
|
"""
|
|
logger.info(f'Downloading video list for {channel} at {channel.url}')
|
|
|
|
# Resolve all entries to dictionaries.
|
|
entries = info['entries'] = list(info['entries'])
|
|
|
|
# yt-dlp may hand back a list of URLs, lets use the "Uploads" URL, if available.
|
|
try:
|
|
entries[0]['id']
|
|
except Exception:
|
|
logger.warning('yt-dlp did not return a list of URLs')
|
|
for entry in entries:
|
|
if entry['title'] == 'Uploads':
|
|
logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.')
|
|
info = extract_info(entry['url'])
|
|
break
|
|
|
|
with get_db_session(commit=True) as session:
|
|
# Get the channel in this new context.
|
|
channel: Channel = session.query(Channel).filter_by(id=channel.id).one()
|
|
|
|
channel.info_json = info
|
|
channel.info_date = now()
|
|
channel.source_id = info.get('id')
|
|
channel_id = channel.id
|
|
|
|
# Write the Channel's info to a JSON file.
|
|
if channel.directory:
|
|
info_json_path = channel.directory / f'{channel.name}.info.json'
|
|
with info_json_path.open('wt') as fh:
|
|
json.dump(info, fh, indent=2)
|
|
logger.debug(f'Wrote channel info json to {info_json_path}')
|
|
else:
|
|
logger.debug(f'Skipping channel info json because it does not have a directory: {channel}')
|
|
|
|
logger.info(f'Finished downloading video list for {channel} found {len(entries)} videos')
|
|
|
|
# Update all view counts using the latest from the Channel's info_json.
|
|
background_task(update_view_counts(channel_id))
|
|
|
|
|
|
UNRECOVERABLE_ERRORS = {
|
|
'404: Not Found',
|
|
'requires payment',
|
|
'Content Warning',
|
|
'Did not get any data blocks',
|
|
'Sign in',
|
|
'This live stream recording is not available.',
|
|
'members-only content',
|
|
"You've asked yt-dlp to download the URL",
|
|
}
|
|
|
|
|
|
def _skip_download(error):
|
|
"""Return True if the error is unrecoverable and the video should be skipped in the future."""
|
|
error_str = str(error)
|
|
for msg in UNRECOVERABLE_ERRORS:
|
|
if msg in error_str:
|
|
return True
|
|
return False
|