ChannelDownloader will add itself to Download.Settings, so VideoDownloader can download to the channel if yt-dlp cannot find the channel.

This commit is contained in:
Roland 2022-11-08 22:41:50 -07:00
parent 456a83b8f4
commit 1e1c8f0b85
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
8 changed files with 126 additions and 77 deletions

View file

@ -308,7 +308,7 @@ def apply_info_json(channel_id: int):
Mark any videos not in the info_json as "censored".
"""
with get_db_session() as session:
channel = session.query(Channel).filter_by(id=channel_id).one()
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
channel_name = channel.name
info = channel.info_json

View file

@ -45,6 +45,8 @@ PREFERRED_VIDEO_FORMAT = ','.join([
'res:720', # Use the best 720p available first
'136+140', # 130=720p video-only, 140= high quality audio only
'22', # 720p video with audio
'mp4-480p',
'mp4-360p',
'18', # 360p video with audio
'bestvideo+bestaudio', # Download the highest resolution as a last resort (1080p is quite large).
])
@ -79,7 +81,9 @@ class ChannelDownloader(Downloader, ABC):
@staticmethod
def is_a_playlist(info: dict):
# A playlist will have an id different from its channel.
if '_type' in info:
return info['_type'] == 'playlist'
# A playlist may have an id different from its channel.
return info['id'] != info['channel_id']
async def do_download(self, download: Download) -> DownloadResult:
@ -92,12 +96,15 @@ class ChannelDownloader(Downloader, ABC):
# May not have a session during testing.
session.commit()
kind = info['uploader']
channel_source_id = info['channel_id']
channel = get_or_create_channel(channel_source_id, download.url, kind)
name = info.get('uploader') or info.get('webpage_url_basename')
if not name:
raise ValueError(f'Could not find name')
channel_source_id = info.get('channel_id') or info.get('id')
channel = get_or_create_channel(channel_source_id, download.url, name)
channel.dict() # get all attributes while we have the session.
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
settings = dict(channel_id=channel.id, channel_url=download.url) if channel else None
is_a_playlist = self.is_a_playlist(info)
try:
@ -105,13 +112,23 @@ class ChannelDownloader(Downloader, ABC):
await self.prepare_channel_for_downloads(download, channel)
downloads = self.get_missing_videos(download, channel)
return DownloadResult(success=True, location=location, downloads=downloads)
return DownloadResult(
success=True,
location=location,
downloads=downloads,
settings=settings,
)
except Exception:
if PYTEST:
raise
kind = 'playlist' if is_a_playlist else 'channel'
logger.warning(f'Failed to update catalog of {kind} {download.url}')
return DownloadResult(success=False, location=location, error=str(traceback.format_exc()))
return DownloadResult(
success=False,
location=location,
error=str(traceback.format_exc()),
settings=settings,
)
@classmethod
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
@ -209,12 +226,34 @@ class VideoDownloader(Downloader, ABC):
download.info_json = info
session.commit()
found_channel = None
channel_name = info.get('channel')
channel_id = info.get('channel_id')
source_channel_id = info.get('channel_id')
channel_url = info.get('channel_url')
channel = None
if channel_name or channel_id:
channel_url = info.get('channel_url')
channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name)
if channel_name or source_channel_id:
# Try to find the channel via info_json from yt-dlp.
channel = get_or_create_channel(source_id=source_channel_id, url=channel_url, name=channel_name)
if channel:
found_channel = 'yt_dlp'
settings = download.settings or dict()
local_channel_id = settings.get('channel_id')
channel_url = settings.get('channel_url')
if not channel and (local_channel_id or channel_url):
# Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
logger.info(f'Using download.settings to find channel')
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
if channel:
found_channel = 'download_settings'
if found_channel == 'yt_dlp':
logger.debug('Found channel using yt_dlp')
elif found_channel == 'download_settings':
logger.info('Found channel using Download.settings')
else:
logger.info('Could not find channel')
# Use the default directory if this video has no channel.
out_dir = get_no_channel_directory()
@ -321,8 +360,7 @@ class VideoDownloader(Downloader, ABC):
options = get_downloader_config().dict()
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
logger.debug(f'Downloading {url} to {out_dir}')
options['format'] = PREFERRED_VIDEO_FORMAT
# Create a new YoutubeDL for the output directory.
ydl = YoutubeDL(options)
@ -334,6 +372,8 @@ class VideoDownloader(Downloader, ABC):
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
if final_filename.suffix.lower() != f'.{PREFERRED_VIDEO_EXTENSION}':
raise DownloadError(f'Cannot download video {url} because yt-dlp filename is invalid.')
logger.debug(f'Downloading {url} to {out_dir}')
return final_filename, entry
@ -410,7 +450,7 @@ def update_channel_catalog(channel: Channel, info: dict):
with get_db_session(commit=True) as session:
# Get the channel in this new context.
channel = session.query(Channel).filter_by(id=channel.id).one()
channel: Channel = session.query(Channel).filter_by(id=channel.id).one()
channel.info_json = info
channel.info_date = now()

View file

@ -153,8 +153,8 @@ class TestVideosDownloaders(TestAPI):
@pytest.mark.asyncio
async def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
mock_video_extract_info, mock_video_process_runner):
async def test_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
mock_video_extract_info, mock_video_process_runner):
"""A video can be downloaded even if it does not have a Channel."""
channel_dir = test_directory / 'NO CHANNEL'
channel_dir.mkdir(parents=True)
@ -216,12 +216,16 @@ async def test_download_channel(test_session, simple_channel, video_download_man
await video_download_manager.wait_for_all_downloads()
downloads = video_download_manager.get_once_downloads(test_session)
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
assert downloads[0].settings == {
'channel_id': 1,
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
}
def test_get_or_create_channel(test_session):
"""
A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
match it.
"""A Channel may need to be created for an arbitrary download.
Attempt to use an existing Channel if we can match it.
"""
c1 = Channel(name='foo', source_id='foo', url='foo')
c2 = Channel(name='bar', source_id='bar')
@ -376,8 +380,8 @@ async def test_invalid_download_url(test_session, test_download_manager, mock_vi
@pytest.mark.asyncio
async def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager,
mock_video_process_runner, image_file):
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
mock_video_process_runner, image_file):
"""A video download is performed, files are associated."""
simple_channel.source_id = example_video_json['channel_id']
simple_channel.directory = test_directory / 'videos/channel name'

View file

@ -187,7 +187,7 @@ class ConfigFile:
raise ValueError(f'Refusing to save config file while testing: {config_file}')
# Only one process can write to the file.
self.file_lock.acquire(block=True, timeout=5.0)
acquired = self.file_lock.acquire(block=True, timeout=5.0)
try:
# Config directory may not exist.
@ -206,7 +206,8 @@ class ConfigFile:
with config_file.open('wt') as fh:
yaml.dump(config, fh)
finally:
self.file_lock.release()
if acquired:
self.file_lock.release()
def get_file(self) -> Path:
if not self.file_name:

View file

@ -50,6 +50,7 @@ class DownloadResult:
info_json: dict = field(default_factory=dict)
location: str = None
success: bool = False
settings: dict = field(default_factory=dict)
class Download(ModelHelper, Base): # noqa
@ -402,7 +403,10 @@ class DownloadManager:
if result.downloads:
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
urls = download.filter_excluded(result.downloads)
self.create_downloads(urls, session, downloader=download.sub_downloader)
downloads = self.create_downloads(urls, session,
downloader=download.sub_downloader,
settings=result.settings,
)
if try_again is False and not download.frequency:
# Only once-downloads can fail.
@ -533,17 +537,9 @@ class DownloadManager:
download.manager = self
return download
@optional_session
def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False,
sub_downloader: str = None, excluded_urls: List[str] = None) -> Download:
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
downloads = self.create_downloads([url], session=session, downloader=downloader, excluded_urls=excluded_urls,
reset_attempts=reset_attempts, sub_downloader=sub_downloader)
return downloads[0]
@optional_session
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
reset_attempts: bool = False, sub_downloader: str = None, excluded_urls: List[str] = None) \
reset_attempts: bool = False, sub_downloader: str = None, settings: dict = None) \
-> List[Download]:
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
if not all(urls):
@ -577,7 +573,7 @@ class DownloadManager:
download.downloader = local_downloader.name
download.info_json = info_json or download.info_json
download.sub_downloader = sub_downloader
download.settings = {'excluded_urls': excluded_urls}
download.settings = settings
downloads.append(download)
try:
@ -592,17 +588,24 @@ class DownloadManager:
return downloads
@optional_session
def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False,
sub_downloader: str = None, settings: Dict = None) -> Download:
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
downloads = self.create_downloads([url], session=session, downloader=downloader, settings=settings,
reset_attempts=reset_attempts, sub_downloader=sub_downloader)
return downloads[0]
def recurring_download(self, url: str, frequency: int, downloader: str = None,
sub_downloader: str = None, reset_attempts: bool = False,
excluded_urls: List[str] = None) -> Download:
settings: Dict = None) -> Download:
"""Schedule a recurring download."""
if not frequency:
raise ValueError('Recurring download must have a frequency!')
with get_db_session(commit=True) as session:
download, = self.create_downloads([url, ], session=session, downloader=downloader,
sub_downloader=sub_downloader, reset_attempts=reset_attempts,
excluded_urls=excluded_urls)
download, = self.create_downloads([url, ], session=session, downloader=downloader, settings=settings,
sub_downloader=sub_downloader, reset_attempts=reset_attempts)
download.frequency = frequency
return download

View file

@ -221,13 +221,14 @@ async def post_download(_: Request, body: DownloadRequest):
downloader = body.downloader
if not downloader or downloader in ('auto', 'None', 'null'):
downloader = None
settings = dict(excluded_urls=excluded_urls)
if body.frequency:
download_manager.recurring_download(urls[0], body.frequency, downloader=downloader,
sub_downloader=body.sub_downloader, reset_attempts=True,
excluded_urls=excluded_urls)
settings=settings)
else:
download_manager.create_downloads(urls, downloader=downloader, sub_downloader=body.sub_downloader,
reset_attempts=True, excluded_urls=excluded_urls)
reset_attempts=True, settings=settings)
return response.empty()

View file

@ -11,7 +11,8 @@ import pytest
from wrolpi.dates import local_timezone, Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config, \
RSSDownloader
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
@ -481,3 +482,37 @@ async def test_downloads_config(test_client, test_session, test_download_manager
# Import again, no duplicates.
await import_downloads_config()
assert_downloads(expected)
@pytest.mark.asyncio
async def test_download_excluded_urls(test_client, test_session, test_download_manager):
"""Test that URLs that have been excluded are ignored by the download worker."""
http_downloader = HTTPDownloader()
http_downloader.already_downloaded = lambda *i, **kw: []
http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(
bozo=0,
entries=[
dict(link='https://example.com/a'),
dict(link='https://example.org/b'), # this should be ignored
dict(link='https://example.com/c'),
dict(link='https://example.gov/d'),
]
)
settings = dict(excluded_urls=['example.org', 'example.gov'])
feed_download: Download = test_download_manager.create_download('https://example.com/feed',
test_session, sub_downloader='http',
downloader=RSSDownloader.name,
settings=settings)
await test_download_manager.wait_for_all_downloads()
downloads = list(test_download_manager.get_downloads(test_session))
assert not feed_download.error, f'Feed download had error {feed_download.error}'
assert len(downloads) == 3, 'Domain was not ignored'
assert [i.url for i in downloads] == \
['https://example.com/feed', 'https://example.com/a', 'https://example.com/c'], f'Domain was not ignored'

View file

@ -2,17 +2,15 @@ import json
from http import HTTPStatus
from itertools import zip_longest
import pytest
from mock import mock
from wrolpi.admin import HotspotStatus
from wrolpi.common import get_config
from wrolpi.dates import strptime
from wrolpi.db import get_db_session
from wrolpi.downloader import Download, RSSDownloader, DownloadResult
from wrolpi.downloader import Download
from wrolpi.root_api import api_app
from wrolpi.test.common import TestAPI, wrap_test_db, skip_circleci
from wrolpi.test.test_downloader import HTTPDownloader
class TestRootAPI(TestAPI):
@ -353,36 +351,3 @@ def test_get_downloaders(test_client):
assert 'downloaders' in response.json, 'Downloaders not returned'
assert isinstance(response.json['downloaders'], list) and len(response.json['downloaders']), \
'No downloaders returned'
@pytest.mark.asyncio
async def test_download_excluded_urls(test_client, test_session, test_download_manager):
"""Test that URLs that have been excluded are ignored by the download worker."""
http_downloader = HTTPDownloader()
http_downloader.already_downloaded = lambda *i, **kw: []
http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
rss_downloader = RSSDownloader()
test_download_manager.register_downloader(rss_downloader)
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
mock_parse_feed.return_value = dict(
bozo=0,
entries=[
dict(link='https://example.com/a'),
dict(link='https://example.org/b'), # this should be ignored
dict(link='https://example.com/c'),
dict(link='https://example.gov/d'),
]
)
feed_download: Download = test_download_manager.create_download('https://example.com/feed',
test_session, sub_downloader='http',
downloader=RSSDownloader.name,
excluded_urls=['example.org', 'example.gov'])
await test_download_manager.wait_for_all_downloads()
downloads = list(test_download_manager.get_downloads(test_session))
assert not feed_download.error, f'Feed download had error {feed_download.error}'
assert len(downloads) == 3, 'Domain was not ignored'
assert [i.url for i in downloads] == \
['https://example.com/feed', 'https://example.com/a', 'https://example.com/c'], f'Domain was not ignored'