mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 23:16:17 -05:00
ChannelDownloader will add itself to Download.Settings, so VideoDownloader can download to the channel if yt-dlp cannot find the channel.
This commit is contained in:
parent
456a83b8f4
commit
1e1c8f0b85
8 changed files with 126 additions and 77 deletions
|
|
@ -308,7 +308,7 @@ def apply_info_json(channel_id: int):
|
|||
Mark any videos not in the info_json as "censored".
|
||||
"""
|
||||
with get_db_session() as session:
|
||||
channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
channel: Channel = session.query(Channel).filter_by(id=channel_id).one()
|
||||
channel_name = channel.name
|
||||
info = channel.info_json
|
||||
|
||||
|
|
|
|||
|
|
@ -45,6 +45,8 @@ PREFERRED_VIDEO_FORMAT = ','.join([
|
|||
'res:720', # Use the best 720p available first
|
||||
'136+140', # 130=720p video-only, 140= high quality audio only
|
||||
'22', # 720p video with audio
|
||||
'mp4-480p',
|
||||
'mp4-360p',
|
||||
'18', # 360p video with audio
|
||||
'bestvideo+bestaudio', # Download the highest resolution as a last resort (1080p is quite large).
|
||||
])
|
||||
|
|
@ -79,7 +81,9 @@ class ChannelDownloader(Downloader, ABC):
|
|||
|
||||
@staticmethod
|
||||
def is_a_playlist(info: dict):
|
||||
# A playlist will have an id different from its channel.
|
||||
if '_type' in info:
|
||||
return info['_type'] == 'playlist'
|
||||
# A playlist may have an id different from its channel.
|
||||
return info['id'] != info['channel_id']
|
||||
|
||||
async def do_download(self, download: Download) -> DownloadResult:
|
||||
|
|
@ -92,12 +96,15 @@ class ChannelDownloader(Downloader, ABC):
|
|||
# May not have a session during testing.
|
||||
session.commit()
|
||||
|
||||
kind = info['uploader']
|
||||
channel_source_id = info['channel_id']
|
||||
channel = get_or_create_channel(channel_source_id, download.url, kind)
|
||||
name = info.get('uploader') or info.get('webpage_url_basename')
|
||||
if not name:
|
||||
raise ValueError(f'Could not find name')
|
||||
channel_source_id = info.get('channel_id') or info.get('id')
|
||||
channel = get_or_create_channel(channel_source_id, download.url, name)
|
||||
channel.dict() # get all attributes while we have the session.
|
||||
|
||||
location = f'/videos/channel/{channel.id}/video' if channel and channel.id else None
|
||||
settings = dict(channel_id=channel.id, channel_url=download.url) if channel else None
|
||||
|
||||
is_a_playlist = self.is_a_playlist(info)
|
||||
try:
|
||||
|
|
@ -105,13 +112,23 @@ class ChannelDownloader(Downloader, ABC):
|
|||
await self.prepare_channel_for_downloads(download, channel)
|
||||
|
||||
downloads = self.get_missing_videos(download, channel)
|
||||
return DownloadResult(success=True, location=location, downloads=downloads)
|
||||
return DownloadResult(
|
||||
success=True,
|
||||
location=location,
|
||||
downloads=downloads,
|
||||
settings=settings,
|
||||
)
|
||||
except Exception:
|
||||
if PYTEST:
|
||||
raise
|
||||
kind = 'playlist' if is_a_playlist else 'channel'
|
||||
logger.warning(f'Failed to update catalog of {kind} {download.url}')
|
||||
return DownloadResult(success=False, location=location, error=str(traceback.format_exc()))
|
||||
return DownloadResult(
|
||||
success=False,
|
||||
location=location,
|
||||
error=str(traceback.format_exc()),
|
||||
settings=settings,
|
||||
)
|
||||
|
||||
@classmethod
|
||||
async def prepare_channel_for_downloads(cls, download: Download, channel: Channel):
|
||||
|
|
@ -209,12 +226,34 @@ class VideoDownloader(Downloader, ABC):
|
|||
download.info_json = info
|
||||
session.commit()
|
||||
|
||||
found_channel = None
|
||||
|
||||
channel_name = info.get('channel')
|
||||
channel_id = info.get('channel_id')
|
||||
source_channel_id = info.get('channel_id')
|
||||
channel_url = info.get('channel_url')
|
||||
channel = None
|
||||
if channel_name or channel_id:
|
||||
channel_url = info.get('channel_url')
|
||||
channel = get_or_create_channel(source_id=channel_id, url=channel_url, name=channel_name)
|
||||
if channel_name or source_channel_id:
|
||||
# Try to find the channel via info_json from yt-dlp.
|
||||
channel = get_or_create_channel(source_id=source_channel_id, url=channel_url, name=channel_name)
|
||||
if channel:
|
||||
found_channel = 'yt_dlp'
|
||||
|
||||
settings = download.settings or dict()
|
||||
local_channel_id = settings.get('channel_id')
|
||||
channel_url = settings.get('channel_url')
|
||||
if not channel and (local_channel_id or channel_url):
|
||||
# Could not find channel via yt-dlp info_json, use info from ChannelDownloader if it created this Download.
|
||||
logger.info(f'Using download.settings to find channel')
|
||||
channel = get_channel(channel_id=local_channel_id, url=channel_url, return_dict=False)
|
||||
if channel:
|
||||
found_channel = 'download_settings'
|
||||
|
||||
if found_channel == 'yt_dlp':
|
||||
logger.debug('Found channel using yt_dlp')
|
||||
elif found_channel == 'download_settings':
|
||||
logger.info('Found channel using Download.settings')
|
||||
else:
|
||||
logger.info('Could not find channel')
|
||||
|
||||
# Use the default directory if this video has no channel.
|
||||
out_dir = get_no_channel_directory()
|
||||
|
|
@ -321,8 +360,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
options = get_downloader_config().dict()
|
||||
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
|
||||
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
||||
|
||||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
options['format'] = PREFERRED_VIDEO_FORMAT
|
||||
|
||||
# Create a new YoutubeDL for the output directory.
|
||||
ydl = YoutubeDL(options)
|
||||
|
|
@ -334,6 +372,8 @@ class VideoDownloader(Downloader, ABC):
|
|||
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
|
||||
if final_filename.suffix.lower() != f'.{PREFERRED_VIDEO_EXTENSION}':
|
||||
raise DownloadError(f'Cannot download video {url} because yt-dlp filename is invalid.')
|
||||
|
||||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
return final_filename, entry
|
||||
|
||||
|
||||
|
|
@ -410,7 +450,7 @@ def update_channel_catalog(channel: Channel, info: dict):
|
|||
|
||||
with get_db_session(commit=True) as session:
|
||||
# Get the channel in this new context.
|
||||
channel = session.query(Channel).filter_by(id=channel.id).one()
|
||||
channel: Channel = session.query(Channel).filter_by(id=channel.id).one()
|
||||
|
||||
channel.info_json = info
|
||||
channel.info_date = now()
|
||||
|
|
|
|||
|
|
@ -153,8 +153,8 @@ class TestVideosDownloaders(TestAPI):
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
|
||||
mock_video_extract_info, mock_video_process_runner):
|
||||
async def test_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
|
||||
mock_video_extract_info, mock_video_process_runner):
|
||||
"""A video can be downloaded even if it does not have a Channel."""
|
||||
channel_dir = test_directory / 'NO CHANNEL'
|
||||
channel_dir.mkdir(parents=True)
|
||||
|
|
@ -216,12 +216,16 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
await video_download_manager.wait_for_all_downloads()
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
assert [i.url for i in downloads] == ['https://youtube.com/watch?v=video_2_url']
|
||||
assert downloads[0].settings == {
|
||||
'channel_id': 1,
|
||||
'channel_url': 'https://www.youtube.com/c/LearningSelfReliance/videos',
|
||||
}
|
||||
|
||||
|
||||
def test_get_or_create_channel(test_session):
|
||||
"""
|
||||
A Channel may need to be created for an arbitrary download. Attempt to use an existing Channel if we can
|
||||
match it.
|
||||
"""A Channel may need to be created for an arbitrary download.
|
||||
|
||||
Attempt to use an existing Channel if we can match it.
|
||||
"""
|
||||
c1 = Channel(name='foo', source_id='foo', url='foo')
|
||||
c2 = Channel(name='bar', source_id='bar')
|
||||
|
|
@ -376,8 +380,8 @@ async def test_invalid_download_url(test_session, test_download_manager, mock_vi
|
|||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager,
|
||||
mock_video_process_runner, image_file):
|
||||
async def test_video_download(test_session, test_directory, simple_channel, video_download_manager,
|
||||
mock_video_process_runner, image_file):
|
||||
"""A video download is performed, files are associated."""
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ class ConfigFile:
|
|||
raise ValueError(f'Refusing to save config file while testing: {config_file}')
|
||||
|
||||
# Only one process can write to the file.
|
||||
self.file_lock.acquire(block=True, timeout=5.0)
|
||||
acquired = self.file_lock.acquire(block=True, timeout=5.0)
|
||||
|
||||
try:
|
||||
# Config directory may not exist.
|
||||
|
|
@ -206,7 +206,8 @@ class ConfigFile:
|
|||
with config_file.open('wt') as fh:
|
||||
yaml.dump(config, fh)
|
||||
finally:
|
||||
self.file_lock.release()
|
||||
if acquired:
|
||||
self.file_lock.release()
|
||||
|
||||
def get_file(self) -> Path:
|
||||
if not self.file_name:
|
||||
|
|
|
|||
|
|
@ -50,6 +50,7 @@ class DownloadResult:
|
|||
info_json: dict = field(default_factory=dict)
|
||||
location: str = None
|
||||
success: bool = False
|
||||
settings: dict = field(default_factory=dict)
|
||||
|
||||
|
||||
class Download(ModelHelper, Base): # noqa
|
||||
|
|
@ -402,7 +403,10 @@ class DownloadManager:
|
|||
if result.downloads:
|
||||
worker_logger.info(f'Adding {len(result.downloads)} downloads from result of {download.url}')
|
||||
urls = download.filter_excluded(result.downloads)
|
||||
self.create_downloads(urls, session, downloader=download.sub_downloader)
|
||||
downloads = self.create_downloads(urls, session,
|
||||
downloader=download.sub_downloader,
|
||||
settings=result.settings,
|
||||
)
|
||||
|
||||
if try_again is False and not download.frequency:
|
||||
# Only once-downloads can fail.
|
||||
|
|
@ -533,17 +537,9 @@ class DownloadManager:
|
|||
download.manager = self
|
||||
return download
|
||||
|
||||
@optional_session
|
||||
def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False,
|
||||
sub_downloader: str = None, excluded_urls: List[str] = None) -> Download:
|
||||
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
|
||||
downloads = self.create_downloads([url], session=session, downloader=downloader, excluded_urls=excluded_urls,
|
||||
reset_attempts=reset_attempts, sub_downloader=sub_downloader)
|
||||
return downloads[0]
|
||||
|
||||
@optional_session
|
||||
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
|
||||
reset_attempts: bool = False, sub_downloader: str = None, excluded_urls: List[str] = None) \
|
||||
reset_attempts: bool = False, sub_downloader: str = None, settings: dict = None) \
|
||||
-> List[Download]:
|
||||
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
|
||||
if not all(urls):
|
||||
|
|
@ -577,7 +573,7 @@ class DownloadManager:
|
|||
download.downloader = local_downloader.name
|
||||
download.info_json = info_json or download.info_json
|
||||
download.sub_downloader = sub_downloader
|
||||
download.settings = {'excluded_urls': excluded_urls}
|
||||
download.settings = settings
|
||||
downloads.append(download)
|
||||
|
||||
try:
|
||||
|
|
@ -592,17 +588,24 @@ class DownloadManager:
|
|||
|
||||
return downloads
|
||||
|
||||
@optional_session
|
||||
def create_download(self, url: str, session: Session = None, downloader: str = None, reset_attempts: bool = False,
|
||||
sub_downloader: str = None, settings: Dict = None) -> Download:
|
||||
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
|
||||
downloads = self.create_downloads([url], session=session, downloader=downloader, settings=settings,
|
||||
reset_attempts=reset_attempts, sub_downloader=sub_downloader)
|
||||
return downloads[0]
|
||||
|
||||
def recurring_download(self, url: str, frequency: int, downloader: str = None,
|
||||
sub_downloader: str = None, reset_attempts: bool = False,
|
||||
excluded_urls: List[str] = None) -> Download:
|
||||
settings: Dict = None) -> Download:
|
||||
"""Schedule a recurring download."""
|
||||
if not frequency:
|
||||
raise ValueError('Recurring download must have a frequency!')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
download, = self.create_downloads([url, ], session=session, downloader=downloader,
|
||||
sub_downloader=sub_downloader, reset_attempts=reset_attempts,
|
||||
excluded_urls=excluded_urls)
|
||||
download, = self.create_downloads([url, ], session=session, downloader=downloader, settings=settings,
|
||||
sub_downloader=sub_downloader, reset_attempts=reset_attempts)
|
||||
download.frequency = frequency
|
||||
|
||||
return download
|
||||
|
|
|
|||
|
|
@ -221,13 +221,14 @@ async def post_download(_: Request, body: DownloadRequest):
|
|||
downloader = body.downloader
|
||||
if not downloader or downloader in ('auto', 'None', 'null'):
|
||||
downloader = None
|
||||
settings = dict(excluded_urls=excluded_urls)
|
||||
if body.frequency:
|
||||
download_manager.recurring_download(urls[0], body.frequency, downloader=downloader,
|
||||
sub_downloader=body.sub_downloader, reset_attempts=True,
|
||||
excluded_urls=excluded_urls)
|
||||
settings=settings)
|
||||
else:
|
||||
download_manager.create_downloads(urls, downloader=downloader, sub_downloader=body.sub_downloader,
|
||||
reset_attempts=True, excluded_urls=excluded_urls)
|
||||
reset_attempts=True, settings=settings)
|
||||
return response.empty()
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -11,7 +11,8 @@ import pytest
|
|||
|
||||
from wrolpi.dates import local_timezone, Seconds
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config, \
|
||||
RSSDownloader
|
||||
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
|
@ -481,3 +482,37 @@ async def test_downloads_config(test_client, test_session, test_download_manager
|
|||
# Import again, no duplicates.
|
||||
await import_downloads_config()
|
||||
assert_downloads(expected)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_excluded_urls(test_client, test_session, test_download_manager):
|
||||
"""Test that URLs that have been excluded are ignored by the download worker."""
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.already_downloaded = lambda *i, **kw: []
|
||||
http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(
|
||||
bozo=0,
|
||||
entries=[
|
||||
dict(link='https://example.com/a'),
|
||||
dict(link='https://example.org/b'), # this should be ignored
|
||||
dict(link='https://example.com/c'),
|
||||
dict(link='https://example.gov/d'),
|
||||
]
|
||||
)
|
||||
settings = dict(excluded_urls=['example.org', 'example.gov'])
|
||||
feed_download: Download = test_download_manager.create_download('https://example.com/feed',
|
||||
test_session, sub_downloader='http',
|
||||
downloader=RSSDownloader.name,
|
||||
settings=settings)
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
downloads = list(test_download_manager.get_downloads(test_session))
|
||||
assert not feed_download.error, f'Feed download had error {feed_download.error}'
|
||||
assert len(downloads) == 3, 'Domain was not ignored'
|
||||
assert [i.url for i in downloads] == \
|
||||
['https://example.com/feed', 'https://example.com/a', 'https://example.com/c'], f'Domain was not ignored'
|
||||
|
|
|
|||
|
|
@ -2,17 +2,15 @@ import json
|
|||
from http import HTTPStatus
|
||||
from itertools import zip_longest
|
||||
|
||||
import pytest
|
||||
from mock import mock
|
||||
|
||||
from wrolpi.admin import HotspotStatus
|
||||
from wrolpi.common import get_config
|
||||
from wrolpi.dates import strptime
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.downloader import Download, RSSDownloader, DownloadResult
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.test.common import TestAPI, wrap_test_db, skip_circleci
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
|
||||
|
||||
class TestRootAPI(TestAPI):
|
||||
|
|
@ -353,36 +351,3 @@ def test_get_downloaders(test_client):
|
|||
assert 'downloaders' in response.json, 'Downloaders not returned'
|
||||
assert isinstance(response.json['downloaders'], list) and len(response.json['downloaders']), \
|
||||
'No downloaders returned'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_download_excluded_urls(test_client, test_session, test_download_manager):
|
||||
"""Test that URLs that have been excluded are ignored by the download worker."""
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.already_downloaded = lambda *i, **kw: []
|
||||
http_downloader.do_download = lambda *i, **kw: DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
rss_downloader = RSSDownloader()
|
||||
test_download_manager.register_downloader(rss_downloader)
|
||||
|
||||
with mock.patch('wrolpi.downloader.parse_feed') as mock_parse_feed:
|
||||
mock_parse_feed.return_value = dict(
|
||||
bozo=0,
|
||||
entries=[
|
||||
dict(link='https://example.com/a'),
|
||||
dict(link='https://example.org/b'), # this should be ignored
|
||||
dict(link='https://example.com/c'),
|
||||
dict(link='https://example.gov/d'),
|
||||
]
|
||||
)
|
||||
feed_download: Download = test_download_manager.create_download('https://example.com/feed',
|
||||
test_session, sub_downloader='http',
|
||||
downloader=RSSDownloader.name,
|
||||
excluded_urls=['example.org', 'example.gov'])
|
||||
await test_download_manager.wait_for_all_downloads()
|
||||
|
||||
downloads = list(test_download_manager.get_downloads(test_session))
|
||||
assert not feed_download.error, f'Feed download had error {feed_download.error}'
|
||||
assert len(downloads) == 3, 'Domain was not ignored'
|
||||
assert [i.url for i in downloads] == \
|
||||
['https://example.com/feed', 'https://example.com/a', 'https://example.com/c'], f'Domain was not ignored'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue