mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 23:46:18 -05:00
DownloadResult.try_again is not necessary.
Removing unused fixture in test. Simplifying logic after download completes. Removing unused download sorting method. ChannelDownloader returns location. Hiding View link when no location is provided. Performing video upsert in try/except so any errors are attached to the Download. Including stderr only. Including logs when returning Downloader.process_runner. Warning when extention is replaced. Providing location link on Downloads page. Showing error modal on errors that fail. Handling empty URL strings on import. Fixing channel download test. Reworking how Channels schedule Video downloads. Bumping version for DownloadResult. Updating tests to use DownloadResult. VideoDownloader returns a DownloadResult.
This commit is contained in:
parent
28e983a3b3
commit
785ad8274c
17 changed files with 393 additions and 272 deletions
33
alembic/versions/1a70196c05bb_download_location.py
Normal file
33
alembic/versions/1a70196c05bb_download_location.py
Normal file
|
|
@ -0,0 +1,33 @@
|
|||
"""Download location
|
||||
|
||||
Revision ID: 1a70196c05bb
|
||||
Revises: d8fcdb7773c0
|
||||
Create Date: 2022-05-02 12:01:51.210070
|
||||
|
||||
"""
|
||||
import os
|
||||
|
||||
from alembic import op
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
# revision identifiers, used by Alembic.
|
||||
revision = '1a70196c05bb'
|
||||
down_revision = 'd8fcdb7773c0'
|
||||
branch_labels = None
|
||||
depends_on = None
|
||||
|
||||
DOCKERIZED = True if os.environ.get('DOCKER', '').lower().startswith('t') else False
|
||||
|
||||
|
||||
def upgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS location TEXT')
|
||||
session.execute('ALTER TABLE download ADD COLUMN IF NOT EXISTS error TEXT')
|
||||
|
||||
|
||||
def downgrade():
|
||||
bind = op.get_bind()
|
||||
session = Session(bind=bind)
|
||||
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS location')
|
||||
session.execute('ALTER TABLE download DROP COLUMN IF EXISTS error')
|
||||
|
|
@ -331,7 +331,7 @@ function ClearFailedDownloads({callback}) {
|
|||
class DownloadRow extends React.Component {
|
||||
|
||||
render() {
|
||||
let {url, frequency, last_successful_download, status, next_download} = this.props;
|
||||
let {url, frequency, last_successful_download, status, location, next_download} = this.props;
|
||||
let positive = false;
|
||||
if (status === 'pending') {
|
||||
positive = true;
|
||||
|
|
@ -342,7 +342,7 @@ class DownloadRow extends React.Component {
|
|||
<Table.Cell>{secondsToFrequency(frequency)}</Table.Cell>
|
||||
<Table.Cell>{last_successful_download ? secondsToDate(last_successful_download) : null}</Table.Cell>
|
||||
<Table.Cell>{secondsToDate(next_download)}</Table.Cell>
|
||||
<Table.Cell>{status}</Table.Cell>
|
||||
<Table.Cell>{location && <a href={location}>View</a>}</Table.Cell>
|
||||
</Table.Row>
|
||||
);
|
||||
}
|
||||
|
|
@ -354,6 +354,7 @@ class StoppableRow extends React.Component {
|
|||
this.state = {
|
||||
stopOpen: false,
|
||||
startOpen: false,
|
||||
errorModalOpen: false,
|
||||
};
|
||||
}
|
||||
|
||||
|
|
@ -389,9 +390,10 @@ class StoppableRow extends React.Component {
|
|||
};
|
||||
|
||||
render() {
|
||||
let {url, last_successful_download, status} = this.props;
|
||||
let {stopOpen, startOpen} = this.state;
|
||||
let {url, last_successful_download, status, location, error} = this.props;
|
||||
let {stopOpen, startOpen, errorModalOpen} = this.state;
|
||||
|
||||
let completedAtCell = last_successful_download ? secondsToDate(last_successful_download) : null;
|
||||
let buttonCell = <Table.Cell/>;
|
||||
let positive = false;
|
||||
let negative = false;
|
||||
|
|
@ -418,10 +420,21 @@ class StoppableRow extends React.Component {
|
|||
warning = status === 'deferred';
|
||||
buttonCell = (
|
||||
<Table.Cell>
|
||||
<Button
|
||||
onClick={this.openStop}
|
||||
color='red'
|
||||
>Stop</Button>
|
||||
<Confirm
|
||||
open={stopOpen}
|
||||
content='Are you sure you want to stop this download? It will not be retried.'
|
||||
confirmButton='Stop'
|
||||
onCancel={this.closeStop}
|
||||
onConfirm={this.handleStop}
|
||||
/>
|
||||
<Button
|
||||
onClick={this.openStart}
|
||||
color='green'
|
||||
>Start</Button>
|
||||
>Retry</Button>
|
||||
<Confirm
|
||||
open={startOpen}
|
||||
content='Are you sure you want to restart this download?'
|
||||
|
|
@ -431,13 +444,38 @@ class StoppableRow extends React.Component {
|
|||
/>
|
||||
</Table.Cell>
|
||||
);
|
||||
} else if (status === 'complete' && location) {
|
||||
buttonCell = (
|
||||
<Table.Cell>
|
||||
<a href={location}>View</a>
|
||||
</Table.Cell>
|
||||
);
|
||||
}
|
||||
if (error) {
|
||||
completedAtCell = (
|
||||
<Modal
|
||||
closeIcon
|
||||
onClose={() => this.setState({errorModalOpen: false})}
|
||||
onOpen={() => this.setState({errorModalOpen: true})}
|
||||
open={errorModalOpen}
|
||||
trigger={<Button icon='exclamation circle' color='red'/>}
|
||||
>
|
||||
<Modal.Header>Error</Modal.Header>
|
||||
<Modal.Content>
|
||||
<pre>{error}</pre>
|
||||
</Modal.Content>
|
||||
<Modal.Actions>
|
||||
<Button onClick={() => this.setState({errorModalOpen: false})}>Close</Button>
|
||||
</Modal.Actions>
|
||||
</Modal>
|
||||
)
|
||||
}
|
||||
|
||||
return (
|
||||
<Table.Row positive={positive} negative={negative} warning={warning}>
|
||||
<Table.Cell><a href={url} target='_blank'>{textEllipsis(url, 50)}</a></Table.Cell>
|
||||
<Table.Cell>{last_successful_download ? secondsToDate(last_successful_download) : null}</Table.Cell>
|
||||
<Table.Cell>{status}</Table.Cell>
|
||||
<Table.Cell>{completedAtCell}</Table.Cell>
|
||||
{buttonCell}
|
||||
</Table.Row>
|
||||
);
|
||||
|
|
@ -498,8 +536,8 @@ class Downloads extends React.Component {
|
|||
<Table.Header>
|
||||
<Table.Row>
|
||||
<Table.HeaderCell>URL</Table.HeaderCell>
|
||||
<Table.HeaderCell>Completed At</Table.HeaderCell>
|
||||
<Table.HeaderCell>Status</Table.HeaderCell>
|
||||
<Table.HeaderCell>Completed At</Table.HeaderCell>
|
||||
<Table.HeaderCell>Control</Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
|
|
@ -512,7 +550,7 @@ class Downloads extends React.Component {
|
|||
<Table.HeaderCell>Download Frequency</Table.HeaderCell>
|
||||
<Table.HeaderCell>Last Successful Download</Table.HeaderCell>
|
||||
<Table.HeaderCell>Next Download</Table.HeaderCell>
|
||||
<Table.HeaderCell>Status</Table.HeaderCell>
|
||||
<Table.HeaderCell>View</Table.HeaderCell>
|
||||
</Table.Row>
|
||||
</Table.Header>
|
||||
);
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@ from abc import ABC
|
|||
from typing import Tuple
|
||||
|
||||
from wrolpi.common import logger
|
||||
from wrolpi.downloader import Downloader, Download
|
||||
from wrolpi.downloader import Downloader, Download, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError
|
||||
from . import lib
|
||||
from .api import bp # noqa
|
||||
|
|
@ -17,7 +17,7 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
pretty_name = 'Archive'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<ArchiveDownloader name={self.name}>'
|
||||
return f'<ArchiveDownloader>'
|
||||
|
||||
@classmethod
|
||||
def valid_url(cls, url: str) -> Tuple[bool, None]:
|
||||
|
|
@ -26,12 +26,12 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
"""
|
||||
return True, None
|
||||
|
||||
def do_download(self, download: Download):
|
||||
def do_download(self, download: Download) -> DownloadResult:
|
||||
if download.attempts > 3:
|
||||
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
||||
|
||||
lib.do_archive(download.url)
|
||||
return True
|
||||
return DownloadResult(success=True)
|
||||
|
||||
|
||||
# Archive downloader is the last downloader which should be used.
|
||||
|
|
|
|||
|
|
@ -152,7 +152,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
|
|||
# Verify that the URL/Name/directory aren't taken
|
||||
check_for_channel_conflicts(
|
||||
session,
|
||||
url=data.url,
|
||||
url=data.url or None,
|
||||
name=data.name,
|
||||
directory=data.directory,
|
||||
source_id=data.source_id,
|
||||
|
|
@ -162,7 +162,7 @@ def create_channel(session: Session, data: schema.ChannelPostRequest, return_dic
|
|||
|
||||
channel = Channel(
|
||||
name=data.name,
|
||||
url=data.url,
|
||||
url=data.url or None,
|
||||
match_regex=data.match_regex,
|
||||
directory=data.directory, # noqa
|
||||
download_frequency=data.download_frequency,
|
||||
|
|
|
|||
|
|
@ -24,7 +24,8 @@ def test_get_channels(test_directory, channel_factory, test_client):
|
|||
assert len(response.json['channels']) == 3
|
||||
|
||||
|
||||
def test_refresh_no_channel(test_session, test_client, test_directory, simple_channel):
|
||||
@mock.patch('modules.videos.lib.validate_video')
|
||||
def test_refresh_no_channel(_, test_session, test_client, test_directory, simple_channel):
|
||||
# A regular user-defined channel
|
||||
vid1 = simple_channel.directory / 'MyChannelName_20000101_abcdefghijk_title1.mp4'
|
||||
vid1.touch()
|
||||
|
|
@ -41,22 +42,7 @@ def test_refresh_no_channel(test_session, test_client, test_directory, simple_ch
|
|||
vid4.touch()
|
||||
|
||||
test_client.post('/api/videos/refresh')
|
||||
|
||||
videos = test_session.query(Video).order_by(Video.title).all()
|
||||
assert len(videos) == 4
|
||||
|
||||
# Add some fake videos, they should be removed.
|
||||
test_session.add(Video(title='vid5'))
|
||||
test_session.add(Video(title='vid6', channel_id=simple_channel.id))
|
||||
test_session.commit()
|
||||
|
||||
videos = test_session.query(Video).order_by(Video.title).all()
|
||||
assert len(videos) == 6
|
||||
|
||||
test_client.post('/api/videos/refresh')
|
||||
|
||||
videos = test_session.query(Video).order_by(Video.title).all()
|
||||
assert len(videos) == 4
|
||||
assert test_session.query(Video).count() == 4
|
||||
|
||||
|
||||
def test_get_video(test_client, test_session, simple_channel, video_factory):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,9 @@
|
|||
import pytest
|
||||
|
||||
from modules.videos import schema
|
||||
from modules.videos.channel import lib
|
||||
from modules.videos.lib import save_channels_config, import_channels_config
|
||||
from modules.videos.models import Channel
|
||||
from wrolpi.errors import UnknownChannel
|
||||
|
||||
|
||||
|
|
@ -57,3 +60,32 @@ def test_get_channel(test_session, test_directory, channel_factory):
|
|||
for p in params:
|
||||
assert lib.get_channel(**p, return_dict=False) == channel_id_channel
|
||||
assert lib.get_channel(**p)
|
||||
|
||||
|
||||
def test_channels_no_url(test_session, test_directory):
|
||||
"""Test that a Channel's URL is coerced to None if it is empty."""
|
||||
channel1_directory = test_directory / 'channel1'
|
||||
channel1_directory.mkdir()
|
||||
channel2_directory = test_directory / 'channel2'
|
||||
channel2_directory.mkdir()
|
||||
|
||||
channel1 = schema.ChannelPostRequest(
|
||||
name='foo',
|
||||
directory=str(channel1_directory),
|
||||
)
|
||||
channel1.url = ''
|
||||
lib.create_channel(test_session, data=channel1)
|
||||
|
||||
channel2 = schema.ChannelPostRequest(
|
||||
name='bar',
|
||||
directory=str(channel2_directory),
|
||||
url=''
|
||||
)
|
||||
lib.create_channel(test_session, data=channel2)
|
||||
|
||||
channels = list(test_session.query(Channel))
|
||||
assert len(channels) == 2
|
||||
assert all(i.url is None for i in channels), [(i.name, i.url) for i in channels]
|
||||
|
||||
save_channels_config()
|
||||
import_channels_config()
|
||||
|
|
|
|||
|
|
@ -100,7 +100,8 @@ def video_factory(test_session, test_directory):
|
|||
if channel_id:
|
||||
path = test_directory / f'{title}.mp4'
|
||||
else:
|
||||
path = test_directory / 'videos' / 'NO CHANNEL' / f'{title}.mp4'
|
||||
(test_directory / 'videos/NO CHANNEL').mkdir(parents=True, exist_ok=True)
|
||||
path = test_directory / f'videos/NO CHANNEL/{title}.mp4'
|
||||
|
||||
if with_video_file:
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', path)
|
||||
|
|
@ -166,3 +167,22 @@ def test_downloader_config(test_directory):
|
|||
set_test_downloader_config(True)
|
||||
yield config_path
|
||||
set_test_downloader_config(False)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_video_extract_info():
|
||||
with mock.patch('modules.videos.downloader.extract_info') as mock_extract_info:
|
||||
yield mock_extract_info
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_video_prepare_filename():
|
||||
with mock.patch('modules.videos.downloader.prepare_filename') as mock_prepare_filename:
|
||||
yield mock_prepare_filename
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def mock_video_process_runner():
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_process_runner.return_value = (0, {})
|
||||
yield mock_process_runner
|
||||
|
|
|
|||
|
|
@ -2,9 +2,11 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import traceback
|
||||
from abc import ABC
|
||||
from typing import Tuple, List, Optional
|
||||
|
||||
import yt_dlp.utils
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
from yt_dlp.extractor import YoutubeTabIE
|
||||
|
|
@ -14,13 +16,13 @@ from wrolpi.cmd import which
|
|||
from wrolpi.common import logger, extract_domain, get_media_directory
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
|
||||
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG, DownloadResult
|
||||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
from .common import apply_info_json, get_no_channel_directory, \
|
||||
get_videos_directory
|
||||
from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id, validate_video
|
||||
from .lib import upsert_video, refresh_channel_videos, get_downloader_config, get_channel_source_id
|
||||
from .models import Video, Channel
|
||||
from .schema import ChannelPostRequest
|
||||
from .video_url_resolver import video_url_resolver
|
||||
|
|
@ -41,16 +43,24 @@ PREFERRED_VIDEO_EXTENSION = 'mp4'
|
|||
PREFERRED_VIDEO_FORMAT = 'best[height=720],22,720p,mp4-480p,mp4-360p,mp4-240p,18,bestvideo+bestaudio'
|
||||
|
||||
|
||||
def extract_info(url: str, ydl: YoutubeDL = YDL) -> dict:
|
||||
"""Get info about a video. Separated for testing."""
|
||||
return ydl.extract_info(url, download=False, process=False)
|
||||
|
||||
|
||||
def prepare_filename(entry: dict, ydl: YoutubeDL = YDL) -> str:
|
||||
"""Get filename from YoutubeDL. Separated for testing."""
|
||||
return ydl.prepare_filename(entry)
|
||||
|
||||
|
||||
class ChannelDownloader(Downloader, ABC):
|
||||
"""
|
||||
Handling downloading of all Videos in a Channel.
|
||||
"""
|
||||
"""Handles downloading of all Videos in a Channel."""
|
||||
name = 'video_channel'
|
||||
pretty_name = 'Video Channel'
|
||||
listable = False
|
||||
|
||||
def __repr__(self):
|
||||
return f'<ChannelDownloader name={self.name}>'
|
||||
return f'<ChannelDownloader>'
|
||||
|
||||
@classmethod
|
||||
def valid_url(cls, url) -> Tuple[bool, None]:
|
||||
|
|
@ -60,10 +70,8 @@ class ChannelDownloader(Downloader, ABC):
|
|||
logger.debug(f'{cls.__name__} not suitable for {url}')
|
||||
return False, None
|
||||
|
||||
def do_download(self, download: Download):
|
||||
"""
|
||||
Update a Channel's catalog, then schedule downloads of every missing video.
|
||||
"""
|
||||
def do_download(self, download: Download) -> DownloadResult:
|
||||
"""Update a Channel's catalog, then schedule downloads of every missing video."""
|
||||
with get_db_session() as session:
|
||||
channel = session.query(Channel).filter_by(url=download.url).one_or_none()
|
||||
|
||||
|
|
@ -71,6 +79,7 @@ class ChannelDownloader(Downloader, ABC):
|
|||
# Couldn't get channel by URL, this is probably a playlist. Find the channel for the playlist.
|
||||
channel_source_id = get_channel_source_id(download.url)
|
||||
channel = get_channel(source_id=channel_source_id, return_dict=False)
|
||||
channel_id = channel.id
|
||||
|
||||
update_channel(channel)
|
||||
|
||||
|
|
@ -91,7 +100,7 @@ class ChannelDownloader(Downloader, ABC):
|
|||
else:
|
||||
self.manager.start_downloads()
|
||||
|
||||
return True
|
||||
return DownloadResult(success=True, location=f'/videos/channel/{channel_id}/video')
|
||||
|
||||
|
||||
YT_DLP_BIN = which(
|
||||
|
|
@ -110,7 +119,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
pretty_name = 'Videos'
|
||||
|
||||
def __repr__(self):
|
||||
return f'<VideoDownloader name={self.name}>'
|
||||
return f'<VideoDownloader>'
|
||||
|
||||
@classmethod
|
||||
def valid_url(cls, url) -> Tuple[bool, Optional[dict]]:
|
||||
|
|
@ -120,7 +129,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
for ie in YDL._ies.values():
|
||||
if ie.suitable(url) and not ChannelDownloader.valid_url(url)[0]:
|
||||
try:
|
||||
info = YDL.extract_info(url, download=False, process=False)
|
||||
info = extract_info(url)
|
||||
return True, info
|
||||
except UnsupportedError:
|
||||
logger.debug(f'Video downloader extract_info failed for {url}')
|
||||
|
|
@ -131,7 +140,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
logger.debug(f'{cls.__name__} not suitable for {url}')
|
||||
return False, None
|
||||
|
||||
def do_download(self, download: Download):
|
||||
def do_download(self, download: Download) -> DownloadResult:
|
||||
if download.attempts >= 10:
|
||||
raise UnrecoverableDownloadError('Max download attempts reached')
|
||||
|
||||
|
|
@ -160,6 +169,7 @@ class VideoDownloader(Downloader, ABC):
|
|||
out_dir = channel.directory.path
|
||||
out_dir.mkdir(exist_ok=True, parents=True)
|
||||
|
||||
logs = None # noqa
|
||||
try:
|
||||
video_path, entry = self.prepare_filename(url, out_dir)
|
||||
# Do the real download.
|
||||
|
|
@ -177,15 +187,24 @@ class VideoDownloader(Downloader, ABC):
|
|||
'-o', file_name_format,
|
||||
url,
|
||||
]
|
||||
return_code = self.process_runner(url, cmd, out_dir)
|
||||
return_code, logs = self.process_runner(url, cmd, out_dir)
|
||||
|
||||
if return_code != 0:
|
||||
raise ValueError(f'video downloader process exited with {return_code}')
|
||||
|
||||
if not video_path.is_file():
|
||||
raise ValueError(f'Video file could not be found! {video_path}')
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# If the video is from a channel, it will already be in the database.
|
||||
source_id = entry['id']
|
||||
existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
|
||||
video = upsert_video(session, video_path, channel, id_=existing_video.id if existing_video else None)
|
||||
video_id = video.id
|
||||
except UnrecoverableDownloadError:
|
||||
raise
|
||||
except yt_dlp.utils.UnsupportedError as e:
|
||||
raise UnrecoverableDownloadError('URL is not supported by yt-dlp') from e
|
||||
except Exception as e:
|
||||
logger.warning(f'VideoDownloader failed to download: {download.url}', exc_info=e)
|
||||
if _skip_download(e):
|
||||
|
|
@ -204,18 +223,22 @@ class VideoDownloader(Downloader, ABC):
|
|||
|
||||
# Skipped downloads should not be tried again.
|
||||
raise UnrecoverableDownloadError() from e
|
||||
return
|
||||
# Download did not succeed, try again later.
|
||||
if logs and (stderr := logs.get('stderr')):
|
||||
error = f'{stderr}\n\n{traceback.format_exc()}'
|
||||
else:
|
||||
error = str(traceback.format_exc())
|
||||
return DownloadResult(success=False, error=error)
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
# If the video is from a channel, it will already be in the database.
|
||||
source_id = entry['id']
|
||||
existing_video = session.query(Video).filter_by(source_id=source_id).one_or_none()
|
||||
id_ = existing_video.id if existing_video else None
|
||||
|
||||
upsert_video(session, video_path, channel, id_=id_)
|
||||
|
||||
# Successful download.
|
||||
return True
|
||||
if channel:
|
||||
location = f'/videos/channel/{channel.id}/video/{video_id}'
|
||||
else:
|
||||
location = f'/videos/video/{video_id}'
|
||||
result = DownloadResult(
|
||||
success=True,
|
||||
location=location,
|
||||
)
|
||||
return result
|
||||
|
||||
@staticmethod
|
||||
def prepare_filename(url: str, out_dir: pathlib.Path) -> Tuple[pathlib.Path, dict]:
|
||||
|
|
@ -234,9 +257,13 @@ class VideoDownloader(Downloader, ABC):
|
|||
ydl.params['logger'] = ydl_logger
|
||||
ydl.add_default_info_extractors()
|
||||
|
||||
# Perform the real download. This saves multiple files.
|
||||
entry = ydl.extract_info(url, download=False, process=True)
|
||||
final_filename = pathlib.Path(ydl.prepare_filename(entry))
|
||||
# Get the path where the video will be saved.
|
||||
entry = extract_info(url, ydl=ydl)
|
||||
final_filename = pathlib.Path(prepare_filename(entry, ydl=ydl)).absolute()
|
||||
if final_filename.suffix == '.NA':
|
||||
# yt-dlp has odd behavior sometimes and ignores our preferred video extension.
|
||||
final_filename = final_filename.with_suffix('.mp4')
|
||||
logger.warning(f'Had to replace video extension: {final_filename}')
|
||||
return final_filename, entry
|
||||
|
||||
|
||||
|
|
@ -284,10 +311,10 @@ def update_channel(channel: Channel):
|
|||
It is expected that any missing videos will be downloaded later.
|
||||
"""
|
||||
logger.info(f'Downloading video list for {channel.name} at {channel.url} This may take several minutes.')
|
||||
info = YDL.extract_info(channel.url, download=False, process=False)
|
||||
info = extract_info(channel.url)
|
||||
if 'url' in info:
|
||||
url = info['url']
|
||||
info = YDL.extract_info(url, download=False, process=False)
|
||||
info = extract_info(url)
|
||||
|
||||
# Resolve all entries to dictionaries.
|
||||
entries = info['entries'] = list(info['entries'])
|
||||
|
|
@ -299,7 +326,7 @@ def update_channel(channel: Channel):
|
|||
for entry in entries:
|
||||
if entry['title'] == 'Uploads':
|
||||
logger.info('Youtube-DL gave back a list of URLs, found the "Uploads" URL and using it.')
|
||||
info = YDL.extract_info(entry['url'], download=False, process=False)
|
||||
info = extract_info(entry['url'])
|
||||
entries = info['entries'] = list(info['entries'])
|
||||
break
|
||||
|
||||
|
|
@ -359,7 +386,6 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
|
|||
if channel_id:
|
||||
where = 'AND channel_id = %s'
|
||||
params = (channel_id,)
|
||||
|
||||
query = f'''
|
||||
SELECT
|
||||
video.id, video.source_id, video.channel_id
|
||||
|
|
@ -380,7 +406,7 @@ def _find_all_missing_videos(channel_id: id = None) -> List[Tuple]:
|
|||
return missing_videos
|
||||
|
||||
|
||||
def find_all_missing_videos(channel_id: str = None) -> Tuple[dict, dict]:
|
||||
def find_all_missing_videos(channel_id: int = None) -> Tuple[dict, dict]:
|
||||
"""
|
||||
Find all videos that don't have a video file, but are found in the DB (taken from the channel's info_json).
|
||||
|
||||
|
|
|
|||
|
|
@ -6,7 +6,6 @@ from typing import Tuple, Optional, List, Union
|
|||
from uuid import uuid1
|
||||
|
||||
from sqlalchemy import or_
|
||||
from sqlalchemy.exc import InvalidRequestError
|
||||
from sqlalchemy.orm import Session
|
||||
from yt_dlp import YoutubeDL
|
||||
|
||||
|
|
@ -18,7 +17,8 @@ from wrolpi.media_path import MediaPath
|
|||
from wrolpi.vars import PYTEST
|
||||
from .captions import get_captions
|
||||
from .common import generate_video_paths, remove_duplicate_video_paths, apply_info_json, get_video_duration, \
|
||||
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError
|
||||
is_valid_poster, convert_image, generate_video_poster, logger, REQUIRED_OPTIONS, ConfigError, \
|
||||
get_no_channel_directory
|
||||
from .models import Channel, Video
|
||||
|
||||
logger = logger.getChild(__name__)
|
||||
|
|
@ -60,7 +60,8 @@ def refresh_channel_videos(channel: Channel):
|
|||
logger.debug(f'{channel.name}: Added {video_path.name}')
|
||||
|
||||
with get_db_curs(commit=True) as curs:
|
||||
curs.execute('DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL RETURNING id', (channel.id,))
|
||||
stmt = 'DELETE FROM video WHERE channel_id=%s AND idempotency IS NULL AND video_path IS NOT NULL RETURNING id'
|
||||
curs.execute(stmt, (channel.id,))
|
||||
deleted_count = len(curs.fetchall())
|
||||
|
||||
if deleted_count:
|
||||
|
|
@ -544,6 +545,9 @@ def import_channels_config():
|
|||
channel.name = data['name']
|
||||
channel.directory = directory
|
||||
|
||||
# A URL should not be an empty string
|
||||
data['url'] = data['url'] or None
|
||||
|
||||
# Copy existing channel data, update all values from the config. This is necessary to clear out
|
||||
# values not in the config.
|
||||
full_data = channel.dict()
|
||||
|
|
@ -571,7 +575,6 @@ def import_channels_config():
|
|||
continue
|
||||
channel_dir = channel.directory.path
|
||||
else:
|
||||
from .downloader import get_no_channel_directory
|
||||
channel_dir = get_no_channel_directory()
|
||||
|
||||
# Set favorite Videos of this Channel.
|
||||
|
|
@ -712,7 +715,11 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
|
|||
raise ValueError(f'Video path is not absolute: {video_path}')
|
||||
|
||||
# This function can update or insert a Video.
|
||||
video = session.query(Video).filter_by(id=id_).one() if id_ else Video()
|
||||
if id_:
|
||||
video = session.query(Video).filter_by(id=id_).one()
|
||||
else:
|
||||
video = Video()
|
||||
session.add(video)
|
||||
|
||||
# Set the file values, all other things can be found using these files.
|
||||
poster_path, description_path, caption_path, info_json_path = find_meta_files(video_path)
|
||||
|
|
@ -728,6 +735,9 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
|
|||
video.channel_id = channel.id
|
||||
video.idempotency = idempotency
|
||||
|
||||
session.flush()
|
||||
session.refresh(video)
|
||||
|
||||
try:
|
||||
# Fill in any missing data. Generate poster if enabled and necessary.
|
||||
validate_video(video, channel.generate_posters if channel else False)
|
||||
|
|
@ -738,11 +748,6 @@ def upsert_video(session: Session, video_path: pathlib.Path, channel: Channel =
|
|||
# inserted.
|
||||
logger.warning(f'Failed to validate {video}', exc_info=e)
|
||||
|
||||
try:
|
||||
session.add(video)
|
||||
except InvalidRequestError:
|
||||
# Video is already in a session.
|
||||
pass
|
||||
session.flush()
|
||||
|
||||
return video
|
||||
|
|
|
|||
|
|
@ -338,7 +338,7 @@ class Channel(ModelHelper, Base):
|
|||
name=self.name,
|
||||
skip_download_videos=self.skip_download_videos or [],
|
||||
source_id=self.source_id,
|
||||
url=self.url or '',
|
||||
url=self.url or None,
|
||||
)
|
||||
return config
|
||||
|
||||
|
|
|
|||
|
|
@ -347,6 +347,7 @@ def test_import_channel_downloads(test_session, channel_factory, test_channels_c
|
|||
assert download.frequency == channel1.download_frequency
|
||||
assert download.downloader == 'video_channel'
|
||||
assert download.next_download
|
||||
assert not download.error
|
||||
|
||||
next_download = str(download.next_download)
|
||||
import_channels_config()
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
import pathlib
|
||||
import shutil
|
||||
from copy import copy
|
||||
from itertools import zip_longest
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -13,22 +14,11 @@ from modules.videos.models import Channel, Video
|
|||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import DownloadManager, Download
|
||||
from wrolpi.errors import InvalidDownload
|
||||
from wrolpi.test.common import wrap_test_db, TestAPI
|
||||
from wrolpi.test.common import TestAPI
|
||||
from wrolpi.test.test_downloader import HTTPDownloader
|
||||
from wrolpi.vars import PROJECT_DIR
|
||||
|
||||
|
||||
class FakeYDL:
|
||||
|
||||
@staticmethod
|
||||
def extract_info(*a, **kw):
|
||||
return {
|
||||
'entries': [
|
||||
{'id': 1},
|
||||
],
|
||||
}
|
||||
|
||||
|
||||
def test_find_all_missing_videos(test_session, channel_factory, video_factory):
|
||||
channel1, channel2 = channel_factory(url='some url'), channel_factory(url='some other url')
|
||||
channel1.info_json = {'entries': [{'id': 'foo', 'view_count': 0}]}
|
||||
|
|
@ -155,68 +145,52 @@ class TestVideosDownloaders(TestAPI):
|
|||
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/c/LearningSelfReliance')[0], self.cd)
|
||||
self.assertEqual(self.mgr.get_downloader('https://www.youtube.com/watch?v=31jPEBiAC3c')[0], self.vd)
|
||||
|
||||
@staticmethod
|
||||
def _make_video_files(channel_dir):
|
||||
video_path = channel_dir / 'video.mp4'
|
||||
poster_path = channel_dir / 'video.jpg'
|
||||
description_path = channel_dir / 'video.description'
|
||||
info_json_path = channel_dir / 'video.info.json'
|
||||
video_path.touch()
|
||||
poster_path.touch()
|
||||
description_path.touch()
|
||||
info_json_path.touch()
|
||||
return video_path, poster_path, description_path, info_json_path
|
||||
|
||||
@wrap_test_db
|
||||
def test_video_download_no_channel(self):
|
||||
_, session = get_db_context()
|
||||
def test_video_download_no_channel(test_session, video_download_manager, video_factory, test_directory,
|
||||
mock_video_extract_info, mock_video_process_runner):
|
||||
"""A video can be downloaded even if it does not have a Channel."""
|
||||
channel_dir = test_directory / 'NO CHANNEL'
|
||||
channel_dir.mkdir(parents=True)
|
||||
video_path = channel_dir / 'video.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_path)
|
||||
|
||||
channel_dir = self.videos_dir / 'NO CHANNEL'
|
||||
channel_dir.mkdir(parents=True)
|
||||
# Video has no channel
|
||||
info_json = copy(example_video_json)
|
||||
del info_json['channel']
|
||||
del info_json['channel_id']
|
||||
del info_json['channel_url']
|
||||
|
||||
# Video paths
|
||||
video_path, _, _, _ = self._make_video_files(channel_dir)
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
mock_video_extract_info.return_value = info_json
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
video_download_manager.create_download(url, test_session)
|
||||
|
||||
# Video has no channel
|
||||
info_json = copy(example_video_json)
|
||||
del info_json['channel']
|
||||
del info_json['channel_id']
|
||||
del info_json['channel_url']
|
||||
mock_video_process_runner.assert_called_once()
|
||||
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_extract_info.return_value = info_json
|
||||
mock_prepare_filename.return_value = (video_path, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
|
||||
self.mgr.create_download(url, session)
|
||||
|
||||
mock_process_runner.assert_called_once()
|
||||
|
||||
video: Video = session.query(Video).one()
|
||||
self.assertEqual(str(video.video_path.path), f'{channel_dir}/video.mp4')
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert str(video.video_path.path) == f'{channel_dir}/video.mp4'
|
||||
|
||||
|
||||
def test_channel_download(test_session, simple_channel, video_download_manager, video_file):
|
||||
def test_download_channel(test_session, simple_channel, video_download_manager, video_file,
|
||||
mock_video_extract_info, mock_video_prepare_filename, mock_video_process_runner):
|
||||
"""Downloading (updating the catalog of) a Channel creates download records for all of it's missing videos."""
|
||||
url = 'https://www.youtube.com/c/LearningSelfReliance/videos'
|
||||
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock_extract_info.return_value = example_channel_json
|
||||
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
mock_video_prepare_filename.return_value = video_file
|
||||
mock_video_extract_info.return_value = example_channel_json
|
||||
with mock.patch('modules.videos.downloader.get_channel') as mock_get_channel:
|
||||
mock_get_channel.return_value = simple_channel
|
||||
video_download_manager.create_download(url, test_session)
|
||||
|
||||
# Two videos are in the example channel.
|
||||
downloads = video_download_manager.get_new_downloads(test_session)
|
||||
expected = ['https://youtube.com/watch?v=HQ_62YwcA80', 'https://youtube.com/watch?v=4gSz6W4Gv-o']
|
||||
for download, expected in zip(downloads, expected):
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
downloads = filter(lambda i: 'watch' in i.url, downloads)
|
||||
expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url']
|
||||
for download, expected in zip_longest(downloads, expected):
|
||||
assert download.url == expected
|
||||
# Download is skipped for test.
|
||||
assert download.status == 'new'
|
||||
# Download is run only for test.
|
||||
assert download.status == 'complete'
|
||||
|
||||
|
||||
def test_get_or_create_channel(test_session):
|
||||
|
|
@ -259,23 +233,25 @@ def test_get_or_create_channel(test_session):
|
|||
assert get_or_create_channel(source_id='quux') == channel
|
||||
|
||||
|
||||
def test_download_info_save(test_session, video_download_manager):
|
||||
def test_download_info_save(test_session, video_download_manager, mock_video_extract_info, mock_video_prepare_filename,
|
||||
mock_video_process_runner):
|
||||
"""
|
||||
An info dict from Downloader.valid_url is saved to the Downloader and can be used by the Downloader later.
|
||||
"""
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_extract_info.return_value = {'some': 'info'}
|
||||
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
|
||||
mock_extract_info.assert_called_once()
|
||||
mock_video_extract_info.return_value = {'some': 'info'}
|
||||
mock_video_prepare_filename.return_value = 'some path'
|
||||
video_download_manager.create_download('https://www.youtube.com/watch?v=HQ_62YwcA80', test_session)
|
||||
assert mock_video_extract_info.call_count == 2 # Only called twice. The "info" gathering call was skipped.
|
||||
|
||||
|
||||
def test_selected_downloader(test_session, video_download_manager):
|
||||
def test_selected_downloader(test_session, video_download_manager, successful_download):
|
||||
"""
|
||||
A user can specify which Downloader to use, no other should ever be used (even if the Download fails).
|
||||
"""
|
||||
# The test HTTP Downloader will claim to be able to handle any HTTP URL; will pretend to have downloaded anything.
|
||||
http_downloader = HTTPDownloader()
|
||||
mock_do_download = http_downloader.do_download = mock.MagicMock()
|
||||
mock_do_download.return_value = successful_download
|
||||
mock_http_valid_url = http_downloader.valid_url = mock.MagicMock()
|
||||
mock_http_valid_url.return_value = (True, None)
|
||||
video_download_manager.register_downloader(http_downloader)
|
||||
|
|
@ -286,7 +262,7 @@ def test_selected_downloader(test_session, video_download_manager):
|
|||
# DownloadManager will attempt to choose the Downloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_extract_info.side_effect = UnsupportedError('not this one')
|
||||
video_download_manager.create_downloads(['https://example.com'], test_session)
|
||||
video_download_manager.create_download('https://example.com', test_session)
|
||||
mock_http_valid_url.assert_called_once()
|
||||
mock_extract_info.assert_called_once()
|
||||
mock_do_download.assert_called_once()
|
||||
|
|
@ -298,7 +274,7 @@ def test_selected_downloader(test_session, video_download_manager):
|
|||
|
||||
# DownloadManager will only use the HTTPDownloader.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
|
||||
video_download_manager.create_download('https://example.com', test_session, downloader='http')
|
||||
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
|
||||
mock_extract_info.assert_not_called()
|
||||
mock_http_valid_url.assert_not_called()
|
||||
|
|
@ -313,7 +289,7 @@ def test_selected_downloader(test_session, video_download_manager):
|
|||
# DownloadManager will only use the HTTPDownloader, even if the download fails.
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info:
|
||||
mock_do_download.side_effect = Exception('oh no!')
|
||||
video_download_manager.create_downloads(['https://example.com'], test_session, downloader='http')
|
||||
video_download_manager.create_download('https://example.com', test_session, downloader='http')
|
||||
# DownloadManager does not try to find the Download, it trusts the "downloader" param.
|
||||
mock_extract_info.assert_not_called()
|
||||
mock_http_valid_url.assert_not_called()
|
||||
|
|
@ -351,22 +327,22 @@ def test_channel_download_no_download(test_session, video_download_manager, simp
|
|||
download_channel(simple_channel.id)
|
||||
|
||||
|
||||
def test_invalid_download_url(test_session, test_download_manager, test_downloader_config):
|
||||
def test_invalid_download_url(test_session, test_download_manager, mock_video_extract_info):
|
||||
"""An invalid url should not be attempted again."""
|
||||
_, session = get_db_context()
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.valid_url') as mock_valid_url, \
|
||||
mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader, \
|
||||
mock.patch('modules.videos.downloader.YDL') as mock_ydl:
|
||||
mock_get_downloader.return_value = (VideoDownloader(), {'channel': 'foo', 'id': 'some id'})
|
||||
mock_valid_url.return_value = (True, None)
|
||||
mock_ydl.extract_info.side_effect = UnsupportedError('oops')
|
||||
test_download_manager.create_download('url', session)
|
||||
mock.patch('wrolpi.downloader.DownloadManager.get_downloader') as mock_get_downloader:
|
||||
mock_get_downloader.return_value = (VideoDownloader(), None)
|
||||
mock_valid_url.return_value = (True, {}) # url is valid, but is not a video URL.
|
||||
mock_video_extract_info.side_effect = UnsupportedError('oops')
|
||||
test_download_manager.create_download('invalid url', session)
|
||||
|
||||
download = test_download_manager.get_downloads(test_session)[0]
|
||||
download = session.query(Download).one()
|
||||
assert download.status == 'failed'
|
||||
|
||||
|
||||
def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager):
|
||||
def test_video_download_1(test_session, test_directory, simple_channel, video_download_manager,
|
||||
mock_video_process_runner):
|
||||
simple_channel.source_id = example_video_json['channel_id']
|
||||
simple_channel.directory = test_directory / 'videos/channel name'
|
||||
simple_channel.directory.mkdir(parents=True)
|
||||
|
|
@ -376,16 +352,14 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do
|
|||
|
||||
url = 'https://www.youtube.com/watch?v=31jPEBiAC3c'
|
||||
with mock.patch('modules.videos.downloader.YDL.extract_info') as mock_extract_info, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename, \
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.process_runner') as mock_process_runner:
|
||||
mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
||||
mock_extract_info.return_value = example_video_json
|
||||
mock_prepare_filename.return_value = (video_file, {'id': 'foo'})
|
||||
mock_process_runner.return_value = 0
|
||||
|
||||
video_download_manager.create_download(url, test_session)
|
||||
|
||||
mock_process_runner.assert_called_once()
|
||||
video_url, _, out_dir = mock_process_runner.call_args[0]
|
||||
mock_video_process_runner.assert_called_once()
|
||||
video_url, _, out_dir = mock_video_process_runner.call_args[0]
|
||||
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert video_url == download.url
|
||||
|
|
@ -393,3 +367,21 @@ def test_video_download_1(test_session, test_directory, simple_channel, video_do
|
|||
|
||||
video: Video = test_session.query(Video).one()
|
||||
assert video.video_path.path.is_absolute()
|
||||
|
||||
|
||||
def test_download_result(test_session, test_directory, video_download_manager, mock_video_process_runner,
|
||||
mock_video_extract_info):
|
||||
"""VideoDownloader returns a DownloadResult when complete."""
|
||||
channel_directory = test_directory / 'videos/channel name'
|
||||
channel_directory.mkdir(parents=True)
|
||||
video_file = channel_directory / 'the video.mp4'
|
||||
shutil.copy(PROJECT_DIR / 'test/big_buck_bunny_720p_1mb.mp4', video_file)
|
||||
|
||||
mock_video_extract_info.return_value = example_video_json
|
||||
with mock.patch('modules.videos.downloader.VideoDownloader.prepare_filename') as mock_prepare_filename:
|
||||
mock_prepare_filename.return_value = [video_file, {'id': 'foo'}]
|
||||
video_download_manager.create_download('https://example.com', test_session)
|
||||
|
||||
download: Download = test_session.query(Download).one()
|
||||
assert download.url == 'https://example.com'
|
||||
assert download.location == '/videos/channel/1/video/1'
|
||||
|
|
|
|||
|
|
@ -555,9 +555,7 @@ def get_relative_to_media_directory(path: Union[str, Path]) -> Path:
|
|||
|
||||
|
||||
def minimize_dict(d: dict, keys: Iterable) -> Optional[dict]:
|
||||
"""
|
||||
Return a new dictionary that contains only the keys provided.
|
||||
"""
|
||||
"""Return a new dictionary that contains only the keys provided."""
|
||||
if d is None:
|
||||
return
|
||||
return {k: d[k] for k in set(keys) & d.keys()}
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ from sqlalchemy.orm import Session, sessionmaker
|
|||
from wrolpi.common import set_test_media_directory, Base, set_test_config
|
||||
from wrolpi.dates import set_test_now
|
||||
from wrolpi.db import postgres_engine, get_db_args
|
||||
from wrolpi.downloader import DownloadManager
|
||||
from wrolpi.downloader import DownloadManager, DownloadResult
|
||||
from wrolpi.root_api import BLUEPRINTS, api_app
|
||||
|
||||
|
||||
|
|
@ -124,3 +124,13 @@ def fake_now():
|
|||
finally:
|
||||
# reset now() to its original functionality.
|
||||
set_test_now(None) # noqa
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def successful_download():
|
||||
return DownloadResult(success=True)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def failed_download():
|
||||
return DownloadResult(error='pytest.fixture failed_download error', success=False)
|
||||
|
|
|
|||
|
|
@ -2,7 +2,8 @@ import asyncio
|
|||
import multiprocessing
|
||||
import pathlib
|
||||
import subprocess
|
||||
from collections import defaultdict
|
||||
import traceback
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import timedelta, datetime
|
||||
from enum import Enum
|
||||
from functools import partial
|
||||
|
|
@ -13,7 +14,7 @@ from sqlalchemy import Column, Integer, String, Text
|
|||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile
|
||||
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
|
||||
|
|
@ -34,6 +35,15 @@ class DownloadFrequency(int, Enum):
|
|||
days90 = 7776000
|
||||
|
||||
|
||||
@dataclass
|
||||
class DownloadResult:
|
||||
downloads: List[str] = field(default_factory=list)
|
||||
error: str = None
|
||||
info_json: dict = field(default_factory=dict)
|
||||
location: str = None
|
||||
success: bool = False
|
||||
|
||||
|
||||
class Download(ModelHelper, Base):
|
||||
"""
|
||||
Model that is used to schedule downloads.
|
||||
|
|
@ -49,6 +59,8 @@ class Download(ModelHelper, Base):
|
|||
status = Column(String, default='new')
|
||||
info_json = Column(JSONB)
|
||||
downloader = Column(Text)
|
||||
location = Column(Text)
|
||||
error = Column(Text)
|
||||
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
|
|
@ -67,6 +79,7 @@ class Download(ModelHelper, Base):
|
|||
frequency=self.frequency,
|
||||
id=self.id,
|
||||
last_successful_download=self.last_successful_download,
|
||||
location=self.location,
|
||||
next_download=self.next_download,
|
||||
status=self.status,
|
||||
url=self.url,
|
||||
|
|
@ -74,39 +87,30 @@ class Download(ModelHelper, Base):
|
|||
return d
|
||||
|
||||
def renew(self, reset_attempts: bool = False):
|
||||
"""
|
||||
Mark this Download as "new" so it will be retried.
|
||||
"""
|
||||
"""Mark this Download as "new" so it will be retried."""
|
||||
self.status = 'new'
|
||||
if reset_attempts:
|
||||
self.attempts = 0
|
||||
|
||||
def defer(self):
|
||||
"""
|
||||
Download should be tried again after a time.
|
||||
"""
|
||||
"""Download should be tried again after a time."""
|
||||
self.status = 'deferred'
|
||||
|
||||
def fail(self):
|
||||
"""
|
||||
Download should not be attempted again. A recurring Download will raise an error.
|
||||
"""
|
||||
"""Download should not be attempted again. A recurring Download will raise an error."""
|
||||
if self.frequency:
|
||||
raise ValueError('Recurring download should not be failed.')
|
||||
self.status = 'failed'
|
||||
|
||||
def started(self):
|
||||
"""
|
||||
Mark this Download as in progress.
|
||||
"""
|
||||
"""Mark this Download as in progress."""
|
||||
self.attempts += 1
|
||||
self.status = 'pending'
|
||||
|
||||
def complete(self):
|
||||
"""
|
||||
Mark this Download as successfully downloaded.
|
||||
"""
|
||||
"""Mark this Download as successfully downloaded."""
|
||||
self.status = 'complete'
|
||||
self.error = None # clear any old errors
|
||||
self.last_successful_download = now()
|
||||
|
||||
def get_downloader(self):
|
||||
|
|
@ -147,7 +151,7 @@ class Downloader:
|
|||
def valid_url(cls, url) -> Tuple[bool, Optional[dict]]:
|
||||
raise NotImplementedError()
|
||||
|
||||
def do_download(self, download: Download):
|
||||
def do_download(self, download: Download) -> DownloadResult:
|
||||
raise NotImplementedError()
|
||||
|
||||
@property
|
||||
|
|
@ -174,7 +178,7 @@ class Downloader:
|
|||
if self._kill.is_set():
|
||||
self._kill.clear()
|
||||
|
||||
def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> int:
|
||||
def process_runner(self, url: str, cmd: List[str], cwd: pathlib.Path, **kwargs) -> Tuple[int, dict]:
|
||||
"""
|
||||
Run a subprocess using the provided arguments. This process can be killed by the Download Manager.
|
||||
"""
|
||||
|
|
@ -202,13 +206,10 @@ class Downloader:
|
|||
# Output all logs from the process.
|
||||
# TODO is there a way to stream this output while the process is running?
|
||||
logger.debug(f'Download exited with {proc.returncode}')
|
||||
outs, errs = proc.communicate()
|
||||
for line in outs.decode().splitlines(): # noqa
|
||||
logger.debug(line)
|
||||
for line in errs.decode().splitlines(): # noqa
|
||||
logger.error(line)
|
||||
stdout, stderr = proc.communicate()
|
||||
logs = {'stdout': stdout, 'stderr': stderr}
|
||||
|
||||
return proc.returncode
|
||||
return proc.returncode, logs
|
||||
|
||||
|
||||
class DownloadManager:
|
||||
|
|
@ -233,15 +234,14 @@ class DownloadManager:
|
|||
|
||||
def get_downloader(self, url: str) -> Tuple[Downloader, Dict]:
|
||||
for i in self.instances:
|
||||
result = i.valid_url(url)
|
||||
valid, info = result
|
||||
valid, info = i.valid_url(url)
|
||||
if valid:
|
||||
return i, info
|
||||
raise InvalidDownload(f'Invalid URL {url=}')
|
||||
|
||||
def get_downloader_by_name(self, name: str) -> Downloader:
|
||||
"""
|
||||
Attempts to find a registered Downloader by its name. Returns None if it cannot be found.
|
||||
Tries to find a registered Downloader by its name. Returns None if it cannot be found.
|
||||
"""
|
||||
return self._instances.get(name)
|
||||
|
||||
|
|
@ -263,9 +263,7 @@ class DownloadManager:
|
|||
yield download
|
||||
|
||||
def get_or_create_download(self, url: str, session: Session) -> Download:
|
||||
"""
|
||||
Get a Download by its URL, if it cannot be found create one.
|
||||
"""
|
||||
"""Get a Download by its URL, if it cannot be found create one."""
|
||||
if not url:
|
||||
raise ValueError('Download must have a URL')
|
||||
|
||||
|
|
@ -360,29 +358,34 @@ class DownloadManager:
|
|||
download.started()
|
||||
session.commit()
|
||||
|
||||
success = True
|
||||
failure = False
|
||||
try_again = True
|
||||
try:
|
||||
success = downloader.do_download(download)
|
||||
result = downloader.do_download(download)
|
||||
except UnrecoverableDownloadError as e:
|
||||
# Download failed and should not be retried.
|
||||
logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e)
|
||||
failure = True
|
||||
result = DownloadResult(success=False, error=str(traceback.format_exc()))
|
||||
try_again = False
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
|
||||
result = DownloadResult(success=False, error=str(traceback.format_exc()))
|
||||
|
||||
# Long-running, get the download again.
|
||||
with get_db_session(commit=True) as session_:
|
||||
download = self.get_download(session_, id_=download_id)
|
||||
if failure:
|
||||
# Use a new location if provided, keep the old location if no new location is provided, otherwise
|
||||
# clear out an outdated location.
|
||||
download.location = result.location or download.location or None
|
||||
# Clear any old errors if the download succeeded.
|
||||
download.error = result.error if download.error else None
|
||||
download.next_download = self.get_next_download(download, session)
|
||||
|
||||
if try_again is False:
|
||||
download.fail()
|
||||
elif success:
|
||||
elif result.success:
|
||||
download.complete()
|
||||
download.next_download = self.get_next_download(download, session)
|
||||
else:
|
||||
download.defer()
|
||||
download.next_download = self.get_next_download(download, session)
|
||||
session_.commit()
|
||||
|
||||
if download_count:
|
||||
logger.info(f'Done doing {download_count} downloads.')
|
||||
|
|
@ -409,9 +412,7 @@ class DownloadManager:
|
|||
|
||||
@staticmethod
|
||||
def reset_downloads():
|
||||
"""
|
||||
Set any incomplete Downloads to `new` so they will be retried.
|
||||
"""
|
||||
"""Set any incomplete Downloads to `new` so they will be retried."""
|
||||
with get_db_curs(commit=True) as curs:
|
||||
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
|
||||
|
||||
|
|
@ -430,20 +431,6 @@ class DownloadManager:
|
|||
|
||||
DOWNLOAD_SORT = ('pending', 'failed', 'new', 'deferred', 'complete')
|
||||
|
||||
@classmethod
|
||||
@iterify(list)
|
||||
def _downloads_sorter(cls, downloads: List[Download]):
|
||||
"""
|
||||
Downloads should be displayed to the user by status.
|
||||
"""
|
||||
grouped_by_statuses = defaultdict(lambda: [])
|
||||
for download in downloads:
|
||||
grouped_by_statuses[download.status].append(download)
|
||||
|
||||
# Yield the downloads in the order defined above.
|
||||
for sort in cls.DOWNLOAD_SORT:
|
||||
yield from grouped_by_statuses[sort] # converted to a list by iterify()
|
||||
|
||||
@optional_session
|
||||
def get_recurring_downloads(self, session: Session = None, limit: int = None):
|
||||
"""Get all Downloads that will be downloaded in the future."""
|
||||
|
|
@ -456,7 +443,6 @@ class DownloadManager:
|
|||
if limit:
|
||||
query = query.limit(limit)
|
||||
downloads = query.all()
|
||||
downloads = self._downloads_sorter(downloads)
|
||||
return downloads
|
||||
|
||||
@optional_session
|
||||
|
|
@ -471,14 +457,11 @@ class DownloadManager:
|
|||
if limit:
|
||||
query = query.limit(limit)
|
||||
downloads = query.all()
|
||||
downloads = self._downloads_sorter(downloads)
|
||||
return downloads
|
||||
|
||||
@optional_session
|
||||
def renew_recurring_downloads(self, session: Session = None):
|
||||
"""
|
||||
Mark any recurring downloads that are due for download as "new". Start a download.
|
||||
"""
|
||||
"""Mark any recurring downloads that are due for download as "new". Start a download."""
|
||||
now_ = now()
|
||||
|
||||
recurring = self.get_recurring_downloads(session)
|
||||
|
|
@ -523,9 +506,7 @@ class DownloadManager:
|
|||
return False
|
||||
|
||||
def kill_download(self, download_id: int):
|
||||
"""
|
||||
Fail a Download. If it is pending, kill the Downloader so the download stops.
|
||||
"""
|
||||
"""Fail a Download. If it is pending, kill the Downloader so the download stops."""
|
||||
with get_db_session(commit=True) as session:
|
||||
download = self.get_download(session, id_=download_id)
|
||||
downloader = download.get_downloader()
|
||||
|
|
@ -535,9 +516,7 @@ class DownloadManager:
|
|||
download.fail()
|
||||
|
||||
def kill(self):
|
||||
"""
|
||||
Kill all downloads.
|
||||
"""
|
||||
"""Kill all downloads. Do not start new downloads."""
|
||||
self.disabled.set()
|
||||
try:
|
||||
with get_db_session(commit=True) as session:
|
||||
|
|
@ -610,9 +589,11 @@ class DownloadManager:
|
|||
stmt = f'''
|
||||
SELECT
|
||||
downloader,
|
||||
error,
|
||||
frequency,
|
||||
id,
|
||||
last_successful_download,
|
||||
location,
|
||||
next_download,
|
||||
status,
|
||||
url
|
||||
|
|
|
|||
|
|
@ -9,7 +9,7 @@ import pytest
|
|||
from wrolpi import downloader
|
||||
from wrolpi.dates import local_timezone, Seconds
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult
|
||||
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
|
||||
from wrolpi.test.common import wrap_test_db, TestAPI
|
||||
|
||||
|
|
@ -72,7 +72,7 @@ class TestDownloader(TestAPI):
|
|||
self.assertEqual(len(self.mgr.get_downloads(session)), 0)
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download.return_value = False
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
self.mgr.register_downloader(http_downloader)
|
||||
|
||||
self.mgr.create_download('https://example.com', session)
|
||||
|
|
@ -88,11 +88,11 @@ class TestDownloader(TestAPI):
|
|||
_, session = get_db_context()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download.return_value = True
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
self.mgr.register_downloader(http_downloader)
|
||||
|
||||
permissive_downloader = PermissiveDownloader(priority=100)
|
||||
permissive_downloader.do_download.return_value = False # returns a failure
|
||||
permissive_downloader.do_download.return_value = DownloadResult(success=False)
|
||||
self.mgr.register_downloader(permissive_downloader)
|
||||
|
||||
# https is handled by the HTTP Downloader.
|
||||
|
|
@ -117,7 +117,7 @@ class TestDownloader(TestAPI):
|
|||
self.assertEqual(download.attempts, 2)
|
||||
|
||||
# finally success
|
||||
permissive_downloader.do_download.return_value = True
|
||||
permissive_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
self.mgr.create_download('foo', session)
|
||||
self.mgr._do_downloads(session)
|
||||
download = self.mgr.get_download(session, 'foo')
|
||||
|
|
@ -126,32 +126,6 @@ class TestDownloader(TestAPI):
|
|||
# No downloads left.
|
||||
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
|
||||
|
||||
@wrap_test_db
|
||||
def test_max_attempts(self):
|
||||
"""
|
||||
A Download will only be attempted so many times, it will eventually be deleted.
|
||||
"""
|
||||
_, session = get_db_context()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download.return_value = False
|
||||
self.mgr.register_downloader(http_downloader)
|
||||
|
||||
self.mgr.create_download('https://example.com', session)
|
||||
download = session.query(Download).one()
|
||||
self.assertEqual(download.attempts, 1)
|
||||
|
||||
self.mgr.create_download('https://example.com', session)
|
||||
download = session.query(Download).one()
|
||||
self.assertEqual(download.attempts, 2)
|
||||
|
||||
# There are no further attempts.
|
||||
http_downloader.do_download.side_effect = UnrecoverableDownloadError()
|
||||
self.mgr.create_download('https://example.com', session)
|
||||
download = session.query(Download).one()
|
||||
self.assertEqual(download.attempts, 3)
|
||||
self.assertEqual(download.status, 'failed')
|
||||
|
||||
@wrap_test_db
|
||||
@mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
|
||||
def test_delete_old_once_downloads(self):
|
||||
|
|
@ -186,12 +160,12 @@ class TestDownloader(TestAPI):
|
|||
self.mgr.delete_old_once_downloads()
|
||||
|
||||
# Two old downloads are deleted.
|
||||
downloads = self.mgr._downloads_sorter(session.query(Download).all())
|
||||
downloads = session.query(Download).order_by(Download.id).all()
|
||||
expected = [
|
||||
dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
|
||||
dict(url='https://example.com/6', attempts=1, status='pending'),
|
||||
dict(url='https://example.com/1', frequency=1, attempts=0, status='new'),
|
||||
dict(url='https://example.com/2', frequency=1, attempts=1, status='pending'),
|
||||
dict(url='https://example.com/5', attempts=0, status='new'),
|
||||
dict(url='https://example.com/6', attempts=1, status='pending'),
|
||||
]
|
||||
for download, expected in zip_longest(downloads, expected):
|
||||
self.assertDictContains(download.dict(), expected)
|
||||
|
|
@ -298,7 +272,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
|
|||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = True
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
# Download every hour.
|
||||
|
|
@ -337,7 +311,7 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
|
|||
|
||||
# Try the download, but it fails.
|
||||
http_downloader.do_download.reset_mock()
|
||||
http_downloader.do_download.return_value = False
|
||||
http_downloader.do_download.return_value = DownloadResult(success=False)
|
||||
test_download_manager.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
download = session.query(Download).one()
|
||||
|
|
@ -345,13 +319,13 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
|
|||
assert download.status == 'deferred'
|
||||
assert download.last_successful_download == now
|
||||
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
|
||||
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
|
||||
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 0, 997200))
|
||||
assert download.next_download == expected
|
||||
|
||||
# Try the download again, it finally succeeds.
|
||||
http_downloader.do_download.reset_mock()
|
||||
now = fake_now(datetime(2020, 1, 1, 4, 0, 1))
|
||||
http_downloader.do_download.return_value = True
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.renew_recurring_downloads(session)
|
||||
test_download_manager.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
|
|
@ -360,3 +334,28 @@ def test_recurring_downloads(test_session, test_download_manager, fake_now):
|
|||
assert download.last_successful_download == now
|
||||
# Floats cause slightly wrong date.
|
||||
assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200))
|
||||
|
||||
|
||||
def test_max_attempts(test_session, test_download_manager):
|
||||
"""A Download will only be attempted so many times, it will eventually be deleted."""
|
||||
_, session = get_db_context()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
test_download_manager.create_download('https://example.com', session)
|
||||
download = session.query(Download).one()
|
||||
assert download.attempts == 1
|
||||
|
||||
test_download_manager.create_download('https://example.com', session)
|
||||
download = session.query(Download).one()
|
||||
assert download.attempts == 2
|
||||
|
||||
# There are no further attempts.
|
||||
http_downloader.do_download.side_effect = UnrecoverableDownloadError()
|
||||
test_download_manager.create_download('https://example.com', session)
|
||||
download = session.query(Download).one()
|
||||
assert download.attempts == 3
|
||||
assert download.status == 'failed'
|
||||
|
|
|
|||
|
|
@ -1,4 +1,4 @@
|
|||
__version__ = '0.4.7-beta'
|
||||
__version__ = '0.4.8-beta'
|
||||
|
||||
import subprocess
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue