Creatind download manager config which will store skipped urls.

This commit is contained in:
Roland 2022-04-02 16:01:43 -06:00
parent 7ac3fbea2b
commit 43709ac9e7
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
5 changed files with 110 additions and 81 deletions

View file

@ -11,10 +11,11 @@ from yt_dlp import YoutubeDL
from yt_dlp.extractor import YoutubeTabIE
from yt_dlp.utils import UnsupportedError, DownloadError
from wrolpi.cmd import which
from wrolpi.common import logger, sanitize_link, extract_domain
from wrolpi.dates import now
from wrolpi.db import get_db_session, get_db_curs
from wrolpi.downloader import Downloader, Download
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
from wrolpi.vars import PYTEST
from .channel.lib import create_channel, get_channel
@ -78,6 +79,9 @@ class ChannelDownloader(Downloader, ABC):
with get_db_session(commit=True) as session:
for video_id, source_id, missing_video in missing_videos:
url = video_url_resolver(domain, missing_video)
if url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
# Don't try to download skipped videos.
continue
# Schedule any missing videos for download.
download = self.manager.get_or_create_download(url, session)
download.downloader = 'video'
@ -90,11 +94,12 @@ class ChannelDownloader(Downloader, ABC):
return True
YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container
if not YT_DLP_BIN.is_file():
YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location
if not YT_DLP_BIN.is_file() and not PYTEST:
logger.error('COULD NOT FIND YT-DLP!!!')
YT_DLP_BIN = which(
'yt-dlp',
'/usr/local/bin/yt-dlp', # Location in docker container
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location
warn=True,
)
class VideoDownloader(Downloader, ABC):
@ -221,8 +226,6 @@ class VideoDownloader(Downloader, ABC):
options = get_downloader_config().dict()
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
# We don't need "format" when getting the filename.
del options['format']
logger.debug(f'Downloading {url} to {out_dir}')

View file

@ -2,13 +2,13 @@ import html
import pathlib
import re
from collections import defaultdict
from typing import Tuple, Optional
from typing import Tuple, Optional, List
from uuid import uuid1
from sqlalchemy.exc import InvalidRequestError
from sqlalchemy.orm import Session
from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig
from wrolpi.common import logger, chunks, ConfigFile
from wrolpi.dates import from_timestamp, Seconds
from wrolpi.db import get_db_curs, get_db_session, optional_session
from wrolpi.media_path import MediaPath
@ -340,13 +340,12 @@ def set_test_channels_config(enable: bool):
TEST_CHANNELS_CONFIG = None
class DownloaderConfig(ConfigFile):
file_name = 'downloader.yaml'
class VideoDownloaderConfig(ConfigFile):
file_name = 'videos_downloader.yaml'
default_config = dict(
continue_dl=True,
dateafter='19900101',
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
format='22 / 135 / 18 / 43',
nooverwrites=True,
quiet=False,
writeautomaticsub=True,
@ -380,14 +379,6 @@ class DownloaderConfig(ConfigFile):
def file_name_format(self, value: str):
self.update({'file_name_format': value})
@property
def format(self) -> str:
return self._config['format']
@format.setter
def format(self, value: str):
self.update({'format': value})
@property
def nooverwrites(self) -> bool:
return self._config['nooverwrites']
@ -445,25 +436,25 @@ class DownloaderConfig(ConfigFile):
self.update({'youtube_include_dash_manifest': value})
DOWNLOADER_CONFIG: DownloaderConfig = DownloaderConfig(global_=True)
TEST_DOWNLOADER_CONFIG: DownloaderConfig = None
VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig(global_=True)
TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None
def get_downloader_config():
global TEST_DOWNLOADER_CONFIG
if isinstance(TEST_DOWNLOADER_CONFIG, DownloaderConfig):
return TEST_DOWNLOADER_CONFIG
global TEST_VIDEO_DOWNLOADER_CONFIG
if isinstance(TEST_VIDEO_DOWNLOADER_CONFIG, VideoDownloaderConfig):
return TEST_VIDEO_DOWNLOADER_CONFIG
global DOWNLOADER_CONFIG
return DOWNLOADER_CONFIG
global VIDEO_DOWNLOADER_CONFIG
return VIDEO_DOWNLOADER_CONFIG
def set_test_downloader_config(enabled: bool):
global TEST_DOWNLOADER_CONFIG
global TEST_VIDEO_DOWNLOADER_CONFIG
if enabled:
TEST_DOWNLOADER_CONFIG = DownloaderConfig()
TEST_VIDEO_DOWNLOADER_CONFIG = VideoDownloaderConfig()
else:
TEST_DOWNLOADER_CONFIG = None
TEST_VIDEO_DOWNLOADER_CONFIG = None
def get_channels_config_from_db(session: Session) -> dict:

View file

@ -177,12 +177,14 @@ class ConfigFile:
default_config: dict = None
def __init__(self, global_: bool = False):
if PYTEST and global_:
self.file_lock = Lock()
if PYTEST:
# Do not load a global config on import while testing. A global instance will never be used for testing.
self._config = self.default_config.copy()
return
config_file = self.get_file()
self.file_lock = Lock()
self._config = Manager().dict()
# Use the default settings to initialize the config.
self._config.update(deepcopy(self.default_config))
@ -322,10 +324,7 @@ def get_config() -> WROLPiConfig:
def set_test_config(enable: bool):
global TEST_WROLPI_CONFIG
if enable:
TEST_WROLPI_CONFIG = WROLPiConfig()
else:
TEST_WROLPI_CONFIG = None
TEST_WROLPI_CONFIG = WROLPiConfig() if enable else None
def wrol_mode_enabled() -> bool:

View file

@ -13,7 +13,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
@ -276,18 +276,14 @@ class DownloadManager:
def create_download(self, url: str, session, downloader: str = None, skip_download: bool = False,
reset_attempts: bool = False) -> Download:
"""
Schedule a URL for download. If the URL failed previously, it may be retried.
"""
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
downloads = self.create_downloads([url], session, downloader, skip_download=skip_download,
reset_attempts=reset_attempts)
return downloads[0]
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
skip_download: bool = False, reset_attempts: bool = False) -> List[Download]:
"""
Schedule all URLs for download. If one cannot be downloaded, none will be added.
"""
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
if not session:
_, session = get_db_context()
@ -326,11 +322,8 @@ class DownloadManager:
@wrol_mode_check
@optional_session
def _do_downloads(self, session=None):
"""
This method calls the Downloader's do_download method.
"""
# This is a long-running function, lets get a session that can be used for a long time.
def _do_downloads(self, session: Session = None):
"""This method calls the Downloader's do_download method."""
if self.disabled.is_set():
raise InvalidDownload('DownloadManager is disabled')
@ -346,6 +339,12 @@ class DownloadManager:
download_id = download.id
url = download.url
if url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
logger.info(f'Skipping download in skip_urls list: {url}')
session.query(Download).filter_by(id=download.id).delete()
session.commit()
continue
downloader = self.get_downloader_by_name(download.downloader)
if not downloader:
downloader, info_json = self.get_downloader(download.url)
@ -357,12 +356,12 @@ class DownloadManager:
session.commit()
success = True
failure = None
failure = False
try:
success = downloader.do_download(download)
except UnrecoverableDownloadError as e:
# Download failed and should not be retried.
logger.warning(f'UnrecoverableDownloadError for {download.url}', exc_info=e)
logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e)
failure = True
except Exception as e:
logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
@ -399,9 +398,7 @@ class DownloadManager:
return self.do_downloads_sync()
def start_downloads(self):
"""
Start an async task to do downloads. This does nothing when testing!
"""
"""Start an async task to do downloads. This does nothing when testing!"""
if not PYTEST:
asyncio.create_task(self.do_downloads())
@ -498,9 +495,7 @@ class DownloadManager:
return downloads
def get_download(self, session: Session, url: str = None, id_: int = None) -> Optional[Download]:
"""
Attempt to find a Download by its URL or by its id.
"""
"""Attempt to find a Download by its URL or by its id."""
query = session.query(Download)
if url:
download = query.filter_by(url=url).one_or_none()
@ -514,9 +509,7 @@ class DownloadManager:
@optional_session
def delete_download(self, download_id: int, session: Session = None):
"""
Delete a Download. Returns True if a Download was deleted, otherwise return False.
"""
"""Delete a Download. Returns True if a Download was deleted, otherwise return False."""
download = self.get_download(session, id_=download_id)
if download:
session.delete(download)
@ -526,7 +519,7 @@ class DownloadManager:
def kill_download(self, download_id: int):
"""
Fail a Download, if it is pending, kill the Downloader so the download stops.
Fail a Download. If it is pending, kill the Downloader so the download stops.
"""
with get_db_session(commit=True) as session:
download = self.get_download(session, id_=download_id)
@ -552,9 +545,7 @@ class DownloadManager:
logger.critical(f'Failed to kill downloads!', exc_info=e)
def enable(self):
"""
Enable downloading. Start downloading.
"""
"""Enable downloading. Start downloading."""
self.disabled.clear()
self.start_downloads()
@ -673,7 +664,7 @@ class DownloadManager:
# Download was successful. Spread the same-frequency downloads out over their iteration.
start_date = local_timezone(datetime(2000, 1, 1))
# Weeks/months/etc since start_date.
# Weeks/months/etc. since start_date.
iterations = ((now() - start_date) // freq).total_seconds()
# Download slots start the next nearest iteration since 2000-01-01.
# For example, if a weekly download was performed on 2000-01-01 it will get the same slot within
@ -696,9 +687,38 @@ class DownloadManager:
@optional_session
def delete_failed(self, session: Session):
"""Delete any failed download records."""
session.query(Download).filter(Download.status == 'failed').delete()
failed_downloads = session.query(Download).filter(
Download.status == 'failed',
Download.frequency == None,
).all()
# Add all downloads to permanent skip list.
ids = [i.id for i in failed_downloads]
urls = list(set(DOWNLOAD_MANAGER_CONFIG.skip_urls) | {i.url for i in failed_downloads})
DOWNLOAD_MANAGER_CONFIG.skip_urls = urls
# Delete all failed once-downloads.
session.execute('DELETE FROM download WHERE id = ANY(:ids)', {'ids': ids})
session.commit()
# The global DownloadManager. This should be used everywhere!
download_manager = DownloadManager()
class DownloadMangerConfig(ConfigFile):
file_name = 'download_manager.yaml'
default_config = dict(
skip_urls=[],
)
@property
def skip_urls(self) -> List[str]:
return self._config['skip_urls']
@skip_urls.setter
def skip_urls(self, value: List[str]):
self.update({'skip_urls': value})
DOWNLOAD_MANAGER_CONFIG: DownloadMangerConfig = DownloadMangerConfig()

View file

@ -217,50 +217,66 @@ def test_throttle_toggle(test_session, test_client, test_config):
def test_clear_downloads(test_session, test_client, test_config):
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
with get_db_session(commit=True) as session:
d1 = Download(url='https://example.com/1', status='complete')
d2 = Download(url='https://example.com/2', status='pending')
d3 = Download(url='https://example.com/3', status='deferred')
d4 = Download(url='https://example.com/4', status='new')
d5 = Download(url='https://example.com/5', status='failed')
session.add_all([d1, d2, d3, d4, d5])
d6 = Download(url='https://example.com/6', status='failed', frequency=60)
session.add_all([d1, d2, d3, d4, d5, d6])
def check_download(download_, expected_):
assert download_['url'] == expected_['url']
assert download_['status'] == expected_['status']
def check_downloads(response_, once_downloads_, recurring_downloads_, status_code=None):
if status_code:
assert response_.status_code == status_code
for download, once_download in zip_longest(response_.json['once_downloads'], once_downloads_):
assert download
assert once_download
assert download['url'] == once_download['url']
assert download['status'] == once_download['status']
for download, recurring_download in zip_longest(response_.json['recurring_downloads'], recurring_downloads_):
assert download
assert recurring_download
assert download['url'] == recurring_download['url']
assert download['status'] == recurring_download['status']
# All created downloads are present.
request, response = api_app.test_client.get('/api/download')
expected = [
once_downloads = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/5', 'status': 'failed'},
{'url': 'https://example.com/4', 'status': 'new'},
{'url': 'https://example.com/3', 'status': 'deferred'},
{'url': 'https://example.com/1', 'status': 'complete'},
]
for download, expected in zip_longest(response.json['once_downloads'], expected):
check_download(download, expected)
recurring_downloads = [{'url': 'https://example.com/6', 'status': 'failed'}]
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
# "complete" download is removed.
request, response = api_app.test_client.post('/api/download/clear_completed')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = api_app.test_client.get('/api/download')
expected = [
once_downloads = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/5', 'status': 'failed'},
{'url': 'https://example.com/4', 'status': 'new'},
{'url': 'https://example.com/3', 'status': 'deferred'},
]
for download, expected in zip_longest(response.json['once_downloads'], expected):
check_download(download, expected)
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
# Failed "once" download is removed, recurring failed is not removed.
request, response = api_app.test_client.post('/api/download/clear_failed')
assert response.status_code == HTTPStatus.NO_CONTENT
request, response = api_app.test_client.get('/api/download')
expected = [
once_downloads = [
{'url': 'https://example.com/2', 'status': 'pending'},
{'url': 'https://example.com/4', 'status': 'new'},
{'url': 'https://example.com/3', 'status': 'deferred'},
]
for download, expected in zip_longest(response.json['once_downloads'], expected):
check_download(download, expected)
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
# Failed once-downloads will not be downloaded again.
assert 'https://example.com/5' in DOWNLOAD_MANAGER_CONFIG.skip_urls