From 43709ac9e7c9dd130234809b5004ffef2fe7a352 Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 2 Apr 2022 16:01:43 -0600 Subject: [PATCH] Creatind download manager config which will store skipped urls. --- modules/videos/downloader.py | 19 +++++---- modules/videos/lib.py | 37 +++++++---------- wrolpi/common.py | 11 +++-- wrolpi/downloader.py | 78 ++++++++++++++++++++++-------------- wrolpi/test/test_root_api.py | 46 ++++++++++++++------- 5 files changed, 110 insertions(+), 81 deletions(-) diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py index d302b551..b13c6a99 100755 --- a/modules/videos/downloader.py +++ b/modules/videos/downloader.py @@ -11,10 +11,11 @@ from yt_dlp import YoutubeDL from yt_dlp.extractor import YoutubeTabIE from yt_dlp.utils import UnsupportedError, DownloadError +from wrolpi.cmd import which from wrolpi.common import logger, sanitize_link, extract_domain from wrolpi.dates import now from wrolpi.db import get_db_session, get_db_curs -from wrolpi.downloader import Downloader, Download +from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError from wrolpi.vars import PYTEST from .channel.lib import create_channel, get_channel @@ -78,6 +79,9 @@ class ChannelDownloader(Downloader, ABC): with get_db_session(commit=True) as session: for video_id, source_id, missing_video in missing_videos: url = video_url_resolver(domain, missing_video) + if url in DOWNLOAD_MANAGER_CONFIG.skip_urls: + # Don't try to download skipped videos. + continue # Schedule any missing videos for download. download = self.manager.get_or_create_download(url, session) download.downloader = 'video' @@ -90,11 +94,12 @@ class ChannelDownloader(Downloader, ABC): return True -YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container -if not YT_DLP_BIN.is_file(): - YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location -if not YT_DLP_BIN.is_file() and not PYTEST: - logger.error('COULD NOT FIND YT-DLP!!!') +YT_DLP_BIN = which( + 'yt-dlp', + '/usr/local/bin/yt-dlp', # Location in docker container + '/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location + warn=True, +) class VideoDownloader(Downloader, ABC): @@ -221,8 +226,6 @@ class VideoDownloader(Downloader, ABC): options = get_downloader_config().dict() options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}' options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION - # We don't need "format" when getting the filename. - del options['format'] logger.debug(f'Downloading {url} to {out_dir}') diff --git a/modules/videos/lib.py b/modules/videos/lib.py index cf9b5b43..5df353fb 100644 --- a/modules/videos/lib.py +++ b/modules/videos/lib.py @@ -2,13 +2,13 @@ import html import pathlib import re from collections import defaultdict -from typing import Tuple, Optional +from typing import Tuple, Optional, List from uuid import uuid1 from sqlalchemy.exc import InvalidRequestError from sqlalchemy.orm import Session -from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig +from wrolpi.common import logger, chunks, ConfigFile from wrolpi.dates import from_timestamp, Seconds from wrolpi.db import get_db_curs, get_db_session, optional_session from wrolpi.media_path import MediaPath @@ -340,13 +340,12 @@ def set_test_channels_config(enable: bool): TEST_CHANNELS_CONFIG = None -class DownloaderConfig(ConfigFile): - file_name = 'downloader.yaml' +class VideoDownloaderConfig(ConfigFile): + file_name = 'videos_downloader.yaml' default_config = dict( continue_dl=True, dateafter='19900101', file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s', - format='22 / 135 / 18 / 43', nooverwrites=True, quiet=False, writeautomaticsub=True, @@ -380,14 +379,6 @@ class DownloaderConfig(ConfigFile): def file_name_format(self, value: str): self.update({'file_name_format': value}) - @property - def format(self) -> str: - return self._config['format'] - - @format.setter - def format(self, value: str): - self.update({'format': value}) - @property def nooverwrites(self) -> bool: return self._config['nooverwrites'] @@ -445,25 +436,25 @@ class DownloaderConfig(ConfigFile): self.update({'youtube_include_dash_manifest': value}) -DOWNLOADER_CONFIG: DownloaderConfig = DownloaderConfig(global_=True) -TEST_DOWNLOADER_CONFIG: DownloaderConfig = None +VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig(global_=True) +TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None def get_downloader_config(): - global TEST_DOWNLOADER_CONFIG - if isinstance(TEST_DOWNLOADER_CONFIG, DownloaderConfig): - return TEST_DOWNLOADER_CONFIG + global TEST_VIDEO_DOWNLOADER_CONFIG + if isinstance(TEST_VIDEO_DOWNLOADER_CONFIG, VideoDownloaderConfig): + return TEST_VIDEO_DOWNLOADER_CONFIG - global DOWNLOADER_CONFIG - return DOWNLOADER_CONFIG + global VIDEO_DOWNLOADER_CONFIG + return VIDEO_DOWNLOADER_CONFIG def set_test_downloader_config(enabled: bool): - global TEST_DOWNLOADER_CONFIG + global TEST_VIDEO_DOWNLOADER_CONFIG if enabled: - TEST_DOWNLOADER_CONFIG = DownloaderConfig() + TEST_VIDEO_DOWNLOADER_CONFIG = VideoDownloaderConfig() else: - TEST_DOWNLOADER_CONFIG = None + TEST_VIDEO_DOWNLOADER_CONFIG = None def get_channels_config_from_db(session: Session) -> dict: diff --git a/wrolpi/common.py b/wrolpi/common.py index e4ff7287..dd8b6404 100644 --- a/wrolpi/common.py +++ b/wrolpi/common.py @@ -177,12 +177,14 @@ class ConfigFile: default_config: dict = None def __init__(self, global_: bool = False): - if PYTEST and global_: + self.file_lock = Lock() + + if PYTEST: # Do not load a global config on import while testing. A global instance will never be used for testing. + self._config = self.default_config.copy() return config_file = self.get_file() - self.file_lock = Lock() self._config = Manager().dict() # Use the default settings to initialize the config. self._config.update(deepcopy(self.default_config)) @@ -322,10 +324,7 @@ def get_config() -> WROLPiConfig: def set_test_config(enable: bool): global TEST_WROLPI_CONFIG - if enable: - TEST_WROLPI_CONFIG = WROLPiConfig() - else: - TEST_WROLPI_CONFIG = None + TEST_WROLPI_CONFIG = WROLPiConfig() if enable else None def wrol_mode_enabled() -> bool: diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index c1c03b02..cb9304f9 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -13,7 +13,7 @@ from sqlalchemy import Column, Integer, String, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session -from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag +from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError @@ -276,18 +276,14 @@ class DownloadManager: def create_download(self, url: str, session, downloader: str = None, skip_download: bool = False, reset_attempts: bool = False) -> Download: - """ - Schedule a URL for download. If the URL failed previously, it may be retried. - """ + """Schedule a URL for download. If the URL failed previously, it may be retried.""" downloads = self.create_downloads([url], session, downloader, skip_download=skip_download, reset_attempts=reset_attempts) return downloads[0] def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None, skip_download: bool = False, reset_attempts: bool = False) -> List[Download]: - """ - Schedule all URLs for download. If one cannot be downloaded, none will be added. - """ + """Schedule all URLs for download. If one cannot be downloaded, none will be added.""" if not session: _, session = get_db_context() @@ -326,11 +322,8 @@ class DownloadManager: @wrol_mode_check @optional_session - def _do_downloads(self, session=None): - """ - This method calls the Downloader's do_download method. - """ - # This is a long-running function, lets get a session that can be used for a long time. + def _do_downloads(self, session: Session = None): + """This method calls the Downloader's do_download method.""" if self.disabled.is_set(): raise InvalidDownload('DownloadManager is disabled') @@ -346,6 +339,12 @@ class DownloadManager: download_id = download.id url = download.url + if url in DOWNLOAD_MANAGER_CONFIG.skip_urls: + logger.info(f'Skipping download in skip_urls list: {url}') + session.query(Download).filter_by(id=download.id).delete() + session.commit() + continue + downloader = self.get_downloader_by_name(download.downloader) if not downloader: downloader, info_json = self.get_downloader(download.url) @@ -357,12 +356,12 @@ class DownloadManager: session.commit() success = True - failure = None + failure = False try: success = downloader.do_download(download) except UnrecoverableDownloadError as e: # Download failed and should not be retried. - logger.warning(f'UnrecoverableDownloadError for {download.url}', exc_info=e) + logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e) failure = True except Exception as e: logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e) @@ -399,9 +398,7 @@ class DownloadManager: return self.do_downloads_sync() def start_downloads(self): - """ - Start an async task to do downloads. This does nothing when testing! - """ + """Start an async task to do downloads. This does nothing when testing!""" if not PYTEST: asyncio.create_task(self.do_downloads()) @@ -498,9 +495,7 @@ class DownloadManager: return downloads def get_download(self, session: Session, url: str = None, id_: int = None) -> Optional[Download]: - """ - Attempt to find a Download by its URL or by its id. - """ + """Attempt to find a Download by its URL or by its id.""" query = session.query(Download) if url: download = query.filter_by(url=url).one_or_none() @@ -514,9 +509,7 @@ class DownloadManager: @optional_session def delete_download(self, download_id: int, session: Session = None): - """ - Delete a Download. Returns True if a Download was deleted, otherwise return False. - """ + """Delete a Download. Returns True if a Download was deleted, otherwise return False.""" download = self.get_download(session, id_=download_id) if download: session.delete(download) @@ -526,7 +519,7 @@ class DownloadManager: def kill_download(self, download_id: int): """ - Fail a Download, if it is pending, kill the Downloader so the download stops. + Fail a Download. If it is pending, kill the Downloader so the download stops. """ with get_db_session(commit=True) as session: download = self.get_download(session, id_=download_id) @@ -552,9 +545,7 @@ class DownloadManager: logger.critical(f'Failed to kill downloads!', exc_info=e) def enable(self): - """ - Enable downloading. Start downloading. - """ + """Enable downloading. Start downloading.""" self.disabled.clear() self.start_downloads() @@ -673,7 +664,7 @@ class DownloadManager: # Download was successful. Spread the same-frequency downloads out over their iteration. start_date = local_timezone(datetime(2000, 1, 1)) - # Weeks/months/etc since start_date. + # Weeks/months/etc. since start_date. iterations = ((now() - start_date) // freq).total_seconds() # Download slots start the next nearest iteration since 2000-01-01. # For example, if a weekly download was performed on 2000-01-01 it will get the same slot within @@ -696,9 +687,38 @@ class DownloadManager: @optional_session def delete_failed(self, session: Session): """Delete any failed download records.""" - session.query(Download).filter(Download.status == 'failed').delete() + failed_downloads = session.query(Download).filter( + Download.status == 'failed', + Download.frequency == None, + ).all() + + # Add all downloads to permanent skip list. + ids = [i.id for i in failed_downloads] + urls = list(set(DOWNLOAD_MANAGER_CONFIG.skip_urls) | {i.url for i in failed_downloads}) + DOWNLOAD_MANAGER_CONFIG.skip_urls = urls + + # Delete all failed once-downloads. + session.execute('DELETE FROM download WHERE id = ANY(:ids)', {'ids': ids}) session.commit() # The global DownloadManager. This should be used everywhere! download_manager = DownloadManager() + + +class DownloadMangerConfig(ConfigFile): + file_name = 'download_manager.yaml' + default_config = dict( + skip_urls=[], + ) + + @property + def skip_urls(self) -> List[str]: + return self._config['skip_urls'] + + @skip_urls.setter + def skip_urls(self, value: List[str]): + self.update({'skip_urls': value}) + + +DOWNLOAD_MANAGER_CONFIG: DownloadMangerConfig = DownloadMangerConfig() diff --git a/wrolpi/test/test_root_api.py b/wrolpi/test/test_root_api.py index 7f80c8b1..a044142e 100644 --- a/wrolpi/test/test_root_api.py +++ b/wrolpi/test/test_root_api.py @@ -217,50 +217,66 @@ def test_throttle_toggle(test_session, test_client, test_config): def test_clear_downloads(test_session, test_client, test_config): + from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG + with get_db_session(commit=True) as session: d1 = Download(url='https://example.com/1', status='complete') d2 = Download(url='https://example.com/2', status='pending') d3 = Download(url='https://example.com/3', status='deferred') d4 = Download(url='https://example.com/4', status='new') d5 = Download(url='https://example.com/5', status='failed') - session.add_all([d1, d2, d3, d4, d5]) + d6 = Download(url='https://example.com/6', status='failed', frequency=60) + session.add_all([d1, d2, d3, d4, d5, d6]) - def check_download(download_, expected_): - assert download_['url'] == expected_['url'] - assert download_['status'] == expected_['status'] + def check_downloads(response_, once_downloads_, recurring_downloads_, status_code=None): + if status_code: + assert response_.status_code == status_code + for download, once_download in zip_longest(response_.json['once_downloads'], once_downloads_): + assert download + assert once_download + assert download['url'] == once_download['url'] + assert download['status'] == once_download['status'] + for download, recurring_download in zip_longest(response_.json['recurring_downloads'], recurring_downloads_): + assert download + assert recurring_download + assert download['url'] == recurring_download['url'] + assert download['status'] == recurring_download['status'] + + # All created downloads are present. request, response = api_app.test_client.get('/api/download') - expected = [ + once_downloads = [ {'url': 'https://example.com/2', 'status': 'pending'}, {'url': 'https://example.com/5', 'status': 'failed'}, {'url': 'https://example.com/4', 'status': 'new'}, {'url': 'https://example.com/3', 'status': 'deferred'}, {'url': 'https://example.com/1', 'status': 'complete'}, ] - for download, expected in zip_longest(response.json['once_downloads'], expected): - check_download(download, expected) + recurring_downloads = [{'url': 'https://example.com/6', 'status': 'failed'}] + check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK) + # "complete" download is removed. request, response = api_app.test_client.post('/api/download/clear_completed') assert response.status_code == HTTPStatus.NO_CONTENT - request, response = api_app.test_client.get('/api/download') - expected = [ + once_downloads = [ {'url': 'https://example.com/2', 'status': 'pending'}, {'url': 'https://example.com/5', 'status': 'failed'}, {'url': 'https://example.com/4', 'status': 'new'}, {'url': 'https://example.com/3', 'status': 'deferred'}, ] - for download, expected in zip_longest(response.json['once_downloads'], expected): - check_download(download, expected) + check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK) + # Failed "once" download is removed, recurring failed is not removed. request, response = api_app.test_client.post('/api/download/clear_failed') assert response.status_code == HTTPStatus.NO_CONTENT - request, response = api_app.test_client.get('/api/download') - expected = [ + once_downloads = [ {'url': 'https://example.com/2', 'status': 'pending'}, {'url': 'https://example.com/4', 'status': 'new'}, {'url': 'https://example.com/3', 'status': 'deferred'}, ] - for download, expected in zip_longest(response.json['once_downloads'], expected): - check_download(download, expected) + check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK) + + # Failed once-downloads will not be downloaded again. + assert 'https://example.com/5' in DOWNLOAD_MANAGER_CONFIG.skip_urls