mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
Creatind download manager config which will store skipped urls.
This commit is contained in:
parent
7ac3fbea2b
commit
43709ac9e7
5 changed files with 110 additions and 81 deletions
|
|
@ -11,10 +11,11 @@ from yt_dlp import YoutubeDL
|
|||
from yt_dlp.extractor import YoutubeTabIE
|
||||
from yt_dlp.utils import UnsupportedError, DownloadError
|
||||
|
||||
from wrolpi.cmd import which
|
||||
from wrolpi.common import logger, sanitize_link, extract_domain
|
||||
from wrolpi.dates import now
|
||||
from wrolpi.db import get_db_session, get_db_curs
|
||||
from wrolpi.downloader import Downloader, Download
|
||||
from wrolpi.downloader import Downloader, Download, DOWNLOAD_MANAGER_CONFIG
|
||||
from wrolpi.errors import UnknownChannel, ChannelURLEmpty, UnrecoverableDownloadError
|
||||
from wrolpi.vars import PYTEST
|
||||
from .channel.lib import create_channel, get_channel
|
||||
|
|
@ -78,6 +79,9 @@ class ChannelDownloader(Downloader, ABC):
|
|||
with get_db_session(commit=True) as session:
|
||||
for video_id, source_id, missing_video in missing_videos:
|
||||
url = video_url_resolver(domain, missing_video)
|
||||
if url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
|
||||
# Don't try to download skipped videos.
|
||||
continue
|
||||
# Schedule any missing videos for download.
|
||||
download = self.manager.get_or_create_download(url, session)
|
||||
download.downloader = 'video'
|
||||
|
|
@ -90,11 +94,12 @@ class ChannelDownloader(Downloader, ABC):
|
|||
return True
|
||||
|
||||
|
||||
YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp') # Location in docker container
|
||||
if not YT_DLP_BIN.is_file():
|
||||
YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp') # Use virtual environment location
|
||||
if not YT_DLP_BIN.is_file() and not PYTEST:
|
||||
logger.error('COULD NOT FIND YT-DLP!!!')
|
||||
YT_DLP_BIN = which(
|
||||
'yt-dlp',
|
||||
'/usr/local/bin/yt-dlp', # Location in docker container
|
||||
'/opt/wrolpi/venv/bin/yt-dlp', # Use virtual environment location
|
||||
warn=True,
|
||||
)
|
||||
|
||||
|
||||
class VideoDownloader(Downloader, ABC):
|
||||
|
|
@ -221,8 +226,6 @@ class VideoDownloader(Downloader, ABC):
|
|||
options = get_downloader_config().dict()
|
||||
options['outtmpl'] = f'{out_dir}/{options["file_name_format"]}'
|
||||
options['merge_output_format'] = PREFERRED_VIDEO_EXTENSION
|
||||
# We don't need "format" when getting the filename.
|
||||
del options['format']
|
||||
|
||||
logger.debug(f'Downloading {url} to {out_dir}')
|
||||
|
||||
|
|
|
|||
|
|
@ -2,13 +2,13 @@ import html
|
|||
import pathlib
|
||||
import re
|
||||
from collections import defaultdict
|
||||
from typing import Tuple, Optional
|
||||
from typing import Tuple, Optional, List
|
||||
from uuid import uuid1
|
||||
|
||||
from sqlalchemy.exc import InvalidRequestError
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import logger, chunks, ConfigFile, WROLPiConfig
|
||||
from wrolpi.common import logger, chunks, ConfigFile
|
||||
from wrolpi.dates import from_timestamp, Seconds
|
||||
from wrolpi.db import get_db_curs, get_db_session, optional_session
|
||||
from wrolpi.media_path import MediaPath
|
||||
|
|
@ -340,13 +340,12 @@ def set_test_channels_config(enable: bool):
|
|||
TEST_CHANNELS_CONFIG = None
|
||||
|
||||
|
||||
class DownloaderConfig(ConfigFile):
|
||||
file_name = 'downloader.yaml'
|
||||
class VideoDownloaderConfig(ConfigFile):
|
||||
file_name = 'videos_downloader.yaml'
|
||||
default_config = dict(
|
||||
continue_dl=True,
|
||||
dateafter='19900101',
|
||||
file_name_format='%(uploader)s_%(upload_date)s_%(id)s_%(title)s.%(ext)s',
|
||||
format='22 / 135 / 18 / 43',
|
||||
nooverwrites=True,
|
||||
quiet=False,
|
||||
writeautomaticsub=True,
|
||||
|
|
@ -380,14 +379,6 @@ class DownloaderConfig(ConfigFile):
|
|||
def file_name_format(self, value: str):
|
||||
self.update({'file_name_format': value})
|
||||
|
||||
@property
|
||||
def format(self) -> str:
|
||||
return self._config['format']
|
||||
|
||||
@format.setter
|
||||
def format(self, value: str):
|
||||
self.update({'format': value})
|
||||
|
||||
@property
|
||||
def nooverwrites(self) -> bool:
|
||||
return self._config['nooverwrites']
|
||||
|
|
@ -445,25 +436,25 @@ class DownloaderConfig(ConfigFile):
|
|||
self.update({'youtube_include_dash_manifest': value})
|
||||
|
||||
|
||||
DOWNLOADER_CONFIG: DownloaderConfig = DownloaderConfig(global_=True)
|
||||
TEST_DOWNLOADER_CONFIG: DownloaderConfig = None
|
||||
VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = VideoDownloaderConfig(global_=True)
|
||||
TEST_VIDEO_DOWNLOADER_CONFIG: VideoDownloaderConfig = None
|
||||
|
||||
|
||||
def get_downloader_config():
|
||||
global TEST_DOWNLOADER_CONFIG
|
||||
if isinstance(TEST_DOWNLOADER_CONFIG, DownloaderConfig):
|
||||
return TEST_DOWNLOADER_CONFIG
|
||||
global TEST_VIDEO_DOWNLOADER_CONFIG
|
||||
if isinstance(TEST_VIDEO_DOWNLOADER_CONFIG, VideoDownloaderConfig):
|
||||
return TEST_VIDEO_DOWNLOADER_CONFIG
|
||||
|
||||
global DOWNLOADER_CONFIG
|
||||
return DOWNLOADER_CONFIG
|
||||
global VIDEO_DOWNLOADER_CONFIG
|
||||
return VIDEO_DOWNLOADER_CONFIG
|
||||
|
||||
|
||||
def set_test_downloader_config(enabled: bool):
|
||||
global TEST_DOWNLOADER_CONFIG
|
||||
global TEST_VIDEO_DOWNLOADER_CONFIG
|
||||
if enabled:
|
||||
TEST_DOWNLOADER_CONFIG = DownloaderConfig()
|
||||
TEST_VIDEO_DOWNLOADER_CONFIG = VideoDownloaderConfig()
|
||||
else:
|
||||
TEST_DOWNLOADER_CONFIG = None
|
||||
TEST_VIDEO_DOWNLOADER_CONFIG = None
|
||||
|
||||
|
||||
def get_channels_config_from_db(session: Session) -> dict:
|
||||
|
|
|
|||
|
|
@ -177,12 +177,14 @@ class ConfigFile:
|
|||
default_config: dict = None
|
||||
|
||||
def __init__(self, global_: bool = False):
|
||||
if PYTEST and global_:
|
||||
self.file_lock = Lock()
|
||||
|
||||
if PYTEST:
|
||||
# Do not load a global config on import while testing. A global instance will never be used for testing.
|
||||
self._config = self.default_config.copy()
|
||||
return
|
||||
|
||||
config_file = self.get_file()
|
||||
self.file_lock = Lock()
|
||||
self._config = Manager().dict()
|
||||
# Use the default settings to initialize the config.
|
||||
self._config.update(deepcopy(self.default_config))
|
||||
|
|
@ -322,10 +324,7 @@ def get_config() -> WROLPiConfig:
|
|||
|
||||
def set_test_config(enable: bool):
|
||||
global TEST_WROLPI_CONFIG
|
||||
if enable:
|
||||
TEST_WROLPI_CONFIG = WROLPiConfig()
|
||||
else:
|
||||
TEST_WROLPI_CONFIG = None
|
||||
TEST_WROLPI_CONFIG = WROLPiConfig() if enable else None
|
||||
|
||||
|
||||
def wrol_mode_enabled() -> bool:
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@ from sqlalchemy import Column, Integer, String, Text
|
|||
from sqlalchemy.dialects.postgresql import JSONB
|
||||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag
|
||||
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag, ConfigFile
|
||||
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
|
||||
from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
|
||||
|
|
@ -276,18 +276,14 @@ class DownloadManager:
|
|||
|
||||
def create_download(self, url: str, session, downloader: str = None, skip_download: bool = False,
|
||||
reset_attempts: bool = False) -> Download:
|
||||
"""
|
||||
Schedule a URL for download. If the URL failed previously, it may be retried.
|
||||
"""
|
||||
"""Schedule a URL for download. If the URL failed previously, it may be retried."""
|
||||
downloads = self.create_downloads([url], session, downloader, skip_download=skip_download,
|
||||
reset_attempts=reset_attempts)
|
||||
return downloads[0]
|
||||
|
||||
def create_downloads(self, urls: List[str], session: Session = None, downloader: str = None,
|
||||
skip_download: bool = False, reset_attempts: bool = False) -> List[Download]:
|
||||
"""
|
||||
Schedule all URLs for download. If one cannot be downloaded, none will be added.
|
||||
"""
|
||||
"""Schedule all URLs for download. If one cannot be downloaded, none will be added."""
|
||||
if not session:
|
||||
_, session = get_db_context()
|
||||
|
||||
|
|
@ -326,11 +322,8 @@ class DownloadManager:
|
|||
|
||||
@wrol_mode_check
|
||||
@optional_session
|
||||
def _do_downloads(self, session=None):
|
||||
"""
|
||||
This method calls the Downloader's do_download method.
|
||||
"""
|
||||
# This is a long-running function, lets get a session that can be used for a long time.
|
||||
def _do_downloads(self, session: Session = None):
|
||||
"""This method calls the Downloader's do_download method."""
|
||||
if self.disabled.is_set():
|
||||
raise InvalidDownload('DownloadManager is disabled')
|
||||
|
||||
|
|
@ -346,6 +339,12 @@ class DownloadManager:
|
|||
download_id = download.id
|
||||
url = download.url
|
||||
|
||||
if url in DOWNLOAD_MANAGER_CONFIG.skip_urls:
|
||||
logger.info(f'Skipping download in skip_urls list: {url}')
|
||||
session.query(Download).filter_by(id=download.id).delete()
|
||||
session.commit()
|
||||
continue
|
||||
|
||||
downloader = self.get_downloader_by_name(download.downloader)
|
||||
if not downloader:
|
||||
downloader, info_json = self.get_downloader(download.url)
|
||||
|
|
@ -357,12 +356,12 @@ class DownloadManager:
|
|||
session.commit()
|
||||
|
||||
success = True
|
||||
failure = None
|
||||
failure = False
|
||||
try:
|
||||
success = downloader.do_download(download)
|
||||
except UnrecoverableDownloadError as e:
|
||||
# Download failed and should not be retried.
|
||||
logger.warning(f'UnrecoverableDownloadError for {download.url}', exc_info=e)
|
||||
logger.warning(f'UnrecoverableDownloadError for {url}', exc_info=e)
|
||||
failure = True
|
||||
except Exception as e:
|
||||
logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
|
||||
|
|
@ -399,9 +398,7 @@ class DownloadManager:
|
|||
return self.do_downloads_sync()
|
||||
|
||||
def start_downloads(self):
|
||||
"""
|
||||
Start an async task to do downloads. This does nothing when testing!
|
||||
"""
|
||||
"""Start an async task to do downloads. This does nothing when testing!"""
|
||||
if not PYTEST:
|
||||
asyncio.create_task(self.do_downloads())
|
||||
|
||||
|
|
@ -498,9 +495,7 @@ class DownloadManager:
|
|||
return downloads
|
||||
|
||||
def get_download(self, session: Session, url: str = None, id_: int = None) -> Optional[Download]:
|
||||
"""
|
||||
Attempt to find a Download by its URL or by its id.
|
||||
"""
|
||||
"""Attempt to find a Download by its URL or by its id."""
|
||||
query = session.query(Download)
|
||||
if url:
|
||||
download = query.filter_by(url=url).one_or_none()
|
||||
|
|
@ -514,9 +509,7 @@ class DownloadManager:
|
|||
|
||||
@optional_session
|
||||
def delete_download(self, download_id: int, session: Session = None):
|
||||
"""
|
||||
Delete a Download. Returns True if a Download was deleted, otherwise return False.
|
||||
"""
|
||||
"""Delete a Download. Returns True if a Download was deleted, otherwise return False."""
|
||||
download = self.get_download(session, id_=download_id)
|
||||
if download:
|
||||
session.delete(download)
|
||||
|
|
@ -526,7 +519,7 @@ class DownloadManager:
|
|||
|
||||
def kill_download(self, download_id: int):
|
||||
"""
|
||||
Fail a Download, if it is pending, kill the Downloader so the download stops.
|
||||
Fail a Download. If it is pending, kill the Downloader so the download stops.
|
||||
"""
|
||||
with get_db_session(commit=True) as session:
|
||||
download = self.get_download(session, id_=download_id)
|
||||
|
|
@ -552,9 +545,7 @@ class DownloadManager:
|
|||
logger.critical(f'Failed to kill downloads!', exc_info=e)
|
||||
|
||||
def enable(self):
|
||||
"""
|
||||
Enable downloading. Start downloading.
|
||||
"""
|
||||
"""Enable downloading. Start downloading."""
|
||||
self.disabled.clear()
|
||||
self.start_downloads()
|
||||
|
||||
|
|
@ -673,7 +664,7 @@ class DownloadManager:
|
|||
|
||||
# Download was successful. Spread the same-frequency downloads out over their iteration.
|
||||
start_date = local_timezone(datetime(2000, 1, 1))
|
||||
# Weeks/months/etc since start_date.
|
||||
# Weeks/months/etc. since start_date.
|
||||
iterations = ((now() - start_date) // freq).total_seconds()
|
||||
# Download slots start the next nearest iteration since 2000-01-01.
|
||||
# For example, if a weekly download was performed on 2000-01-01 it will get the same slot within
|
||||
|
|
@ -696,9 +687,38 @@ class DownloadManager:
|
|||
@optional_session
|
||||
def delete_failed(self, session: Session):
|
||||
"""Delete any failed download records."""
|
||||
session.query(Download).filter(Download.status == 'failed').delete()
|
||||
failed_downloads = session.query(Download).filter(
|
||||
Download.status == 'failed',
|
||||
Download.frequency == None,
|
||||
).all()
|
||||
|
||||
# Add all downloads to permanent skip list.
|
||||
ids = [i.id for i in failed_downloads]
|
||||
urls = list(set(DOWNLOAD_MANAGER_CONFIG.skip_urls) | {i.url for i in failed_downloads})
|
||||
DOWNLOAD_MANAGER_CONFIG.skip_urls = urls
|
||||
|
||||
# Delete all failed once-downloads.
|
||||
session.execute('DELETE FROM download WHERE id = ANY(:ids)', {'ids': ids})
|
||||
session.commit()
|
||||
|
||||
|
||||
# The global DownloadManager. This should be used everywhere!
|
||||
download_manager = DownloadManager()
|
||||
|
||||
|
||||
class DownloadMangerConfig(ConfigFile):
|
||||
file_name = 'download_manager.yaml'
|
||||
default_config = dict(
|
||||
skip_urls=[],
|
||||
)
|
||||
|
||||
@property
|
||||
def skip_urls(self) -> List[str]:
|
||||
return self._config['skip_urls']
|
||||
|
||||
@skip_urls.setter
|
||||
def skip_urls(self, value: List[str]):
|
||||
self.update({'skip_urls': value})
|
||||
|
||||
|
||||
DOWNLOAD_MANAGER_CONFIG: DownloadMangerConfig = DownloadMangerConfig()
|
||||
|
|
|
|||
|
|
@ -217,50 +217,66 @@ def test_throttle_toggle(test_session, test_client, test_config):
|
|||
|
||||
|
||||
def test_clear_downloads(test_session, test_client, test_config):
|
||||
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
|
||||
|
||||
with get_db_session(commit=True) as session:
|
||||
d1 = Download(url='https://example.com/1', status='complete')
|
||||
d2 = Download(url='https://example.com/2', status='pending')
|
||||
d3 = Download(url='https://example.com/3', status='deferred')
|
||||
d4 = Download(url='https://example.com/4', status='new')
|
||||
d5 = Download(url='https://example.com/5', status='failed')
|
||||
session.add_all([d1, d2, d3, d4, d5])
|
||||
d6 = Download(url='https://example.com/6', status='failed', frequency=60)
|
||||
session.add_all([d1, d2, d3, d4, d5, d6])
|
||||
|
||||
def check_download(download_, expected_):
|
||||
assert download_['url'] == expected_['url']
|
||||
assert download_['status'] == expected_['status']
|
||||
def check_downloads(response_, once_downloads_, recurring_downloads_, status_code=None):
|
||||
if status_code:
|
||||
assert response_.status_code == status_code
|
||||
|
||||
for download, once_download in zip_longest(response_.json['once_downloads'], once_downloads_):
|
||||
assert download
|
||||
assert once_download
|
||||
assert download['url'] == once_download['url']
|
||||
assert download['status'] == once_download['status']
|
||||
for download, recurring_download in zip_longest(response_.json['recurring_downloads'], recurring_downloads_):
|
||||
assert download
|
||||
assert recurring_download
|
||||
assert download['url'] == recurring_download['url']
|
||||
assert download['status'] == recurring_download['status']
|
||||
|
||||
# All created downloads are present.
|
||||
request, response = api_app.test_client.get('/api/download')
|
||||
expected = [
|
||||
once_downloads = [
|
||||
{'url': 'https://example.com/2', 'status': 'pending'},
|
||||
{'url': 'https://example.com/5', 'status': 'failed'},
|
||||
{'url': 'https://example.com/4', 'status': 'new'},
|
||||
{'url': 'https://example.com/3', 'status': 'deferred'},
|
||||
{'url': 'https://example.com/1', 'status': 'complete'},
|
||||
]
|
||||
for download, expected in zip_longest(response.json['once_downloads'], expected):
|
||||
check_download(download, expected)
|
||||
recurring_downloads = [{'url': 'https://example.com/6', 'status': 'failed'}]
|
||||
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
||||
|
||||
# "complete" download is removed.
|
||||
request, response = api_app.test_client.post('/api/download/clear_completed')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
request, response = api_app.test_client.get('/api/download')
|
||||
expected = [
|
||||
once_downloads = [
|
||||
{'url': 'https://example.com/2', 'status': 'pending'},
|
||||
{'url': 'https://example.com/5', 'status': 'failed'},
|
||||
{'url': 'https://example.com/4', 'status': 'new'},
|
||||
{'url': 'https://example.com/3', 'status': 'deferred'},
|
||||
]
|
||||
for download, expected in zip_longest(response.json['once_downloads'], expected):
|
||||
check_download(download, expected)
|
||||
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
||||
|
||||
# Failed "once" download is removed, recurring failed is not removed.
|
||||
request, response = api_app.test_client.post('/api/download/clear_failed')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
|
||||
request, response = api_app.test_client.get('/api/download')
|
||||
expected = [
|
||||
once_downloads = [
|
||||
{'url': 'https://example.com/2', 'status': 'pending'},
|
||||
{'url': 'https://example.com/4', 'status': 'new'},
|
||||
{'url': 'https://example.com/3', 'status': 'deferred'},
|
||||
]
|
||||
for download, expected in zip_longest(response.json['once_downloads'], expected):
|
||||
check_download(download, expected)
|
||||
check_downloads(response, once_downloads, recurring_downloads, status_code=HTTPStatus.OK)
|
||||
|
||||
# Failed once-downloads will not be downloaded again.
|
||||
assert 'https://example.com/5' in DOWNLOAD_MANAGER_CONFIG.skip_urls
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue