mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Downloads are saved to the download manager config. They are imported on startup.
This commit is contained in:
parent
241b6175c6
commit
6c2a5d723e
4 changed files with 200 additions and 12 deletions
7
main.py
7
main.py
|
|
@ -10,7 +10,7 @@ from sanic.signals import Event
|
|||
from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin
|
||||
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, wrol_mode_enabled
|
||||
from wrolpi.dates import set_timezone
|
||||
from wrolpi.downloader import download_manager
|
||||
from wrolpi.downloader import download_manager, import_downloads_config
|
||||
from wrolpi.files.lib import cancel_refresh_tasks
|
||||
from wrolpi.root_api import api_app
|
||||
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
|
||||
|
|
@ -170,6 +170,11 @@ def set_log_level(args):
|
|||
logger.warning(f'Logging level: {level_name}')
|
||||
|
||||
|
||||
@api_app.before_server_start
|
||||
async def download_startup(app: Sanic):
|
||||
await import_downloads_config()
|
||||
|
||||
|
||||
@api_app.after_server_start
|
||||
async def periodic_downloads(app: Sanic):
|
||||
"""
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
import asyncio
|
||||
import pathlib
|
||||
import shutil
|
||||
from copy import copy
|
||||
from itertools import zip_longest
|
||||
from unittest import mock
|
||||
|
||||
import pytest
|
||||
|
|
@ -196,14 +196,15 @@ async def test_download_channel(test_session, simple_channel, video_download_man
|
|||
video_download_manager.create_download(url)
|
||||
await video_download_manager.wait_for_all_downloads()
|
||||
|
||||
# Let background tasks run.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# Two videos are in the example channel.
|
||||
downloads = video_download_manager.get_once_downloads(test_session)
|
||||
downloads = filter(lambda i: 'watch' in i.url, downloads)
|
||||
expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url']
|
||||
for download, expected in zip_longest(downloads, expected):
|
||||
assert download.url == expected
|
||||
# Download is run only for test.
|
||||
assert download.status == 'complete'
|
||||
assert {i.url for i in downloads} == \
|
||||
{'https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'}
|
||||
assert all(i.status == 'complete' for i in downloads)
|
||||
|
||||
# A channel with `match_regex` only returns matching video URLs.
|
||||
simple_channel.match_regex = '.*(2).*'
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ from functools import partial
|
|||
from itertools import filterfalse
|
||||
from operator import attrgetter
|
||||
from queue import Empty
|
||||
from typing import List, Dict, Generator
|
||||
from typing import List, Dict, Generator, Iterable
|
||||
from typing import Tuple, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
|
|
@ -23,7 +23,7 @@ from sqlalchemy.dialects.postgresql import JSONB
|
|||
from sqlalchemy.orm import Session
|
||||
|
||||
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \
|
||||
background_task
|
||||
background_task, limit_concurrent
|
||||
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
|
||||
from wrolpi.db import get_db_session, get_db_curs, optional_session
|
||||
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
|
||||
|
|
@ -91,6 +91,7 @@ class Download(ModelHelper, Base): # noqa
|
|||
location=self.location,
|
||||
next_download=self.next_download,
|
||||
status=self.status,
|
||||
sub_downloader=self.sub_downloader,
|
||||
url=self.url,
|
||||
)
|
||||
return d
|
||||
|
|
@ -126,7 +127,8 @@ class Download(ModelHelper, Base): # noqa
|
|||
if self.downloader:
|
||||
return self.manager.get_downloader_by_name(self.downloader)
|
||||
|
||||
return self.manager.get_downloader(self.url)
|
||||
downloader, _ = self.manager.get_downloader(self.url)
|
||||
return downloader
|
||||
|
||||
@property
|
||||
def domain(self):
|
||||
|
|
@ -362,6 +364,7 @@ class DownloadManager:
|
|||
download.manager = self
|
||||
|
||||
downloader: Downloader = download.get_downloader()
|
||||
logger.warning(f'{downloader=}')
|
||||
if not downloader:
|
||||
worker_logger.warning(f'Could not find downloader for {download.downloader=}')
|
||||
|
||||
|
|
@ -379,6 +382,7 @@ class DownloadManager:
|
|||
result = DownloadResult(success=False, error=str(traceback.format_exc()))
|
||||
try_again = False
|
||||
except Exception as e:
|
||||
|
||||
worker_logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
|
||||
result = DownloadResult(success=False, error=str(traceback.format_exc()))
|
||||
|
||||
|
|
@ -412,6 +416,8 @@ class DownloadManager:
|
|||
self._remove_domain(download.domain)
|
||||
# Request any new downloads be added to the queue.
|
||||
background_task(self.queue_downloads())
|
||||
# Save the config now that the Download has finished.
|
||||
background_task(save_downloads_config())
|
||||
except asyncio.CancelledError:
|
||||
worker_logger.warning('Canceled!')
|
||||
self.download_queue.task_done()
|
||||
|
|
@ -574,9 +580,11 @@ class DownloadManager:
|
|||
download.settings = {'excluded_urls': excluded_urls}
|
||||
downloads.append(download)
|
||||
|
||||
# Start downloading ASAP.
|
||||
try:
|
||||
# Start downloading ASAP.
|
||||
background_task(self.queue_downloads())
|
||||
# Save the config now that new Downloads exist.
|
||||
background_task(save_downloads_config())
|
||||
except RuntimeError:
|
||||
# Event loop isn't running. Probably testing?
|
||||
if not PYTEST:
|
||||
|
|
@ -745,6 +753,9 @@ class DownloadManager:
|
|||
self.log_debug(f'Renewed {renewed_count} recurring downloads')
|
||||
session.commit()
|
||||
|
||||
# Save the config now that some Downloads renewed.
|
||||
background_task(save_downloads_config())
|
||||
|
||||
def get_downloads(self, session: Session) -> List[Download]:
|
||||
downloads = list(session.query(Download).all())
|
||||
for download in downloads:
|
||||
|
|
@ -1017,6 +1028,7 @@ class DownloadMangerConfig(ConfigFile):
|
|||
file_name = 'download_manager.yaml'
|
||||
default_config = dict(
|
||||
skip_urls=[],
|
||||
downloads=[],
|
||||
)
|
||||
|
||||
@property
|
||||
|
|
@ -1027,6 +1039,14 @@ class DownloadMangerConfig(ConfigFile):
|
|||
def skip_urls(self, value: List[str]):
|
||||
self.update({'skip_urls': value})
|
||||
|
||||
@property
|
||||
def downloads(self) -> List[dict]:
|
||||
return self._config['downloads']
|
||||
|
||||
@downloads.setter
|
||||
def downloads(self, value: List[dict]):
|
||||
self.update({'downloads': value})
|
||||
|
||||
|
||||
DOWNLOAD_MANAGER_CONFIG: DownloadMangerConfig = DownloadMangerConfig()
|
||||
|
||||
|
|
@ -1038,6 +1058,79 @@ def set_test_download_manager_config(enabled: bool):
|
|||
DOWNLOAD_MANAGER_CONFIG = DownloadMangerConfig()
|
||||
|
||||
|
||||
@optional_session(commit=False)
|
||||
async def save_downloads_config(session: Session):
|
||||
"""Fetch all Downloads from the DB, save them to the Download Manager Config."""
|
||||
downloads: Iterable[Download] = session.query(Download).order_by(Download.url)
|
||||
config = []
|
||||
for download in downloads:
|
||||
if download.last_successful_download and not download.frequency:
|
||||
# This once-download has completed, do not save it.
|
||||
continue
|
||||
config.append(dict(
|
||||
downloader=download.downloader,
|
||||
frequency=download.frequency,
|
||||
last_successful_download=download.last_successful_download,
|
||||
location=download.location,
|
||||
next_download=download.next_download,
|
||||
status=download.status,
|
||||
sub_downloader=download.sub_downloader,
|
||||
url=download.url,
|
||||
))
|
||||
if config != DOWNLOAD_MANAGER_CONFIG.downloads:
|
||||
# Only save if there are changes.
|
||||
DOWNLOAD_MANAGER_CONFIG.downloads = config
|
||||
|
||||
|
||||
@optional_session
|
||||
@limit_concurrent(1)
|
||||
async def import_downloads_config(session: Session):
|
||||
"""Upsert all Downloads in the Download Manager Config into the DB.
|
||||
|
||||
The config is the source of truth."""
|
||||
if not DOWNLOAD_MANAGER_CONFIG.downloads:
|
||||
return
|
||||
|
||||
try:
|
||||
logger.warning('Importing downloads in config')
|
||||
|
||||
downloads_by_url = {i['url']: i for i in DOWNLOAD_MANAGER_CONFIG.downloads}
|
||||
existing_downloads = list(session.query(Download))
|
||||
for existing in existing_downloads:
|
||||
download = downloads_by_url.pop(existing.url, None)
|
||||
if download:
|
||||
# Download in config already exists, update the DB record.
|
||||
# The config is the source of truth.
|
||||
existing.downloader = download['downloader']
|
||||
existing.frequency = download['frequency']
|
||||
existing.last_successful_download = download['last_successful_download']
|
||||
existing.location = download['location']
|
||||
existing.next_download = download['next_download']
|
||||
existing.status = download['status']
|
||||
existing.sub_downloader = download['sub_downloader']
|
||||
logger.debug(f'Updating download {existing.url} with {download}')
|
||||
|
||||
for download in downloads_by_url.values():
|
||||
# These downloads are new, import them.
|
||||
download = Download(
|
||||
downloader=download['downloader'],
|
||||
frequency=download['frequency'],
|
||||
last_successful_download=download['last_successful_download'],
|
||||
location=download['location'],
|
||||
next_download=download['next_download'],
|
||||
status=download['status'],
|
||||
sub_downloader=download['sub_downloader'],
|
||||
url=(url := download['url']),
|
||||
)
|
||||
session.add(download)
|
||||
logger.info(f'Adding new download {url}')
|
||||
|
||||
session.commit()
|
||||
except Exception as e:
|
||||
logger.error('Failed to import downloads', exc_info=e)
|
||||
raise
|
||||
|
||||
|
||||
def parse_feed(url: str) -> FeedParserDict:
|
||||
"""Calls `feedparser.parse`, used for testing."""
|
||||
return feedparser.parse(url)
|
||||
|
|
@ -1079,6 +1172,9 @@ class RSSDownloader(Downloader, ABC):
|
|||
# Only download URLs that have not yet been downloaded.
|
||||
urls = []
|
||||
sub_downloader = self.manager.get_downloader_by_name(download.sub_downloader)
|
||||
if not sub_downloader:
|
||||
raise ValueError(f'Unable to find sub_downloader for {download.url}')
|
||||
|
||||
for idx, entry in enumerate(feed['entries']):
|
||||
if url := entry.get('link'):
|
||||
urls.append(url.strip())
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
import asyncio
|
||||
import json
|
||||
from abc import ABC
|
||||
from datetime import datetime
|
||||
|
|
@ -10,7 +11,7 @@ import pytest
|
|||
|
||||
from wrolpi.dates import local_timezone, Seconds
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config
|
||||
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
|
||||
from wrolpi.test.common import assert_dict_contains
|
||||
|
||||
|
|
@ -395,3 +396,88 @@ def test_crud_download(test_client, test_session, test_download_manager):
|
|||
request, response = test_client.delete(f'/api/download/{download.id}')
|
||||
assert response.status_code == HTTPStatus.NO_CONTENT
|
||||
assert test_session.query(Download).count() == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_downloads_config(test_client, test_session, test_download_manager, test_download_manager_config):
|
||||
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
|
||||
|
||||
def assert_downloads(expected_):
|
||||
assert test_session.query(Download).count() == len(expected_), 'Download count does not match'
|
||||
downloads = test_session.query(Download).order_by(Download.url)
|
||||
for download_, expected_ in zip_longest(downloads, expected_):
|
||||
assert_dict_contains(download_.__json__(), expected_)
|
||||
|
||||
# Can import with an empty config.
|
||||
await import_downloads_config()
|
||||
assert_downloads([])
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = DownloadResult(success=True)
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
download1, download2 = test_download_manager.create_downloads([
|
||||
'https://example.com/1',
|
||||
'https://example.com/2',
|
||||
], downloader=HTTPDownloader.name)
|
||||
test_download_manager.recurring_download(
|
||||
'https://example.com/3',
|
||||
frequency=DownloadFrequency.weekly,
|
||||
downloader=HTTPDownloader.name,
|
||||
)
|
||||
download2.next_download = local_timezone(datetime(2000, 1, 2, 0, 0, 0))
|
||||
# Completed once-downloads should be ignored.
|
||||
download1.last_successful_download = local_timezone(datetime(2000, 1, 1, 0, 0, 0))
|
||||
test_session.commit()
|
||||
|
||||
# Allow background tasks to run.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
# Downloads were saved to config.
|
||||
assert DOWNLOAD_MANAGER_CONFIG.downloads, 'No downloads were saved'
|
||||
expected = [
|
||||
# https://example.com/1 is missing because it is completed.
|
||||
dict(
|
||||
url='https://example.com/2',
|
||||
frequency=None,
|
||||
next_download=local_timezone(datetime(2000, 1, 2, 0, 0, 0)),
|
||||
downloader='http',
|
||||
sub_downloader=None,
|
||||
),
|
||||
dict(
|
||||
url='https://example.com/3',
|
||||
frequency=DownloadFrequency.weekly,
|
||||
downloader='http',
|
||||
sub_downloader=None,
|
||||
),
|
||||
]
|
||||
for download, expected in zip_longest(DOWNLOAD_MANAGER_CONFIG.downloads, expected):
|
||||
assert_dict_contains(download, expected)
|
||||
|
||||
# Delete a Downloads, so we can import what was saved.
|
||||
test_session.query(Download).delete()
|
||||
test_session.commit()
|
||||
|
||||
# Import the saved downloads.
|
||||
await import_downloads_config()
|
||||
expected = [
|
||||
dict(
|
||||
url='https://example.com/2',
|
||||
frequency=None,
|
||||
next_download=local_timezone(datetime(2000, 1, 2, 0, 0, 0)),
|
||||
downloader='http',
|
||||
sub_downloader=None,
|
||||
),
|
||||
dict(
|
||||
url='https://example.com/3',
|
||||
frequency=DownloadFrequency.weekly,
|
||||
downloader='http',
|
||||
sub_downloader=None,
|
||||
),
|
||||
]
|
||||
assert_downloads(expected)
|
||||
|
||||
# Import again, no duplicates.
|
||||
await import_downloads_config()
|
||||
assert_downloads(expected)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue