From 6c2a5d723eee18c004784240f635e0d3bf14345d Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 4 Nov 2022 19:59:42 -0600 Subject: [PATCH] Downloads are saved to the download manager config. They are imported on startup. --- main.py | 7 +- modules/videos/test/test_downloader.py | 13 ++-- wrolpi/downloader.py | 104 ++++++++++++++++++++++++- wrolpi/test/test_downloader.py | 88 ++++++++++++++++++++- 4 files changed, 200 insertions(+), 12 deletions(-) diff --git a/main.py b/main.py index 585becf0..16df9a23 100755 --- a/main.py +++ b/main.py @@ -10,7 +10,7 @@ from sanic.signals import Event from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, wrol_mode_enabled from wrolpi.dates import set_timezone -from wrolpi.downloader import download_manager +from wrolpi.downloader import download_manager, import_downloads_config from wrolpi.files.lib import cancel_refresh_tasks from wrolpi.root_api import api_app from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST @@ -170,6 +170,11 @@ def set_log_level(args): logger.warning(f'Logging level: {level_name}') +@api_app.before_server_start +async def download_startup(app: Sanic): + await import_downloads_config() + + @api_app.after_server_start async def periodic_downloads(app: Sanic): """ diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py index e8bf6b8c..9dd98fe8 100644 --- a/modules/videos/test/test_downloader.py +++ b/modules/videos/test/test_downloader.py @@ -1,7 +1,7 @@ +import asyncio import pathlib import shutil from copy import copy -from itertools import zip_longest from unittest import mock import pytest @@ -196,14 +196,15 @@ async def test_download_channel(test_session, simple_channel, video_download_man video_download_manager.create_download(url) await video_download_manager.wait_for_all_downloads() + # Let background tasks run. + await asyncio.sleep(0) + # Two videos are in the example channel. downloads = video_download_manager.get_once_downloads(test_session) downloads = filter(lambda i: 'watch' in i.url, downloads) - expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'] - for download, expected in zip_longest(downloads, expected): - assert download.url == expected - # Download is run only for test. - assert download.status == 'complete' + assert {i.url for i in downloads} == \ + {'https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'} + assert all(i.status == 'complete' for i in downloads) # A channel with `match_regex` only returns matching video URLs. simple_channel.match_regex = '.*(2).*' diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 4885b4f2..2ecd8343 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -12,7 +12,7 @@ from functools import partial from itertools import filterfalse from operator import attrgetter from queue import Empty -from typing import List, Dict, Generator +from typing import List, Dict, Generator, Iterable from typing import Tuple, Optional from urllib.parse import urlparse @@ -23,7 +23,7 @@ from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \ - background_task + background_task, limit_concurrent from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz from wrolpi.db import get_db_session, get_db_curs, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError @@ -91,6 +91,7 @@ class Download(ModelHelper, Base): # noqa location=self.location, next_download=self.next_download, status=self.status, + sub_downloader=self.sub_downloader, url=self.url, ) return d @@ -126,7 +127,8 @@ class Download(ModelHelper, Base): # noqa if self.downloader: return self.manager.get_downloader_by_name(self.downloader) - return self.manager.get_downloader(self.url) + downloader, _ = self.manager.get_downloader(self.url) + return downloader @property def domain(self): @@ -362,6 +364,7 @@ class DownloadManager: download.manager = self downloader: Downloader = download.get_downloader() + logger.warning(f'{downloader=}') if not downloader: worker_logger.warning(f'Could not find downloader for {download.downloader=}') @@ -379,6 +382,7 @@ class DownloadManager: result = DownloadResult(success=False, error=str(traceback.format_exc())) try_again = False except Exception as e: + worker_logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e) result = DownloadResult(success=False, error=str(traceback.format_exc())) @@ -412,6 +416,8 @@ class DownloadManager: self._remove_domain(download.domain) # Request any new downloads be added to the queue. background_task(self.queue_downloads()) + # Save the config now that the Download has finished. + background_task(save_downloads_config()) except asyncio.CancelledError: worker_logger.warning('Canceled!') self.download_queue.task_done() @@ -574,9 +580,11 @@ class DownloadManager: download.settings = {'excluded_urls': excluded_urls} downloads.append(download) - # Start downloading ASAP. try: + # Start downloading ASAP. background_task(self.queue_downloads()) + # Save the config now that new Downloads exist. + background_task(save_downloads_config()) except RuntimeError: # Event loop isn't running. Probably testing? if not PYTEST: @@ -745,6 +753,9 @@ class DownloadManager: self.log_debug(f'Renewed {renewed_count} recurring downloads') session.commit() + # Save the config now that some Downloads renewed. + background_task(save_downloads_config()) + def get_downloads(self, session: Session) -> List[Download]: downloads = list(session.query(Download).all()) for download in downloads: @@ -1017,6 +1028,7 @@ class DownloadMangerConfig(ConfigFile): file_name = 'download_manager.yaml' default_config = dict( skip_urls=[], + downloads=[], ) @property @@ -1027,6 +1039,14 @@ class DownloadMangerConfig(ConfigFile): def skip_urls(self, value: List[str]): self.update({'skip_urls': value}) + @property + def downloads(self) -> List[dict]: + return self._config['downloads'] + + @downloads.setter + def downloads(self, value: List[dict]): + self.update({'downloads': value}) + DOWNLOAD_MANAGER_CONFIG: DownloadMangerConfig = DownloadMangerConfig() @@ -1038,6 +1058,79 @@ def set_test_download_manager_config(enabled: bool): DOWNLOAD_MANAGER_CONFIG = DownloadMangerConfig() +@optional_session(commit=False) +async def save_downloads_config(session: Session): + """Fetch all Downloads from the DB, save them to the Download Manager Config.""" + downloads: Iterable[Download] = session.query(Download).order_by(Download.url) + config = [] + for download in downloads: + if download.last_successful_download and not download.frequency: + # This once-download has completed, do not save it. + continue + config.append(dict( + downloader=download.downloader, + frequency=download.frequency, + last_successful_download=download.last_successful_download, + location=download.location, + next_download=download.next_download, + status=download.status, + sub_downloader=download.sub_downloader, + url=download.url, + )) + if config != DOWNLOAD_MANAGER_CONFIG.downloads: + # Only save if there are changes. + DOWNLOAD_MANAGER_CONFIG.downloads = config + + +@optional_session +@limit_concurrent(1) +async def import_downloads_config(session: Session): + """Upsert all Downloads in the Download Manager Config into the DB. + + The config is the source of truth.""" + if not DOWNLOAD_MANAGER_CONFIG.downloads: + return + + try: + logger.warning('Importing downloads in config') + + downloads_by_url = {i['url']: i for i in DOWNLOAD_MANAGER_CONFIG.downloads} + existing_downloads = list(session.query(Download)) + for existing in existing_downloads: + download = downloads_by_url.pop(existing.url, None) + if download: + # Download in config already exists, update the DB record. + # The config is the source of truth. + existing.downloader = download['downloader'] + existing.frequency = download['frequency'] + existing.last_successful_download = download['last_successful_download'] + existing.location = download['location'] + existing.next_download = download['next_download'] + existing.status = download['status'] + existing.sub_downloader = download['sub_downloader'] + logger.debug(f'Updating download {existing.url} with {download}') + + for download in downloads_by_url.values(): + # These downloads are new, import them. + download = Download( + downloader=download['downloader'], + frequency=download['frequency'], + last_successful_download=download['last_successful_download'], + location=download['location'], + next_download=download['next_download'], + status=download['status'], + sub_downloader=download['sub_downloader'], + url=(url := download['url']), + ) + session.add(download) + logger.info(f'Adding new download {url}') + + session.commit() + except Exception as e: + logger.error('Failed to import downloads', exc_info=e) + raise + + def parse_feed(url: str) -> FeedParserDict: """Calls `feedparser.parse`, used for testing.""" return feedparser.parse(url) @@ -1079,6 +1172,9 @@ class RSSDownloader(Downloader, ABC): # Only download URLs that have not yet been downloaded. urls = [] sub_downloader = self.manager.get_downloader_by_name(download.sub_downloader) + if not sub_downloader: + raise ValueError(f'Unable to find sub_downloader for {download.url}') + for idx, entry in enumerate(feed['entries']): if url := entry.get('link'): urls.append(url.strip()) diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index 1bec1d81..8e4e5aac 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1,3 +1,4 @@ +import asyncio import json from abc import ABC from datetime import datetime @@ -10,7 +11,7 @@ import pytest from wrolpi.dates import local_timezone, Seconds from wrolpi.db import get_db_context -from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult +from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled from wrolpi.test.common import assert_dict_contains @@ -395,3 +396,88 @@ def test_crud_download(test_client, test_session, test_download_manager): request, response = test_client.delete(f'/api/download/{download.id}') assert response.status_code == HTTPStatus.NO_CONTENT assert test_session.query(Download).count() == 0 + + +@pytest.mark.asyncio +async def test_downloads_config(test_client, test_session, test_download_manager, test_download_manager_config): + from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG + + def assert_downloads(expected_): + assert test_session.query(Download).count() == len(expected_), 'Download count does not match' + downloads = test_session.query(Download).order_by(Download.url) + for download_, expected_ in zip_longest(downloads, expected_): + assert_dict_contains(download_.__json__(), expected_) + + # Can import with an empty config. + await import_downloads_config() + assert_downloads([]) + + http_downloader = HTTPDownloader() + http_downloader.do_download = MagicMock() + http_downloader.do_download.return_value = DownloadResult(success=True) + test_download_manager.register_downloader(http_downloader) + + download1, download2 = test_download_manager.create_downloads([ + 'https://example.com/1', + 'https://example.com/2', + ], downloader=HTTPDownloader.name) + test_download_manager.recurring_download( + 'https://example.com/3', + frequency=DownloadFrequency.weekly, + downloader=HTTPDownloader.name, + ) + download2.next_download = local_timezone(datetime(2000, 1, 2, 0, 0, 0)) + # Completed once-downloads should be ignored. + download1.last_successful_download = local_timezone(datetime(2000, 1, 1, 0, 0, 0)) + test_session.commit() + + # Allow background tasks to run. + await asyncio.sleep(0) + + # Downloads were saved to config. + assert DOWNLOAD_MANAGER_CONFIG.downloads, 'No downloads were saved' + expected = [ + # https://example.com/1 is missing because it is completed. + dict( + url='https://example.com/2', + frequency=None, + next_download=local_timezone(datetime(2000, 1, 2, 0, 0, 0)), + downloader='http', + sub_downloader=None, + ), + dict( + url='https://example.com/3', + frequency=DownloadFrequency.weekly, + downloader='http', + sub_downloader=None, + ), + ] + for download, expected in zip_longest(DOWNLOAD_MANAGER_CONFIG.downloads, expected): + assert_dict_contains(download, expected) + + # Delete a Downloads, so we can import what was saved. + test_session.query(Download).delete() + test_session.commit() + + # Import the saved downloads. + await import_downloads_config() + expected = [ + dict( + url='https://example.com/2', + frequency=None, + next_download=local_timezone(datetime(2000, 1, 2, 0, 0, 0)), + downloader='http', + sub_downloader=None, + ), + dict( + url='https://example.com/3', + frequency=DownloadFrequency.weekly, + downloader='http', + sub_downloader=None, + ), + ] + assert_downloads(expected) + + # Import again, no duplicates. + await import_downloads_config() + assert_downloads(expected)