Downloads are saved to the download manager config. They are imported on startup.

This commit is contained in:
Roland 2022-11-04 19:59:42 -06:00
parent 241b6175c6
commit 6c2a5d723e
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 200 additions and 12 deletions

View file

@ -10,7 +10,7 @@ from sanic.signals import Event
from wrolpi import root_api, BEFORE_STARTUP_FUNCTIONS, admin
from wrolpi.common import logger, get_config, import_modules, check_media_directory, limit_concurrent, wrol_mode_enabled
from wrolpi.dates import set_timezone
from wrolpi.downloader import download_manager
from wrolpi.downloader import download_manager, import_downloads_config
from wrolpi.files.lib import cancel_refresh_tasks
from wrolpi.root_api import api_app
from wrolpi.vars import PROJECT_DIR, DOCKERIZED, PYTEST
@ -170,6 +170,11 @@ def set_log_level(args):
logger.warning(f'Logging level: {level_name}')
@api_app.before_server_start
async def download_startup(app: Sanic):
await import_downloads_config()
@api_app.after_server_start
async def periodic_downloads(app: Sanic):
"""

View file

@ -1,7 +1,7 @@
import asyncio
import pathlib
import shutil
from copy import copy
from itertools import zip_longest
from unittest import mock
import pytest
@ -196,14 +196,15 @@ async def test_download_channel(test_session, simple_channel, video_download_man
video_download_manager.create_download(url)
await video_download_manager.wait_for_all_downloads()
# Let background tasks run.
await asyncio.sleep(0)
# Two videos are in the example channel.
downloads = video_download_manager.get_once_downloads(test_session)
downloads = filter(lambda i: 'watch' in i.url, downloads)
expected = ['https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url']
for download, expected in zip_longest(downloads, expected):
assert download.url == expected
# Download is run only for test.
assert download.status == 'complete'
assert {i.url for i in downloads} == \
{'https://youtube.com/watch?v=video_2_url', 'https://youtube.com/watch?v=video_1_url'}
assert all(i.status == 'complete' for i in downloads)
# A channel with `match_regex` only returns matching video URLs.
simple_channel.match_regex = '.*(2).*'

View file

@ -12,7 +12,7 @@ from functools import partial
from itertools import filterfalse
from operator import attrgetter
from queue import Empty
from typing import List, Dict, Generator
from typing import List, Dict, Generator, Iterable
from typing import Tuple, Optional
from urllib.parse import urlparse
@ -23,7 +23,7 @@ from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
from wrolpi.common import Base, ModelHelper, logger, wrol_mode_check, zig_zag, ConfigFile, WROLPI_CONFIG, \
background_task
background_task, limit_concurrent
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone, recursive_replace_tz
from wrolpi.db import get_db_session, get_db_curs, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
@ -91,6 +91,7 @@ class Download(ModelHelper, Base): # noqa
location=self.location,
next_download=self.next_download,
status=self.status,
sub_downloader=self.sub_downloader,
url=self.url,
)
return d
@ -126,7 +127,8 @@ class Download(ModelHelper, Base): # noqa
if self.downloader:
return self.manager.get_downloader_by_name(self.downloader)
return self.manager.get_downloader(self.url)
downloader, _ = self.manager.get_downloader(self.url)
return downloader
@property
def domain(self):
@ -362,6 +364,7 @@ class DownloadManager:
download.manager = self
downloader: Downloader = download.get_downloader()
logger.warning(f'{downloader=}')
if not downloader:
worker_logger.warning(f'Could not find downloader for {download.downloader=}')
@ -379,6 +382,7 @@ class DownloadManager:
result = DownloadResult(success=False, error=str(traceback.format_exc()))
try_again = False
except Exception as e:
worker_logger.warning(f'Failed to download {url}. Will be tried again later.', exc_info=e)
result = DownloadResult(success=False, error=str(traceback.format_exc()))
@ -412,6 +416,8 @@ class DownloadManager:
self._remove_domain(download.domain)
# Request any new downloads be added to the queue.
background_task(self.queue_downloads())
# Save the config now that the Download has finished.
background_task(save_downloads_config())
except asyncio.CancelledError:
worker_logger.warning('Canceled!')
self.download_queue.task_done()
@ -574,9 +580,11 @@ class DownloadManager:
download.settings = {'excluded_urls': excluded_urls}
downloads.append(download)
# Start downloading ASAP.
try:
# Start downloading ASAP.
background_task(self.queue_downloads())
# Save the config now that new Downloads exist.
background_task(save_downloads_config())
except RuntimeError:
# Event loop isn't running. Probably testing?
if not PYTEST:
@ -745,6 +753,9 @@ class DownloadManager:
self.log_debug(f'Renewed {renewed_count} recurring downloads')
session.commit()
# Save the config now that some Downloads renewed.
background_task(save_downloads_config())
def get_downloads(self, session: Session) -> List[Download]:
downloads = list(session.query(Download).all())
for download in downloads:
@ -1017,6 +1028,7 @@ class DownloadMangerConfig(ConfigFile):
file_name = 'download_manager.yaml'
default_config = dict(
skip_urls=[],
downloads=[],
)
@property
@ -1027,6 +1039,14 @@ class DownloadMangerConfig(ConfigFile):
def skip_urls(self, value: List[str]):
self.update({'skip_urls': value})
@property
def downloads(self) -> List[dict]:
return self._config['downloads']
@downloads.setter
def downloads(self, value: List[dict]):
self.update({'downloads': value})
DOWNLOAD_MANAGER_CONFIG: DownloadMangerConfig = DownloadMangerConfig()
@ -1038,6 +1058,79 @@ def set_test_download_manager_config(enabled: bool):
DOWNLOAD_MANAGER_CONFIG = DownloadMangerConfig()
@optional_session(commit=False)
async def save_downloads_config(session: Session):
"""Fetch all Downloads from the DB, save them to the Download Manager Config."""
downloads: Iterable[Download] = session.query(Download).order_by(Download.url)
config = []
for download in downloads:
if download.last_successful_download and not download.frequency:
# This once-download has completed, do not save it.
continue
config.append(dict(
downloader=download.downloader,
frequency=download.frequency,
last_successful_download=download.last_successful_download,
location=download.location,
next_download=download.next_download,
status=download.status,
sub_downloader=download.sub_downloader,
url=download.url,
))
if config != DOWNLOAD_MANAGER_CONFIG.downloads:
# Only save if there are changes.
DOWNLOAD_MANAGER_CONFIG.downloads = config
@optional_session
@limit_concurrent(1)
async def import_downloads_config(session: Session):
"""Upsert all Downloads in the Download Manager Config into the DB.
The config is the source of truth."""
if not DOWNLOAD_MANAGER_CONFIG.downloads:
return
try:
logger.warning('Importing downloads in config')
downloads_by_url = {i['url']: i for i in DOWNLOAD_MANAGER_CONFIG.downloads}
existing_downloads = list(session.query(Download))
for existing in existing_downloads:
download = downloads_by_url.pop(existing.url, None)
if download:
# Download in config already exists, update the DB record.
# The config is the source of truth.
existing.downloader = download['downloader']
existing.frequency = download['frequency']
existing.last_successful_download = download['last_successful_download']
existing.location = download['location']
existing.next_download = download['next_download']
existing.status = download['status']
existing.sub_downloader = download['sub_downloader']
logger.debug(f'Updating download {existing.url} with {download}')
for download in downloads_by_url.values():
# These downloads are new, import them.
download = Download(
downloader=download['downloader'],
frequency=download['frequency'],
last_successful_download=download['last_successful_download'],
location=download['location'],
next_download=download['next_download'],
status=download['status'],
sub_downloader=download['sub_downloader'],
url=(url := download['url']),
)
session.add(download)
logger.info(f'Adding new download {url}')
session.commit()
except Exception as e:
logger.error('Failed to import downloads', exc_info=e)
raise
def parse_feed(url: str) -> FeedParserDict:
"""Calls `feedparser.parse`, used for testing."""
return feedparser.parse(url)
@ -1079,6 +1172,9 @@ class RSSDownloader(Downloader, ABC):
# Only download URLs that have not yet been downloaded.
urls = []
sub_downloader = self.manager.get_downloader_by_name(download.sub_downloader)
if not sub_downloader:
raise ValueError(f'Unable to find sub_downloader for {download.url}')
for idx, entry in enumerate(feed['entries']):
if url := entry.get('link'):
urls.append(url.strip())

View file

@ -1,3 +1,4 @@
import asyncio
import json
from abc import ABC
from datetime import datetime
@ -10,7 +11,7 @@ import pytest
from wrolpi.dates import local_timezone, Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult
from wrolpi.downloader import Downloader, Download, DownloadFrequency, DownloadResult, import_downloads_config
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
from wrolpi.test.common import assert_dict_contains
@ -395,3 +396,88 @@ def test_crud_download(test_client, test_session, test_download_manager):
request, response = test_client.delete(f'/api/download/{download.id}')
assert response.status_code == HTTPStatus.NO_CONTENT
assert test_session.query(Download).count() == 0
@pytest.mark.asyncio
async def test_downloads_config(test_client, test_session, test_download_manager, test_download_manager_config):
from wrolpi.downloader import DOWNLOAD_MANAGER_CONFIG
def assert_downloads(expected_):
assert test_session.query(Download).count() == len(expected_), 'Download count does not match'
downloads = test_session.query(Download).order_by(Download.url)
for download_, expected_ in zip_longest(downloads, expected_):
assert_dict_contains(download_.__json__(), expected_)
# Can import with an empty config.
await import_downloads_config()
assert_downloads([])
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = DownloadResult(success=True)
test_download_manager.register_downloader(http_downloader)
download1, download2 = test_download_manager.create_downloads([
'https://example.com/1',
'https://example.com/2',
], downloader=HTTPDownloader.name)
test_download_manager.recurring_download(
'https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader=HTTPDownloader.name,
)
download2.next_download = local_timezone(datetime(2000, 1, 2, 0, 0, 0))
# Completed once-downloads should be ignored.
download1.last_successful_download = local_timezone(datetime(2000, 1, 1, 0, 0, 0))
test_session.commit()
# Allow background tasks to run.
await asyncio.sleep(0)
# Downloads were saved to config.
assert DOWNLOAD_MANAGER_CONFIG.downloads, 'No downloads were saved'
expected = [
# https://example.com/1 is missing because it is completed.
dict(
url='https://example.com/2',
frequency=None,
next_download=local_timezone(datetime(2000, 1, 2, 0, 0, 0)),
downloader='http',
sub_downloader=None,
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='http',
sub_downloader=None,
),
]
for download, expected in zip_longest(DOWNLOAD_MANAGER_CONFIG.downloads, expected):
assert_dict_contains(download, expected)
# Delete a Downloads, so we can import what was saved.
test_session.query(Download).delete()
test_session.commit()
# Import the saved downloads.
await import_downloads_config()
expected = [
dict(
url='https://example.com/2',
frequency=None,
next_download=local_timezone(datetime(2000, 1, 2, 0, 0, 0)),
downloader='http',
sub_downloader=None,
),
dict(
url='https://example.com/3',
frequency=DownloadFrequency.weekly,
downloader='http',
sub_downloader=None,
),
]
assert_downloads(expected)
# Import again, no duplicates.
await import_downloads_config()
assert_downloads(expected)