From 3dda813b9e010faceb0bf59ee52860caae93a535 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 26 Jan 2022 17:10:18 -0700 Subject: [PATCH] Download.next_download is assigned by the DownloadManager. --- modules/archive/test/test_lib.py | 128 ++++++++++++++++--------------- wrolpi/downloader.py | 39 ++++++++-- wrolpi/test/test_downloader.py | 26 ++++--- 3 files changed, 113 insertions(+), 80 deletions(-) diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 39fe6da6..e868be84 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -51,35 +51,6 @@ class TestArchive(TestAPI): assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json') assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png') - @wrap_test_db - def test_new_archive(self): - with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - archive1 = do_archive('https://example.com') - # Everything is filled out. - self.assertIsInstance(archive1, Archive) - self.assertIsNotNone(archive1.archive_datetime) - self.assertIsInstance(archive1.singlefile_path, MediaPath) - self.assertIsInstance(archive1.readability_path, MediaPath) - self.assertIsInstance(archive1.readability_txt_path, MediaPath) - self.assertIsInstance(archive1.screenshot_path, MediaPath) - self.assertEqual(archive1.title, 'ジにてこちら') - self.assertIsNotNone(archive1.url) - self.assertIsNotNone(archive1.domain) - - # The actual files were dumped and read correctly. - with open(archive1.singlefile_path.path) as fh: - self.assertEqual(fh.read(), '\ntest single-file\nジにてこちら\nsome title') - with open(archive1.readability_path.path) as fh: - self.assertEqual(fh.read(), 'test readability content') - with open(archive1.readability_txt_path.path) as fh: - self.assertEqual(fh.read(), 'test readability textContent') - with open(archive1.readability_json_path.path) as fh: - self.assertEqual(json.load(fh), {'title': 'ジにてこちら', 'url': 'https://example.com'}) - - archive2 = do_archive('https://example.com') - # Domain is reused. - self.assertEqual(archive1.domain, archive2.domain) - @wrap_test_db def test_no_screenshot(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)): @@ -132,39 +103,6 @@ class TestArchive(TestAPI): except ValueError as e: self.assertIn('relative', str(e), f'Relative path error was not raised') - @wrap_test_db - def test_get_title_from_html(self): - with wrap_media_directory(): - with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): - archive = do_archive('example.com') - self.assertEqual(archive.title, 'ジにてこちら') - - def fake_request_archive(_): - singlefile = '\ntest single-file\nジにてこちら\nsome title' - r = dict( - content=f'test readability content', - textContent='test readability textContent', - ) - s = b'screenshot data' - return singlefile, r, s - - with mock.patch('modules.archive.lib.request_archive', fake_request_archive): - archive = do_archive('example.com') - self.assertEqual(archive.title, 'some title') - - def fake_request_archive(_): - singlefile = '' - r = dict( - content=f'missing a title', - textContent='', - ) - s = b'screenshot data' - return singlefile, r, s - - with mock.patch('modules.archive.lib.request_archive', fake_request_archive): - archive = do_archive('example.com') - self.assertIsNone(archive.title) - @wrap_test_db def test_refresh_archives(self): with wrap_media_directory(): @@ -324,3 +262,69 @@ def test_get_domains(test_session, archive_factory): archive3.delete() test_session.commit() assert [i['domain'] for i in get_domains()] == [] + + +def test_new_archive(test_session, fake_now): + with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): + fake_now(datetime(2000, 1, 1)) + archive1 = do_archive('https://example.com') + # Everything is filled out. + assert isinstance(archive1, Archive) + assert archive1.archive_datetime is not None + assert isinstance(archive1.singlefile_path, MediaPath) + assert isinstance(archive1.readability_path, MediaPath) + assert isinstance(archive1.readability_txt_path, MediaPath) + assert isinstance(archive1.screenshot_path, MediaPath) + assert archive1.title == 'ジにてこちら' + assert archive1.url is not None + assert archive1.domain is not None + + # The actual files were dumped and read correctly. + with open(archive1.singlefile_path.path) as fh: + assert fh.read() == '\ntest single-file\nジにてこちら\nsome title' + with open(archive1.readability_path.path) as fh: + assert fh.read() == 'test readability content' + with open(archive1.readability_txt_path.path) as fh: + assert fh.read() == 'test readability textContent' + with open(archive1.readability_json_path.path) as fh: + assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'} + + fake_now(datetime(2000, 1, 2)) + archive2 = do_archive('https://example.com') + # Domain is reused. + assert archive1.domain == archive2.domain + + +def test_get_title_from_html(test_session, fake_now): + fake_now(datetime(2000, 1, 1)) + with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): + archive = do_archive('example.com') + assert archive.title == 'ジにてこちら' + + def fake_request_archive(_): + singlefile = '\ntest single-file\nジにてこちら\nsome title' + r = dict( + content=f'test readability content', + textContent='test readability textContent', + ) + s = b'screenshot data' + return singlefile, r, s + + fake_now(datetime(2000, 1, 2)) + with mock.patch('modules.archive.lib.request_archive', fake_request_archive): + archive = do_archive('example.com') + assert archive.title == 'some title' + + def fake_request_archive(_): + singlefile = '' + r = dict( + content=f'missing a title', + textContent='', + ) + s = b'screenshot data' + return singlefile, r, s + + fake_now(datetime(2000, 1, 3)) + with mock.patch('modules.archive.lib.request_archive', fake_request_archive): + archive = do_archive('example.com') + assert archive.title is None diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 8dc60bcd..648a610c 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -13,7 +13,7 @@ from sqlalchemy import Column, Integer, String, Text from sqlalchemy.dialects.postgresql import JSONB from sqlalchemy.orm import Session -from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check +from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag from wrolpi.dates import TZDateTime, now, Seconds, local_timezone from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError @@ -374,8 +374,10 @@ class DownloadManager: download.fail() elif success: download.complete() + download.next_download = self.get_next_download(download, session) else: download.defer() + download.next_download = self.get_next_download(download, session) session_.commit() if download_count: @@ -598,23 +600,44 @@ class DownloadManager: return session.query(Download).filter_by(status='pending').all() @optional_session - def get_next_download(self, download: Download, session: Session): + def get_next_download(self, download: Download, session: Session) -> Optional[datetime]: + """ + If the download is "deferred", download soon. But, slowly increase the time between attempts. + + If the download is "complete" and the download has a frequency, schedule the download in it's next iteration. + (Next week, month, etc.) + """ if download.status == 'deferred': # Increase next_download slowly at first, then by large gaps later. The largest gap is the download # frequency. hours = 3 ** (download.attempts or 1) seconds = hours * Seconds.hour - seconds = min(seconds, download.frequency) + if download.frequency: + seconds = min(seconds, download.frequency) delta = timedelta(seconds=seconds) return now() + delta + if not download.frequency: + return None + if not download.id: + raise ValueError('Cannot get next download when Download is not in DB.') + + freq = download.frequency + + # Keep this Download in it's position within the like-frequency downloads. + downloads = [i.id for i in session.query(Download).filter_by(frequency=freq).order_by(Download.id)] + index = downloads.index(download.id) + # Download was successful. Spread the same-frequency downloads out. start_date = local_timezone(datetime(2000, 1, 1)) - elapsed = now() - start_date - chunk = elapsed // download.frequency - this_frequency = start_date + (download.frequency * chunk) - next_frequency = this_frequency + timedelta(seconds=download.frequency) - print(f'{this_frequency=} {next_frequency=}') + # Weeks/months/etc since start_date. + iterations = ((now() - start_date) // freq).total_seconds() + start_date = start_date + timedelta(seconds=iterations * freq) + end_date = start_date + timedelta(seconds=freq) + zagger = zig_zag(start_date, end_date) + next_download = [i for i, j in zip(zagger, range(len(downloads)))][index] + next_download = local_timezone(next_download) + return next_download # The global DownloadManager. This should be used everywhere! diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py index d34d501b..57d4f7c7 100644 --- a/wrolpi/test/test_downloader.py +++ b/wrolpi/test/test_downloader.py @@ -1,5 +1,5 @@ from abc import ABC -from datetime import datetime, timedelta +from datetime import datetime from itertools import zip_longest from unittest import mock from unittest.mock import MagicMock @@ -185,7 +185,7 @@ class TestDownloader(TestAPI): downloads = list(self.mgr.get_recurring_downloads(session)) self.assertEqual(len(downloads), 1) download = downloads[0] - expected = local_timezone(datetime(2020, 1, 1, 1, 0, 0)) + expected = local_timezone(datetime(2020, 1, 1, 0, 0, 0)) self.assertEqual(download.next_download, expected) self.assertEqual(download.last_successful_download, now) @@ -195,7 +195,7 @@ class TestDownloader(TestAPI): self.assertEqual(download.last_successful_download, now) # Download is due an hour later. - mock_now.return_value = local_timezone(datetime(2020, 1, 1, 1, 0, 1)) + mock_now.return_value = local_timezone(datetime(2020, 1, 1, 2, 0, 1)) self.mgr.renew_recurring_downloads(session) downloads = list(self.mgr.get_new_downloads(session)) self.assertEqual(len(downloads), 1) @@ -214,11 +214,12 @@ class TestDownloader(TestAPI): self.assertEqual(download.status, 'deferred') self.assertEqual(download.last_successful_download, now) # Download should be retried after the DEFAULT_RETRY_FREQUENCY. - self.assertEqual(download.next_download, expected + timedelta(hours=1, seconds=1)) + expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1)) + self.assertEqual(download.next_download, expected) # Try the download again, it finally succeeds. http_downloader.do_download.reset_mock() - now = local_timezone(datetime(2020, 1, 1, 3, 0, 0)) + now = local_timezone(datetime(2020, 1, 1, 4, 0, 1)) mock_now.return_value = now http_downloader.do_download.return_value = True self.mgr.renew_recurring_downloads(session) @@ -227,7 +228,8 @@ class TestDownloader(TestAPI): download = session.query(Download).one() self.assertEqual(download.status, 'complete') self.assertEqual(download.last_successful_download, now) - self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0))) + # Floats cause slightly wrong date. + self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0, 997200))) @wrap_test_db @mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0))) @@ -356,7 +358,11 @@ def test_get_next_download(test_session, test_download_manager, fake_now): result = test_download_manager.get_next_download(download) assert result == expected, f'{attempts} != {result}' - download = Download() - download.frequency = DownloadFrequency.weekly - assert test_download_manager.get_next_download(download) == local_timezone(datetime(2000, 1, 1)) - + d1 = Download(url='https://example.com/1', frequency=DownloadFrequency.weekly) + d2 = Download(url='https://example.com/2', frequency=DownloadFrequency.weekly) + d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly) + test_session.add_all([d1, d2, d3]) + test_session.commit() + assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 1)) + assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 4, 12)) + assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 2, 18))