rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 19:01:39 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Download.next_download is assigned by the DownloadManager.

Browse source
This commit is contained in:
Roland 2022-01-26 17:10:18 -07:00
parent 418613e83e
commit 3dda813b9e
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
3 changed files with 113 additions and 80 deletions
Download patch file Download diff file Expand all files Collapse all files

128
modules/archive/test/test_lib.py
View file

@ -51,35 +51,6 @@ class TestArchive(TestAPI):
assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
@wrap_test_db
def test_new_archive(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive1 = do_archive('https://example.com')
# Everything is filled out.
self.assertIsInstance(archive1, Archive)
self.assertIsNotNone(archive1.archive_datetime)
self.assertIsInstance(archive1.singlefile_path, MediaPath)
self.assertIsInstance(archive1.readability_path, MediaPath)
self.assertIsInstance(archive1.readability_txt_path, MediaPath)
self.assertIsInstance(archive1.screenshot_path, MediaPath)
self.assertEqual(archive1.title, 'ジにてこちら')
self.assertIsNotNone(archive1.url)
self.assertIsNotNone(archive1.domain)
# The actual files were dumped and read correctly.
with open(archive1.singlefile_path.path) as fh:
self.assertEqual(fh.read(), '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>')
with open(archive1.readability_path.path) as fh:
self.assertEqual(fh.read(), '<html>test readability content</html>')
with open(archive1.readability_txt_path.path) as fh:
self.assertEqual(fh.read(), '<html>test readability textContent</html>')
with open(archive1.readability_json_path.path) as fh:
self.assertEqual(json.load(fh), {'title': 'ジにてこちら', 'url': 'https://example.com'})
archive2 = do_archive('https://example.com')
# Domain is reused.
self.assertEqual(archive1.domain, archive2.domain)
@wrap_test_db
def test_no_screenshot(self):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
@ -132,39 +103,6 @@ class TestArchive(TestAPI):
except ValueError as e:
self.assertIn('relative', str(e), f'Relative path error was not raised')
@wrap_test_db
def test_get_title_from_html(self):
with wrap_media_directory():
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive = do_archive('example.com')
self.assertEqual(archive.title, 'ジにてこちら')
def fake_request_archive(_):
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
r = dict(
content=f'<html>test readability content</html>',
textContent='<html>test readability textContent</html>',
)
s = b'screenshot data'
return singlefile, r, s
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = do_archive('example.com')
self.assertEqual(archive.title, 'some title')
def fake_request_archive(_):
singlefile = '<html></html>'
r = dict(
content=f'<html>missing a title</html>',
textContent='',
)
s = b'screenshot data'
return singlefile, r, s
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = do_archive('example.com')
self.assertIsNone(archive.title)
@wrap_test_db
def test_refresh_archives(self):
with wrap_media_directory():
@ -324,3 +262,69 @@ def test_get_domains(test_session, archive_factory):
archive3.delete()
test_session.commit()
assert [i['domain'] for i in get_domains()] == []
def test_new_archive(test_session, fake_now):
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
fake_now(datetime(2000, 1, 1))
archive1 = do_archive('https://example.com')
# Everything is filled out.
assert isinstance(archive1, Archive)
assert archive1.archive_datetime is not None
assert isinstance(archive1.singlefile_path, MediaPath)
assert isinstance(archive1.readability_path, MediaPath)
assert isinstance(archive1.readability_txt_path, MediaPath)
assert isinstance(archive1.screenshot_path, MediaPath)
assert archive1.title == 'ジにてこちら'
assert archive1.url is not None
assert archive1.domain is not None
# The actual files were dumped and read correctly.
with open(archive1.singlefile_path.path) as fh:
assert fh.read() == '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
with open(archive1.readability_path.path) as fh:
assert fh.read() == '<html>test readability content</html>'
with open(archive1.readability_txt_path.path) as fh:
assert fh.read() == '<html>test readability textContent</html>'
with open(archive1.readability_json_path.path) as fh:
assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
fake_now(datetime(2000, 1, 2))
archive2 = do_archive('https://example.com')
# Domain is reused.
assert archive1.domain == archive2.domain
def test_get_title_from_html(test_session, fake_now):
fake_now(datetime(2000, 1, 1))
with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
archive = do_archive('example.com')
assert archive.title == 'ジにてこちら'
def fake_request_archive(_):
singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
r = dict(
content=f'<html>test readability content</html>',
textContent='<html>test readability textContent</html>',
)
s = b'screenshot data'
return singlefile, r, s
fake_now(datetime(2000, 1, 2))
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = do_archive('example.com')
assert archive.title == 'some title'
def fake_request_archive(_):
singlefile = '<html></html>'
r = dict(
content=f'<html>missing a title</html>',
textContent='',
)
s = b'screenshot data'
return singlefile, r, s
fake_now(datetime(2000, 1, 3))
with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
archive = do_archive('example.com')
assert archive.title is None

39
wrolpi/downloader.py
View file

@ -13,7 +13,7 @@ from sqlalchemy import Column, Integer, String, Text
from sqlalchemy.dialects.postgresql import JSONB
from sqlalchemy.orm import Session
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check
from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag
from wrolpi.dates import TZDateTime, now, Seconds, local_timezone
from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
@ -374,8 +374,10 @@ class DownloadManager:
download.fail()
elif success:
download.complete()
download.next_download = self.get_next_download(download, session)
else:
download.defer()
download.next_download = self.get_next_download(download, session)
session_.commit()
if download_count:
@ -598,23 +600,44 @@ class DownloadManager:
return session.query(Download).filter_by(status='pending').all()
@optional_session
def get_next_download(self, download: Download, session: Session):
def get_next_download(self, download: Download, session: Session) -> Optional[datetime]:
"""
If the download is "deferred", download soon. But, slowly increase the time between attempts.
If the download is "complete" and the download has a frequency, schedule the download in it's next iteration.
(Next week, month, etc.)
"""
if download.status == 'deferred':
# Increase next_download slowly at first, then by large gaps later. The largest gap is the download
# frequency.
hours = 3 ** (download.attempts or 1)
seconds = hours * Seconds.hour
seconds = min(seconds, download.frequency)
if download.frequency:
seconds = min(seconds, download.frequency)
delta = timedelta(seconds=seconds)
return now() + delta
if not download.frequency:
return None
if not download.id:
raise ValueError('Cannot get next download when Download is not in DB.')
freq = download.frequency
# Keep this Download in it's position within the like-frequency downloads.
downloads = [i.id for i in session.query(Download).filter_by(frequency=freq).order_by(Download.id)]
index = downloads.index(download.id)
# Download was successful. Spread the same-frequency downloads out.
start_date = local_timezone(datetime(2000, 1, 1))
elapsed = now() - start_date
chunk = elapsed // download.frequency
this_frequency = start_date + (download.frequency * chunk)
next_frequency = this_frequency + timedelta(seconds=download.frequency)
print(f'{this_frequency=} {next_frequency=}')
# Weeks/months/etc since start_date.
iterations = ((now() - start_date) // freq).total_seconds()
start_date = start_date + timedelta(seconds=iterations * freq)
end_date = start_date + timedelta(seconds=freq)
zagger = zig_zag(start_date, end_date)
next_download = [i for i, j in zip(zagger, range(len(downloads)))][index]
next_download = local_timezone(next_download)
return next_download
# The global DownloadManager. This should be used everywhere!

26
wrolpi/test/test_downloader.py
View file

@ -1,5 +1,5 @@
from abc import ABC
from datetime import datetime, timedelta
from datetime import datetime
from itertools import zip_longest
from unittest import mock
from unittest.mock import MagicMock
@ -185,7 +185,7 @@ class TestDownloader(TestAPI):
downloads = list(self.mgr.get_recurring_downloads(session))
self.assertEqual(len(downloads), 1)
download = downloads[0]
expected = local_timezone(datetime(2020, 1, 1, 1, 0, 0))
expected = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
self.assertEqual(download.next_download, expected)
self.assertEqual(download.last_successful_download, now)
@ -195,7 +195,7 @@ class TestDownloader(TestAPI):
self.assertEqual(download.last_successful_download, now)
# Download is due an hour later.
mock_now.return_value = local_timezone(datetime(2020, 1, 1, 1, 0, 1))
mock_now.return_value = local_timezone(datetime(2020, 1, 1, 2, 0, 1))
self.mgr.renew_recurring_downloads(session)
downloads = list(self.mgr.get_new_downloads(session))
self.assertEqual(len(downloads), 1)
@ -214,11 +214,12 @@ class TestDownloader(TestAPI):
self.assertEqual(download.status, 'deferred')
self.assertEqual(download.last_successful_download, now)
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
self.assertEqual(download.next_download, expected + timedelta(hours=1, seconds=1))
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
self.assertEqual(download.next_download, expected)
# Try the download again, it finally succeeds.
http_downloader.do_download.reset_mock()
now = local_timezone(datetime(2020, 1, 1, 3, 0, 0))
now = local_timezone(datetime(2020, 1, 1, 4, 0, 1))
mock_now.return_value = now
http_downloader.do_download.return_value = True
self.mgr.renew_recurring_downloads(session)
@ -227,7 +228,8 @@ class TestDownloader(TestAPI):
download = session.query(Download).one()
self.assertEqual(download.status, 'complete')
self.assertEqual(download.last_successful_download, now)
self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0)))
# Floats cause slightly wrong date.
self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0, 997200)))
@wrap_test_db
@mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
@ -356,7 +358,11 @@ def test_get_next_download(test_session, test_download_manager, fake_now):
result = test_download_manager.get_next_download(download)
assert result == expected, f'{attempts} != {result}'
download = Download()
download.frequency = DownloadFrequency.weekly
assert test_download_manager.get_next_download(download) == local_timezone(datetime(2000, 1, 1))
d1 = Download(url='https://example.com/1', frequency=DownloadFrequency.weekly)
d2 = Download(url='https://example.com/2', frequency=DownloadFrequency.weekly)
d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly)
test_session.add_all([d1, d2, d3])
test_session.commit()
assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 1))
assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 4, 12))
assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 2, 18))