Fixing bug where a daily download would be performed repeatedly on the same day.

This commit is contained in:
Roland 2022-01-31 12:39:55 -07:00
parent 8d93268552
commit 70dc356adc
No known key found for this signature in database
GPG key ID: 9D6DBD3ACF11AB93
4 changed files with 89 additions and 90 deletions

View file

@ -129,4 +129,5 @@ def fake_now():
try:
yield set_test_now
finally:
set_test_now(None) # reset now() to its original functionality.
# reset now() to its original functionality.
set_test_now(None) # noqa

View file

@ -27,6 +27,7 @@ def set_test_now(dt: datetime):
dt = local_timezone(dt)
global TEST_DATETIME
TEST_DATETIME = dt
return dt
def set_timezone(tz: pytz.timezone):

View file

@ -356,7 +356,7 @@ class DownloadManager:
download.started()
session.commit()
success = None
success = True
failure = None
try:
success = downloader.do_download(download)
@ -414,9 +414,7 @@ class DownloadManager:
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
def recurring_download(self, url: str, frequency: int, skip_download: bool = False) -> Download:
"""
Schedule a recurring download.
"""
"""Schedule a recurring download."""
with get_db_session(commit=True) as session:
download = self.create_download(url, session=session, skip_download=True)
download.frequency = frequency
@ -628,12 +626,17 @@ class DownloadManager:
downloads = [i.id for i in session.query(Download).filter_by(frequency=freq).order_by(Download.id)]
index = downloads.index(download.id)
# Download was successful. Spread the same-frequency downloads out.
# Download was successful. Spread the same-frequency downloads out over their iteration.
start_date = local_timezone(datetime(2000, 1, 1))
# Weeks/months/etc since start_date.
iterations = ((now() - start_date) // freq).total_seconds()
start_date = start_date + timedelta(seconds=iterations * freq)
# Download slots start the next nearest iteration since 2000-01-01.
# For example, if a weekly download was performed on 2000-01-01 it will get the same slot within
# 2000-01-01 to 2000-01-08.
start_date = start_date + timedelta(seconds=(iterations * freq) + freq)
end_date = start_date + timedelta(seconds=freq)
# Get this downloads position in the next iteration. If a download is performed at the 3rd slot this week,
# it will be downloaded the 3rd slot of next week.
zagger = zig_zag(start_date, end_date)
next_download = [i for i, j in zip(zagger, range(len(downloads)))][index]
next_download = local_timezone(next_download)

View file

@ -7,7 +7,7 @@ from unittest.mock import MagicMock
import pytest
from wrolpi import downloader
from wrolpi.dates import local_timezone
from wrolpi.dates import local_timezone, Seconds
from wrolpi.db import get_db_context
from wrolpi.downloader import Downloader, Download, DownloadFrequency
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
@ -152,85 +152,6 @@ class TestDownloader(TestAPI):
self.assertEqual(download.attempts, 3)
self.assertEqual(download.status, 'failed')
@wrap_test_db
def test_recurring_downloads(self):
"""
A recurring Download should be downloaded repeatedly forever.
"""
_, session = get_db_context()
http_downloader = HTTPDownloader()
http_downloader.do_download.return_value = True
http_downloader.do_download: MagicMock # noqa
self.mgr.register_downloader(http_downloader)
# Download every hour.
self.mgr.recurring_download('https://example.com', 3600, skip_download=True)
# One download is scheduled.
downloads = self.mgr.get_new_downloads(session)
self.assertEqual(
[(i.url, i.frequency) for i in downloads],
[('https://example.com', 3600)]
)
with mock.patch('wrolpi.downloader.now') as mock_now:
now = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
mock_now.return_value = now
# Download is processed and successful, no longer "new".
self.mgr.do_downloads_sync()
http_downloader.do_download.assert_called_once()
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
downloads = list(self.mgr.get_recurring_downloads(session))
self.assertEqual(len(downloads), 1)
download = downloads[0]
expected = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
self.assertEqual(download.next_download, expected)
self.assertEqual(download.last_successful_download, now)
# Download is not due for an hour.
self.mgr.renew_recurring_downloads(session)
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
self.assertEqual(download.last_successful_download, now)
# Download is due an hour later.
mock_now.return_value = local_timezone(datetime(2020, 1, 1, 2, 0, 1))
self.mgr.renew_recurring_downloads(session)
downloads = list(self.mgr.get_new_downloads(session))
self.assertEqual(len(downloads), 1)
download = downloads[0]
# Download is "new" but has not been done a second time.
self.assertEqual(download.next_download, expected)
self.assertEqual(download.last_successful_download, now)
# Try the download, but it fails.
http_downloader.do_download.reset_mock()
http_downloader.do_download.return_value = False
self.mgr.do_downloads_sync()
http_downloader.do_download.assert_called_once()
download = session.query(Download).one()
# Download is deferred, last successful download remains the same.
self.assertEqual(download.status, 'deferred')
self.assertEqual(download.last_successful_download, now)
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
self.assertEqual(download.next_download, expected)
# Try the download again, it finally succeeds.
http_downloader.do_download.reset_mock()
now = local_timezone(datetime(2020, 1, 1, 4, 0, 1))
mock_now.return_value = now
http_downloader.do_download.return_value = True
self.mgr.renew_recurring_downloads(session)
self.mgr.do_downloads_sync()
http_downloader.do_download.assert_called_once()
download = session.query(Download).one()
self.assertEqual(download.status, 'complete')
self.assertEqual(download.last_successful_download, now)
# Floats cause slightly wrong date.
self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0, 997200)))
@wrap_test_db
@mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
def test_delete_old_once_downloads(self):
@ -363,6 +284,79 @@ def test_get_next_download(test_session, test_download_manager, fake_now):
d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly)
test_session.add_all([d1, d2, d3])
test_session.commit()
assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 1))
assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 4, 12))
assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 2, 18))
# Downloads are spread out over the next week.
assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 8))
assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 11, 12))
assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 9, 18))
def test_recurring_downloads(test_session, test_download_manager, fake_now):
"""
A recurring Download should be downloaded repeatedly forever.
"""
_, session = get_db_context()
http_downloader = HTTPDownloader()
http_downloader.do_download = MagicMock()
http_downloader.do_download.return_value = True
test_download_manager.register_downloader(http_downloader)
# Download every hour.
test_download_manager.recurring_download('https://example.com', Seconds.hour, skip_download=True)
# One download is scheduled.
downloads = test_download_manager.get_new_downloads(session)
assert [(i.url, i.frequency) for i in downloads] == [('https://example.com', Seconds.hour)]
now = fake_now(datetime(2020, 1, 1, 0, 0, 0))
# Download is processed and successful, no longer "new".
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
assert list(test_download_manager.get_new_downloads(session)) == []
downloads = list(test_download_manager.get_recurring_downloads(session))
assert len(downloads) == 1
download = downloads[0]
expected = local_timezone(datetime(2020, 1, 1, 1, 0, 0))
assert download.next_download == expected
assert download.last_successful_download == now
# Download is not due for an hour.
test_download_manager.renew_recurring_downloads(session)
assert list(test_download_manager.get_new_downloads(session)) == []
assert download.last_successful_download == now
# Download is due an hour later.
fake_now(datetime(2020, 1, 1, 2, 0, 1))
test_download_manager.renew_recurring_downloads(session)
(download,) = list(test_download_manager.get_new_downloads(session))
# Download is "new" but has not been downloaded a second time.
assert download.next_download == expected
assert download.last_successful_download == now
assert download.status == 'new'
# Try the download, but it fails.
http_downloader.do_download.reset_mock()
http_downloader.do_download.return_value = False
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
download = session.query(Download).one()
# Download is deferred, last successful download remains the same.
assert download.status == 'deferred'
assert download.last_successful_download == now
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
assert download.next_download == expected
# Try the download again, it finally succeeds.
http_downloader.do_download.reset_mock()
now = fake_now(datetime(2020, 1, 1, 4, 0, 1))
http_downloader.do_download.return_value = True
test_download_manager.renew_recurring_downloads(session)
test_download_manager.do_downloads_sync()
http_downloader.do_download.assert_called_once()
download = session.query(Download).one()
assert download.status == 'complete'
assert download.last_successful_download == now
# Floats cause slightly wrong date.
assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200))