mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Fixing bug where a daily download would be performed repeatedly on the same day.
This commit is contained in:
parent
8d93268552
commit
70dc356adc
4 changed files with 89 additions and 90 deletions
|
|
@ -129,4 +129,5 @@ def fake_now():
|
|||
try:
|
||||
yield set_test_now
|
||||
finally:
|
||||
set_test_now(None) # reset now() to its original functionality.
|
||||
# reset now() to its original functionality.
|
||||
set_test_now(None) # noqa
|
||||
|
|
|
|||
|
|
@ -27,6 +27,7 @@ def set_test_now(dt: datetime):
|
|||
dt = local_timezone(dt)
|
||||
global TEST_DATETIME
|
||||
TEST_DATETIME = dt
|
||||
return dt
|
||||
|
||||
|
||||
def set_timezone(tz: pytz.timezone):
|
||||
|
|
|
|||
|
|
@ -356,7 +356,7 @@ class DownloadManager:
|
|||
download.started()
|
||||
session.commit()
|
||||
|
||||
success = None
|
||||
success = True
|
||||
failure = None
|
||||
try:
|
||||
success = downloader.do_download(download)
|
||||
|
|
@ -414,9 +414,7 @@ class DownloadManager:
|
|||
curs.execute("UPDATE download SET status='new' WHERE status='pending' OR status='deferred'")
|
||||
|
||||
def recurring_download(self, url: str, frequency: int, skip_download: bool = False) -> Download:
|
||||
"""
|
||||
Schedule a recurring download.
|
||||
"""
|
||||
"""Schedule a recurring download."""
|
||||
with get_db_session(commit=True) as session:
|
||||
download = self.create_download(url, session=session, skip_download=True)
|
||||
download.frequency = frequency
|
||||
|
|
@ -628,12 +626,17 @@ class DownloadManager:
|
|||
downloads = [i.id for i in session.query(Download).filter_by(frequency=freq).order_by(Download.id)]
|
||||
index = downloads.index(download.id)
|
||||
|
||||
# Download was successful. Spread the same-frequency downloads out.
|
||||
# Download was successful. Spread the same-frequency downloads out over their iteration.
|
||||
start_date = local_timezone(datetime(2000, 1, 1))
|
||||
# Weeks/months/etc since start_date.
|
||||
iterations = ((now() - start_date) // freq).total_seconds()
|
||||
start_date = start_date + timedelta(seconds=iterations * freq)
|
||||
# Download slots start the next nearest iteration since 2000-01-01.
|
||||
# For example, if a weekly download was performed on 2000-01-01 it will get the same slot within
|
||||
# 2000-01-01 to 2000-01-08.
|
||||
start_date = start_date + timedelta(seconds=(iterations * freq) + freq)
|
||||
end_date = start_date + timedelta(seconds=freq)
|
||||
# Get this downloads position in the next iteration. If a download is performed at the 3rd slot this week,
|
||||
# it will be downloaded the 3rd slot of next week.
|
||||
zagger = zig_zag(start_date, end_date)
|
||||
next_download = [i for i, j in zip(zagger, range(len(downloads)))][index]
|
||||
next_download = local_timezone(next_download)
|
||||
|
|
|
|||
|
|
@ -7,7 +7,7 @@ from unittest.mock import MagicMock
|
|||
import pytest
|
||||
|
||||
from wrolpi import downloader
|
||||
from wrolpi.dates import local_timezone
|
||||
from wrolpi.dates import local_timezone, Seconds
|
||||
from wrolpi.db import get_db_context
|
||||
from wrolpi.downloader import Downloader, Download, DownloadFrequency
|
||||
from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
|
||||
|
|
@ -152,85 +152,6 @@ class TestDownloader(TestAPI):
|
|||
self.assertEqual(download.attempts, 3)
|
||||
self.assertEqual(download.status, 'failed')
|
||||
|
||||
@wrap_test_db
|
||||
def test_recurring_downloads(self):
|
||||
"""
|
||||
A recurring Download should be downloaded repeatedly forever.
|
||||
"""
|
||||
_, session = get_db_context()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download.return_value = True
|
||||
http_downloader.do_download: MagicMock # noqa
|
||||
self.mgr.register_downloader(http_downloader)
|
||||
|
||||
# Download every hour.
|
||||
self.mgr.recurring_download('https://example.com', 3600, skip_download=True)
|
||||
|
||||
# One download is scheduled.
|
||||
downloads = self.mgr.get_new_downloads(session)
|
||||
self.assertEqual(
|
||||
[(i.url, i.frequency) for i in downloads],
|
||||
[('https://example.com', 3600)]
|
||||
)
|
||||
|
||||
with mock.patch('wrolpi.downloader.now') as mock_now:
|
||||
now = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
|
||||
mock_now.return_value = now
|
||||
|
||||
# Download is processed and successful, no longer "new".
|
||||
self.mgr.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
|
||||
downloads = list(self.mgr.get_recurring_downloads(session))
|
||||
self.assertEqual(len(downloads), 1)
|
||||
download = downloads[0]
|
||||
expected = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
|
||||
self.assertEqual(download.next_download, expected)
|
||||
self.assertEqual(download.last_successful_download, now)
|
||||
|
||||
# Download is not due for an hour.
|
||||
self.mgr.renew_recurring_downloads(session)
|
||||
self.assertEqual(list(self.mgr.get_new_downloads(session)), [])
|
||||
self.assertEqual(download.last_successful_download, now)
|
||||
|
||||
# Download is due an hour later.
|
||||
mock_now.return_value = local_timezone(datetime(2020, 1, 1, 2, 0, 1))
|
||||
self.mgr.renew_recurring_downloads(session)
|
||||
downloads = list(self.mgr.get_new_downloads(session))
|
||||
self.assertEqual(len(downloads), 1)
|
||||
download = downloads[0]
|
||||
# Download is "new" but has not been done a second time.
|
||||
self.assertEqual(download.next_download, expected)
|
||||
self.assertEqual(download.last_successful_download, now)
|
||||
|
||||
# Try the download, but it fails.
|
||||
http_downloader.do_download.reset_mock()
|
||||
http_downloader.do_download.return_value = False
|
||||
self.mgr.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
download = session.query(Download).one()
|
||||
# Download is deferred, last successful download remains the same.
|
||||
self.assertEqual(download.status, 'deferred')
|
||||
self.assertEqual(download.last_successful_download, now)
|
||||
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
|
||||
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
|
||||
self.assertEqual(download.next_download, expected)
|
||||
|
||||
# Try the download again, it finally succeeds.
|
||||
http_downloader.do_download.reset_mock()
|
||||
now = local_timezone(datetime(2020, 1, 1, 4, 0, 1))
|
||||
mock_now.return_value = now
|
||||
http_downloader.do_download.return_value = True
|
||||
self.mgr.renew_recurring_downloads(session)
|
||||
self.mgr.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
download = session.query(Download).one()
|
||||
self.assertEqual(download.status, 'complete')
|
||||
self.assertEqual(download.last_successful_download, now)
|
||||
# Floats cause slightly wrong date.
|
||||
self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0, 997200)))
|
||||
|
||||
@wrap_test_db
|
||||
@mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
|
||||
def test_delete_old_once_downloads(self):
|
||||
|
|
@ -363,6 +284,79 @@ def test_get_next_download(test_session, test_download_manager, fake_now):
|
|||
d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly)
|
||||
test_session.add_all([d1, d2, d3])
|
||||
test_session.commit()
|
||||
assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 1))
|
||||
assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 4, 12))
|
||||
assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 2, 18))
|
||||
# Downloads are spread out over the next week.
|
||||
assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 8))
|
||||
assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 11, 12))
|
||||
assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 9, 18))
|
||||
|
||||
|
||||
def test_recurring_downloads(test_session, test_download_manager, fake_now):
|
||||
"""
|
||||
A recurring Download should be downloaded repeatedly forever.
|
||||
"""
|
||||
_, session = get_db_context()
|
||||
|
||||
http_downloader = HTTPDownloader()
|
||||
http_downloader.do_download = MagicMock()
|
||||
http_downloader.do_download.return_value = True
|
||||
test_download_manager.register_downloader(http_downloader)
|
||||
|
||||
# Download every hour.
|
||||
test_download_manager.recurring_download('https://example.com', Seconds.hour, skip_download=True)
|
||||
|
||||
# One download is scheduled.
|
||||
downloads = test_download_manager.get_new_downloads(session)
|
||||
assert [(i.url, i.frequency) for i in downloads] == [('https://example.com', Seconds.hour)]
|
||||
|
||||
now = fake_now(datetime(2020, 1, 1, 0, 0, 0))
|
||||
|
||||
# Download is processed and successful, no longer "new".
|
||||
test_download_manager.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
assert list(test_download_manager.get_new_downloads(session)) == []
|
||||
downloads = list(test_download_manager.get_recurring_downloads(session))
|
||||
assert len(downloads) == 1
|
||||
download = downloads[0]
|
||||
expected = local_timezone(datetime(2020, 1, 1, 1, 0, 0))
|
||||
assert download.next_download == expected
|
||||
assert download.last_successful_download == now
|
||||
|
||||
# Download is not due for an hour.
|
||||
test_download_manager.renew_recurring_downloads(session)
|
||||
assert list(test_download_manager.get_new_downloads(session)) == []
|
||||
assert download.last_successful_download == now
|
||||
|
||||
# Download is due an hour later.
|
||||
fake_now(datetime(2020, 1, 1, 2, 0, 1))
|
||||
test_download_manager.renew_recurring_downloads(session)
|
||||
(download,) = list(test_download_manager.get_new_downloads(session))
|
||||
# Download is "new" but has not been downloaded a second time.
|
||||
assert download.next_download == expected
|
||||
assert download.last_successful_download == now
|
||||
assert download.status == 'new'
|
||||
|
||||
# Try the download, but it fails.
|
||||
http_downloader.do_download.reset_mock()
|
||||
http_downloader.do_download.return_value = False
|
||||
test_download_manager.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
download = session.query(Download).one()
|
||||
# Download is deferred, last successful download remains the same.
|
||||
assert download.status == 'deferred'
|
||||
assert download.last_successful_download == now
|
||||
# Download should be retried after the DEFAULT_RETRY_FREQUENCY.
|
||||
expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
|
||||
assert download.next_download == expected
|
||||
|
||||
# Try the download again, it finally succeeds.
|
||||
http_downloader.do_download.reset_mock()
|
||||
now = fake_now(datetime(2020, 1, 1, 4, 0, 1))
|
||||
http_downloader.do_download.return_value = True
|
||||
test_download_manager.renew_recurring_downloads(session)
|
||||
test_download_manager.do_downloads_sync()
|
||||
http_downloader.do_download.assert_called_once()
|
||||
download = session.query(Download).one()
|
||||
assert download.status == 'complete'
|
||||
assert download.last_successful_download == now
|
||||
# Floats cause slightly wrong date.
|
||||
assert download.next_download == local_timezone(datetime(2020, 1, 1, 5, 0, 0, 997200))
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue