From 0062511931559ec3184841921d9395fd2132381d Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 26 Jan 2022 15:16:49 -0700
Subject: [PATCH 1/8] Creating date_zig_zag function which creates an infinite
 amount of dates between two dates.

---
 wrolpi/common.py               | 43 +++++++++++++++++++++++++++++++++-
 wrolpi/dates.py                |  8 +++++++
 wrolpi/downloader.py           | 36 ++++++++++++++++++----------
 wrolpi/test/test_common.py     | 38 +++++++++++++++++++++++++++++-
 wrolpi/test/test_downloader.py | 29 ++++++++++++++++++++++-
 5 files changed, 139 insertions(+), 15 deletions(-)

diff --git a/wrolpi/common.py b/wrolpi/common.py
index c9875ab5..b5db67a7 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -11,11 +11,12 @@ import re
 import string
 import tempfile
 from datetime import datetime, date
+from decimal import Decimal
 from functools import wraps
 from itertools import islice, filterfalse, tee
 from multiprocessing import Event, Queue
 from pathlib import Path
-from typing import Union, Callable, Tuple, Dict, Mapping, List, Iterable, Optional
+from typing import Union, Callable, Tuple, Dict, Mapping, List, Iterable, Optional, Generator
 from urllib.parse import urlunsplit, urlparse
 
 import yaml
@@ -600,3 +601,43 @@ def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = Fa
         else:
             del os.environ['HOME']
     return
+
+
+numeric = Union[int, float, complex, Decimal]
+
+
+def zig_zag(low: numeric, high: numeric) -> Generator[numeric, None, None]:
+    """
+    Generate numbers between `low` and `high` that are spread out evenly.  Produces infinite results.
+
+    >>> list(zig_zag(0, 10))
+    [0, 5, 2, 7, 1, 3, 6, 8, 0, 1, 3, 4, 5, 6, 8, 9]
+    >>> list(zig_zag(0, 5))
+    [0, 2, 1, 3, 0, 1, 3, 4]
+    >>> list(zig_zag(50, 100))
+    [50, 75, 62, 87, 56, 68, 81, 93, 53, 59, 65]
+    """
+    low_type = type(low)
+    if not isinstance(high, low_type):
+        raise ValueError(f'high and low must be same type')
+
+    # Special thanks to my wife for helping me solve this! :*
+    results = set()
+    num = low
+    divisor = 2
+    diff = high - low
+    while True:
+        if num not in results:
+            yield low_type(num)
+            results.add(num)
+        num += diff / divisor
+        if num >= high:
+            divisor *= 2
+            num = low + (diff / divisor)
+
+
+def date_zig_zag(low: datetime, high: datetime) -> Generator[datetime, None, None]:
+    """
+    Generate datetimes between `low` and `high`.  See `zig_zag`.
+    """
+    yield from map(datetime.fromtimestamp, zig_zag(low.timestamp(), high.timestamp()))
diff --git a/wrolpi/dates.py b/wrolpi/dates.py
index 09dc36f6..14926e7e 100644
--- a/wrolpi/dates.py
+++ b/wrolpi/dates.py
@@ -1,5 +1,6 @@
 import logging
 from datetime import datetime, date, timezone
+from enum import Enum
 
 import pytest
 import pytz
@@ -14,6 +15,13 @@ DEFAULT_TIMEZONE = pytz.timezone(DEFAULT_TIMEZONE_STR)
 TEST_DATETIME: datetime = None
 
 
+class Seconds(int, Enum):
+    minute = 60
+    hour = minute * 60
+    day = hour * 24
+    week = day * 7
+
+
 @pytest.fixture
 def fake_now():
     def _(dt: datetime):
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index 0d5ab963..8dc60bcd 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -3,7 +3,7 @@ import multiprocessing
 import pathlib
 import subprocess
 from collections import defaultdict
-from datetime import timedelta
+from datetime import timedelta, datetime
 from enum import Enum
 from functools import partial
 from operator import attrgetter
@@ -14,7 +14,7 @@ from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Session
 
 from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check
-from wrolpi.dates import TZDateTime, now
+from wrolpi.dates import TZDateTime, now, Seconds, local_timezone
 from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
 from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
 from wrolpi.vars import PYTEST
@@ -73,13 +73,6 @@ class Download(ModelHelper, Base):
         )
         return d
 
-    def increment_next_download(self):
-        if self.frequency:
-            delta = timedelta(seconds=self.frequency)
-        else:
-            delta = DEFAULT_RETRY_FREQUENCY
-        self.next_download = now() + delta
-
     def renew(self, reset_attempts: bool = False):
         """
         Mark this Download as "new" so it will be retried.
@@ -93,7 +86,6 @@ class Download(ModelHelper, Base):
         Download should be tried again after a time.
         """
         self.status = 'deferred'
-        self.increment_next_download()
 
     def fail(self):
         """
@@ -116,8 +108,6 @@ class Download(ModelHelper, Base):
         """
         self.status = 'complete'
         self.last_successful_download = now()
-        if self.frequency:
-            self.increment_next_download()
 
     def get_downloader(self):
         if self.downloader:
@@ -594,6 +584,9 @@ class DownloadManager:
 
     @optional_session
     def get_fe_downloads(self, session: Session = None):
+        """
+        Get downloads for the Frontend.
+        """
         data = dict(
             recurring_downloads=self.get_recurring_downloads(session=session),
             once_downloads=self.get_once_downloads(session=session, limit=100),
@@ -604,6 +597,25 @@ class DownloadManager:
     def get_pending_downloads(self, session: Session):
         return session.query(Download).filter_by(status='pending').all()
 
+    @optional_session
+    def get_next_download(self, download: Download, session: Session):
+        if download.status == 'deferred':
+            # Increase next_download slowly at first, then by large gaps later.  The largest gap is the download
+            # frequency.
+            hours = 3 ** (download.attempts or 1)
+            seconds = hours * Seconds.hour
+            seconds = min(seconds, download.frequency)
+            delta = timedelta(seconds=seconds)
+            return now() + delta
+
+        # Download was successful.  Spread the same-frequency downloads out.
+        start_date = local_timezone(datetime(2000, 1, 1))
+        elapsed = now() - start_date
+        chunk = elapsed // download.frequency
+        this_frequency = start_date + (download.frequency * chunk)
+        next_frequency = this_frequency + timedelta(seconds=download.frequency)
+        print(f'{this_frequency=} {next_frequency=}')
+
 
 # The global DownloadManager.  This should be used everywhere!
 download_manager = DownloadManager()
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index c9c4a103..3c047060 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -3,11 +3,12 @@ import pathlib
 import tempfile
 import unittest
 from datetime import date, datetime
+from decimal import Decimal
 
 import pytest
 from sanic_openapi import doc
 
-from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir
+from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag, date_zig_zag
 from wrolpi.dates import set_timezone, now
 from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone
 from wrolpi.schema import validate_data
@@ -346,3 +347,38 @@ def test_chdir():
         with chdir(pathlib.Path(d)):
             assert os.getcwd() == d
             assert os.environ['HOME'] != os.getcwd()
+
+
+@pytest.mark.parametrize('low,high,expected', [
+    (0, 10000, [0, 5000, 2500, 7500, 1250, 3750, 6250, 8750, 625, 1875, 3125, 4375, 5625, 6875, 8125, 9375]),
+    (0, 1000, [0, 500, 250, 750, 125, 375, 625, 875, 62, 187, 312, 437, 562, 687, 812, 937]),
+    (0, 100, [0, 50, 25, 75, 12, 37, 62, 87, 6, 18, 31, 43, 56, 68, 81, 93]),
+    (0, 10, [0, 5, 2, 7, 1, 3, 6, 8, 0, 1, 3, 4, 5, 6, 8, 9]),
+    (0, 5, [0, 2, 1, 3, 0, 1, 3, 4]),
+    (50, 100, [50, 75, 62, 87, 56, 68, 81, 93, 53, 59, 65]),
+    (50.0, 100.0, [50.0, 75.0, 62.5, 87.5, 56.25, 68.75, 81.25, 93.75, 53.125, 59.375, 65.625]),
+    (Decimal('50.0'), Decimal('100.0'), [50.0, 75.0, 62.5, 87.5, 56.25, 68.75, 81.25, 93.75, 53.125, 59.375, 65.625]),
+])
+def test_zig_zag(low, high, expected):
+    zagger = zig_zag(low, high)
+    for i in expected:
+        result = next(zagger)
+        assert result == i
+        assert low <= result < high
+
+
+def test_date_zig_zag():
+    low, high = datetime(2000, 1, 1), datetime(2000, 1, 8)
+    zagger = date_zig_zag(low, high)
+    expected = [
+        datetime(2000, 1, 1),
+        datetime(2000, 1, 4, 12),
+        datetime(2000, 1, 2, 18),
+        datetime(2000, 1, 6, 6),
+        datetime(2000, 1, 1, 21),
+        datetime(2000, 1, 3, 15),
+    ]
+    for i in expected:
+        result = next(zagger)
+        assert result == i
+        assert low <= result < high
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index f575bba5..d34d501b 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -9,7 +9,7 @@ import pytest
 from wrolpi import downloader
 from wrolpi.dates import local_timezone
 from wrolpi.db import get_db_context
-from wrolpi.downloader import Downloader, Download
+from wrolpi.downloader import Downloader, Download, DownloadFrequency
 from wrolpi.errors import UnrecoverableDownloadError, InvalidDownload, WROLModeEnabled
 from wrolpi.test.common import wrap_test_db, TestAPI
 
@@ -333,3 +333,30 @@ def test_download_get_downloader(test_session, test_download_manager):
     download2 = test_download_manager.create_download('https://example.com', test_session, skip_download=True)
     download2.downloader = 'bad downloader'
     assert download2.get_downloader() is None
+
+
+def test_get_next_download(test_session, test_download_manager, fake_now):
+    fake_now(datetime(2000, 1, 1))
+    download = Download()
+    download.frequency = DownloadFrequency.weekly
+    download.status = 'deferred'
+
+    # next_download slowly increases as we accumulate attempts.  Largest gap is the download frequency.
+    attempts_expected = [
+        (0, local_timezone(datetime(2000, 1, 1, 3))),
+        (1, local_timezone(datetime(2000, 1, 1, 3))),
+        (2, local_timezone(datetime(2000, 1, 1, 9))),
+        (3, local_timezone(datetime(2000, 1, 2, 3))),
+        (4, local_timezone(datetime(2000, 1, 4, 9))),
+        (5, local_timezone(datetime(2000, 1, 8))),
+        (6, local_timezone(datetime(2000, 1, 8))),
+    ]
+    for attempts, expected in attempts_expected:
+        download.attempts = attempts
+        result = test_download_manager.get_next_download(download)
+        assert result == expected, f'{attempts} != {result}'
+
+    download = Download()
+    download.frequency = DownloadFrequency.weekly
+    assert test_download_manager.get_next_download(download) == local_timezone(datetime(2000, 1, 1))
+

From 418613e83ef557c5045b479c5c3198f65cad55e5 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 26 Jan 2022 15:45:29 -0700
Subject: [PATCH 2/8] Combining date_zig_zag and into zig_zag.

---
 wrolpi/common.py           | 27 ++++++++++++---------------
 wrolpi/test/test_common.py | 34 +++++++++++++++-------------------
 2 files changed, 27 insertions(+), 34 deletions(-)

diff --git a/wrolpi/common.py b/wrolpi/common.py
index b5db67a7..20b82646 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -603,23 +603,27 @@ def chdir(directory: Union[pathlib.Path, str, None] = None, with_home: bool = Fa
     return
 
 
-numeric = Union[int, float, complex, Decimal]
+ZIG_TYPE = Union[int, float, complex, Decimal, datetime]
 
 
-def zig_zag(low: numeric, high: numeric) -> Generator[numeric, None, None]:
+def zig_zag(low: ZIG_TYPE, high: ZIG_TYPE) -> Generator[ZIG_TYPE, None, None]:
     """
-    Generate numbers between `low` and `high` that are spread out evenly.  Produces infinite results.
+    Generate numbers between `low` and `high` that are
+    spread out evenly.  Produces infinite results.
 
     >>> list(zig_zag(0, 10))
     [0, 5, 2, 7, 1, 3, 6, 8, 0, 1, 3, 4, 5, 6, 8, 9]
     >>> list(zig_zag(0, 5))
     [0, 2, 1, 3, 0, 1, 3, 4]
-    >>> list(zig_zag(50, 100))
-    [50, 75, 62, 87, 56, 68, 81, 93, 53, 59, 65]
+    >>> list(zig_zag(50.0, 100.0))
+    [50.0, 75.0, 62.5, 87.5, 56.25, 68.75, 81.25, 93.75, 53.125, 59.375, 65.625]
     """
-    low_type = type(low)
-    if not isinstance(high, low_type):
+    output_type = type(low)
+    if not isinstance(high, type(low)):
         raise ValueError(f'high and low must be same type')
+    if isinstance(low, datetime) and isinstance(high, datetime):
+        low, high = low.timestamp(), high.timestamp()
+        output_type = datetime.fromtimestamp
 
     # Special thanks to my wife for helping me solve this! :*
     results = set()
@@ -628,16 +632,9 @@ def zig_zag(low: numeric, high: numeric) -> Generator[numeric, None, None]:
     diff = high - low
     while True:
         if num not in results:
-            yield low_type(num)
+            yield output_type(num)
             results.add(num)
         num += diff / divisor
         if num >= high:
             divisor *= 2
             num = low + (diff / divisor)
-
-
-def date_zig_zag(low: datetime, high: datetime) -> Generator[datetime, None, None]:
-    """
-    Generate datetimes between `low` and `high`.  See `zig_zag`.
-    """
-    yield from map(datetime.fromtimestamp, zig_zag(low.timestamp(), high.timestamp()))
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index 3c047060..c09164b5 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -8,7 +8,7 @@ from decimal import Decimal
 import pytest
 from sanic_openapi import doc
 
-from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag, date_zig_zag
+from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag
 from wrolpi.dates import set_timezone, now
 from wrolpi.errors import NoBodyContents, MissingRequiredField, ExcessJSONFields, InvalidTimezone
 from wrolpi.schema import validate_data
@@ -353,11 +353,24 @@ def test_chdir():
     (0, 10000, [0, 5000, 2500, 7500, 1250, 3750, 6250, 8750, 625, 1875, 3125, 4375, 5625, 6875, 8125, 9375]),
     (0, 1000, [0, 500, 250, 750, 125, 375, 625, 875, 62, 187, 312, 437, 562, 687, 812, 937]),
     (0, 100, [0, 50, 25, 75, 12, 37, 62, 87, 6, 18, 31, 43, 56, 68, 81, 93]),
+    # Values repeat when there are not enough.
     (0, 10, [0, 5, 2, 7, 1, 3, 6, 8, 0, 1, 3, 4, 5, 6, 8, 9]),
     (0, 5, [0, 2, 1, 3, 0, 1, 3, 4]),
+    (0, 2, [0, 1, 0, 1, 0, 0, 1, 1, 0, 0, 0, 0, 1, 1, 1, 1]),
     (50, 100, [50, 75, 62, 87, 56, 68, 81, 93, 53, 59, 65]),
+    (8, 98, [8, 53, 30, 75, 19, 41, 64]),
+    # Floats are supported.  Output is identical to above.
     (50.0, 100.0, [50.0, 75.0, 62.5, 87.5, 56.25, 68.75, 81.25, 93.75, 53.125, 59.375, 65.625]),
-    (Decimal('50.0'), Decimal('100.0'), [50.0, 75.0, 62.5, 87.5, 56.25, 68.75, 81.25, 93.75, 53.125, 59.375, 65.625]),
+    # Decimals are supported.  Output is identical to above.
+    (Decimal('50.0'), Decimal('100.0'), [
+        Decimal('50'), Decimal('75'), Decimal('62.5'), Decimal('87.5'), Decimal('56.25'), Decimal('68.75'),
+        Decimal('81.25'), Decimal('93.75'), Decimal('53.125'), Decimal('59.375'), Decimal('65.625')
+    ]),
+    # Datetimes are supported.
+    (datetime(2000, 1, 1), datetime(2000, 1, 8), [
+        datetime(2000, 1, 1), datetime(2000, 1, 4, 12), datetime(2000, 1, 2, 18), datetime(2000, 1, 6, 6),
+        datetime(2000, 1, 1, 21), datetime(2000, 1, 3, 15),
+    ])
 ])
 def test_zig_zag(low, high, expected):
     zagger = zig_zag(low, high)
@@ -365,20 +378,3 @@ def test_zig_zag(low, high, expected):
         result = next(zagger)
         assert result == i
         assert low <= result < high
-
-
-def test_date_zig_zag():
-    low, high = datetime(2000, 1, 1), datetime(2000, 1, 8)
-    zagger = date_zig_zag(low, high)
-    expected = [
-        datetime(2000, 1, 1),
-        datetime(2000, 1, 4, 12),
-        datetime(2000, 1, 2, 18),
-        datetime(2000, 1, 6, 6),
-        datetime(2000, 1, 1, 21),
-        datetime(2000, 1, 3, 15),
-    ]
-    for i in expected:
-        result = next(zagger)
-        assert result == i
-        assert low <= result < high

From 3dda813b9e010faceb0bf59ee52860caae93a535 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 26 Jan 2022 17:10:18 -0700
Subject: [PATCH 3/8] Download.next_download is assigned by the
 DownloadManager.

---
 modules/archive/test/test_lib.py | 128 ++++++++++++++++---------------
 wrolpi/downloader.py             |  39 ++++++++--
 wrolpi/test/test_downloader.py   |  26 ++++---
 3 files changed, 113 insertions(+), 80 deletions(-)

diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index 39fe6da6..e868be84 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -51,35 +51,6 @@ class TestArchive(TestAPI):
         assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
         assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
 
-    @wrap_test_db
-    def test_new_archive(self):
-        with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
-            archive1 = do_archive('https://example.com')
-            # Everything is filled out.
-            self.assertIsInstance(archive1, Archive)
-            self.assertIsNotNone(archive1.archive_datetime)
-            self.assertIsInstance(archive1.singlefile_path, MediaPath)
-            self.assertIsInstance(archive1.readability_path, MediaPath)
-            self.assertIsInstance(archive1.readability_txt_path, MediaPath)
-            self.assertIsInstance(archive1.screenshot_path, MediaPath)
-            self.assertEqual(archive1.title, 'ジにてこちら')
-            self.assertIsNotNone(archive1.url)
-            self.assertIsNotNone(archive1.domain)
-
-            # The actual files were dumped and read correctly.
-            with open(archive1.singlefile_path.path) as fh:
-                self.assertEqual(fh.read(), '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>')
-            with open(archive1.readability_path.path) as fh:
-                self.assertEqual(fh.read(), '<html>test readability content</html>')
-            with open(archive1.readability_txt_path.path) as fh:
-                self.assertEqual(fh.read(), '<html>test readability textContent</html>')
-            with open(archive1.readability_json_path.path) as fh:
-                self.assertEqual(json.load(fh), {'title': 'ジにてこちら', 'url': 'https://example.com'})
-
-            archive2 = do_archive('https://example.com')
-            # Domain is reused.
-            self.assertEqual(archive1.domain, archive2.domain)
-
     @wrap_test_db
     def test_no_screenshot(self):
         with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
@@ -132,39 +103,6 @@ class TestArchive(TestAPI):
             except ValueError as e:
                 self.assertIn('relative', str(e), f'Relative path error was not raised')
 
-    @wrap_test_db
-    def test_get_title_from_html(self):
-        with wrap_media_directory():
-            with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
-                archive = do_archive('example.com')
-                self.assertEqual(archive.title, 'ジにてこちら')
-
-            def fake_request_archive(_):
-                singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
-                r = dict(
-                    content=f'<html>test readability content</html>',
-                    textContent='<html>test readability textContent</html>',
-                )
-                s = b'screenshot data'
-                return singlefile, r, s
-
-            with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
-                archive = do_archive('example.com')
-                self.assertEqual(archive.title, 'some title')
-
-            def fake_request_archive(_):
-                singlefile = '<html></html>'
-                r = dict(
-                    content=f'<html>missing a title</html>',
-                    textContent='',
-                )
-                s = b'screenshot data'
-                return singlefile, r, s
-
-            with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
-                archive = do_archive('example.com')
-                self.assertIsNone(archive.title)
-
     @wrap_test_db
     def test_refresh_archives(self):
         with wrap_media_directory():
@@ -324,3 +262,69 @@ def test_get_domains(test_session, archive_factory):
     archive3.delete()
     test_session.commit()
     assert [i['domain'] for i in get_domains()] == []
+
+
+def test_new_archive(test_session, fake_now):
+    with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
+        fake_now(datetime(2000, 1, 1))
+        archive1 = do_archive('https://example.com')
+        # Everything is filled out.
+        assert isinstance(archive1, Archive)
+        assert archive1.archive_datetime is not None
+        assert isinstance(archive1.singlefile_path, MediaPath)
+        assert isinstance(archive1.readability_path, MediaPath)
+        assert isinstance(archive1.readability_txt_path, MediaPath)
+        assert isinstance(archive1.screenshot_path, MediaPath)
+        assert archive1.title == 'ジにてこちら'
+        assert archive1.url is not None
+        assert archive1.domain is not None
+
+        # The actual files were dumped and read correctly.
+        with open(archive1.singlefile_path.path) as fh:
+            assert fh.read() == '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
+        with open(archive1.readability_path.path) as fh:
+            assert fh.read() == '<html>test readability content</html>'
+        with open(archive1.readability_txt_path.path) as fh:
+            assert fh.read() == '<html>test readability textContent</html>'
+        with open(archive1.readability_json_path.path) as fh:
+            assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'}
+
+        fake_now(datetime(2000, 1, 2))
+        archive2 = do_archive('https://example.com')
+        # Domain is reused.
+        assert archive1.domain == archive2.domain
+
+
+def test_get_title_from_html(test_session, fake_now):
+    fake_now(datetime(2000, 1, 1))
+    with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
+        archive = do_archive('example.com')
+        assert archive.title == 'ジにてこちら'
+
+    def fake_request_archive(_):
+        singlefile = '<html>\ntest single-file\nジにてこちら\n<title>some title</title></html>'
+        r = dict(
+            content=f'<html>test readability content</html>',
+            textContent='<html>test readability textContent</html>',
+        )
+        s = b'screenshot data'
+        return singlefile, r, s
+
+    fake_now(datetime(2000, 1, 2))
+    with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
+        archive = do_archive('example.com')
+        assert archive.title == 'some title'
+
+    def fake_request_archive(_):
+        singlefile = '<html></html>'
+        r = dict(
+            content=f'<html>missing a title</html>',
+            textContent='',
+        )
+        s = b'screenshot data'
+        return singlefile, r, s
+
+    fake_now(datetime(2000, 1, 3))
+    with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
+        archive = do_archive('example.com')
+        assert archive.title is None
diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py
index 8dc60bcd..648a610c 100644
--- a/wrolpi/downloader.py
+++ b/wrolpi/downloader.py
@@ -13,7 +13,7 @@ from sqlalchemy import Column, Integer, String, Text
 from sqlalchemy.dialects.postgresql import JSONB
 from sqlalchemy.orm import Session
 
-from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check
+from wrolpi.common import Base, ModelHelper, logger, iterify, wrol_mode_check, zig_zag
 from wrolpi.dates import TZDateTime, now, Seconds, local_timezone
 from wrolpi.db import get_db_session, get_db_curs, get_db_context, optional_session
 from wrolpi.errors import InvalidDownload, UnrecoverableDownloadError
@@ -374,8 +374,10 @@ class DownloadManager:
                         download.fail()
                     elif success:
                         download.complete()
+                        download.next_download = self.get_next_download(download, session)
                     else:
                         download.defer()
+                        download.next_download = self.get_next_download(download, session)
                     session_.commit()
 
             if download_count:
@@ -598,23 +600,44 @@ class DownloadManager:
         return session.query(Download).filter_by(status='pending').all()
 
     @optional_session
-    def get_next_download(self, download: Download, session: Session):
+    def get_next_download(self, download: Download, session: Session) -> Optional[datetime]:
+        """
+        If the download is "deferred", download soon.  But, slowly increase the time between attempts.
+
+        If the download is "complete" and the download has a frequency, schedule the download in it's next iteration.
+        (Next week, month, etc.)
+        """
         if download.status == 'deferred':
             # Increase next_download slowly at first, then by large gaps later.  The largest gap is the download
             # frequency.
             hours = 3 ** (download.attempts or 1)
             seconds = hours * Seconds.hour
-            seconds = min(seconds, download.frequency)
+            if download.frequency:
+                seconds = min(seconds, download.frequency)
             delta = timedelta(seconds=seconds)
             return now() + delta
 
+        if not download.frequency:
+            return None
+        if not download.id:
+            raise ValueError('Cannot get next download when Download is not in DB.')
+
+        freq = download.frequency
+
+        # Keep this Download in it's position within the like-frequency downloads.
+        downloads = [i.id for i in session.query(Download).filter_by(frequency=freq).order_by(Download.id)]
+        index = downloads.index(download.id)
+
         # Download was successful.  Spread the same-frequency downloads out.
         start_date = local_timezone(datetime(2000, 1, 1))
-        elapsed = now() - start_date
-        chunk = elapsed // download.frequency
-        this_frequency = start_date + (download.frequency * chunk)
-        next_frequency = this_frequency + timedelta(seconds=download.frequency)
-        print(f'{this_frequency=} {next_frequency=}')
+        # Weeks/months/etc since start_date.
+        iterations = ((now() - start_date) // freq).total_seconds()
+        start_date = start_date + timedelta(seconds=iterations * freq)
+        end_date = start_date + timedelta(seconds=freq)
+        zagger = zig_zag(start_date, end_date)
+        next_download = [i for i, j in zip(zagger, range(len(downloads)))][index]
+        next_download = local_timezone(next_download)
+        return next_download
 
 
 # The global DownloadManager.  This should be used everywhere!
diff --git a/wrolpi/test/test_downloader.py b/wrolpi/test/test_downloader.py
index d34d501b..57d4f7c7 100644
--- a/wrolpi/test/test_downloader.py
+++ b/wrolpi/test/test_downloader.py
@@ -1,5 +1,5 @@
 from abc import ABC
-from datetime import datetime, timedelta
+from datetime import datetime
 from itertools import zip_longest
 from unittest import mock
 from unittest.mock import MagicMock
@@ -185,7 +185,7 @@ class TestDownloader(TestAPI):
             downloads = list(self.mgr.get_recurring_downloads(session))
             self.assertEqual(len(downloads), 1)
             download = downloads[0]
-            expected = local_timezone(datetime(2020, 1, 1, 1, 0, 0))
+            expected = local_timezone(datetime(2020, 1, 1, 0, 0, 0))
             self.assertEqual(download.next_download, expected)
             self.assertEqual(download.last_successful_download, now)
 
@@ -195,7 +195,7 @@ class TestDownloader(TestAPI):
             self.assertEqual(download.last_successful_download, now)
 
             # Download is due an hour later.
-            mock_now.return_value = local_timezone(datetime(2020, 1, 1, 1, 0, 1))
+            mock_now.return_value = local_timezone(datetime(2020, 1, 1, 2, 0, 1))
             self.mgr.renew_recurring_downloads(session)
             downloads = list(self.mgr.get_new_downloads(session))
             self.assertEqual(len(downloads), 1)
@@ -214,11 +214,12 @@ class TestDownloader(TestAPI):
             self.assertEqual(download.status, 'deferred')
             self.assertEqual(download.last_successful_download, now)
             # Download should be retried after the DEFAULT_RETRY_FREQUENCY.
-            self.assertEqual(download.next_download, expected + timedelta(hours=1, seconds=1))
+            expected = local_timezone(datetime(2020, 1, 1, 3, 0, 1))
+            self.assertEqual(download.next_download, expected)
 
             # Try the download again, it finally succeeds.
             http_downloader.do_download.reset_mock()
-            now = local_timezone(datetime(2020, 1, 1, 3, 0, 0))
+            now = local_timezone(datetime(2020, 1, 1, 4, 0, 1))
             mock_now.return_value = now
             http_downloader.do_download.return_value = True
             self.mgr.renew_recurring_downloads(session)
@@ -227,7 +228,8 @@ class TestDownloader(TestAPI):
             download = session.query(Download).one()
             self.assertEqual(download.status, 'complete')
             self.assertEqual(download.last_successful_download, now)
-            self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0)))
+            # Floats cause slightly wrong date.
+            self.assertEqual(download.next_download, local_timezone(datetime(2020, 1, 1, 4, 0, 0, 997200)))
 
     @wrap_test_db
     @mock.patch('wrolpi.downloader.now', lambda: local_timezone(datetime(2020, 6, 5, 0, 0)))
@@ -356,7 +358,11 @@ def test_get_next_download(test_session, test_download_manager, fake_now):
         result = test_download_manager.get_next_download(download)
         assert result == expected, f'{attempts} != {result}'
 
-    download = Download()
-    download.frequency = DownloadFrequency.weekly
-    assert test_download_manager.get_next_download(download) == local_timezone(datetime(2000, 1, 1))
-
+    d1 = Download(url='https://example.com/1', frequency=DownloadFrequency.weekly)
+    d2 = Download(url='https://example.com/2', frequency=DownloadFrequency.weekly)
+    d3 = Download(url='https://example.com/3', frequency=DownloadFrequency.weekly)
+    test_session.add_all([d1, d2, d3])
+    test_session.commit()
+    assert test_download_manager.get_next_download(d1) == local_timezone(datetime(2000, 1, 1))
+    assert test_download_manager.get_next_download(d2) == local_timezone(datetime(2000, 1, 4, 12))
+    assert test_download_manager.get_next_download(d3) == local_timezone(datetime(2000, 1, 2, 18))

From b0465cb7f26ac4d54849207a671506675e0f2355 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 26 Jan 2022 17:15:35 -0700
Subject: [PATCH 4/8] Restoring pytest fixture so PyCharm can find fixtures.

---
 modules/videos/downloader.py | 2 +-
 wrolpi/conftest.py           | 7 +++----
 2 files changed, 4 insertions(+), 5 deletions(-)

diff --git a/modules/videos/downloader.py b/modules/videos/downloader.py
index 8b8138d8..0e072ef2 100755
--- a/modules/videos/downloader.py
+++ b/modules/videos/downloader.py
@@ -93,7 +93,7 @@ class ChannelDownloader(Downloader, ABC):
 YT_DLP_BIN = pathlib.Path('/usr/local/bin/yt-dlp')  # Location in docker container
 if not YT_DLP_BIN.is_file():
     YT_DLP_BIN = pathlib.Path('/opt/wrolpi/venv/bin/yt-dlp')  # Use virtual environment location
-if not YT_DLP_BIN.is_file():
+if not YT_DLP_BIN.is_file() and not PYTEST:
     logger.error('COULD NOT FIND YT-DLP!!!')
 
 
diff --git a/wrolpi/conftest.py b/wrolpi/conftest.py
index fede0150..c8d72b87 100644
--- a/wrolpi/conftest.py
+++ b/wrolpi/conftest.py
@@ -9,7 +9,6 @@ from unittest import mock
 from uuid import uuid1
 
 import pytest
-import pytest_asyncio
 import yaml
 from sanic_testing.testing import SanicTestClient
 from sqlalchemy.engine import Engine, create_engine
@@ -52,7 +51,7 @@ def test_db() -> Tuple[Engine, Session]:
     return test_engine, session
 
 
-@pytest_asyncio.fixture()
+@pytest.fixture()
 def test_session() -> Session:
     """
     Pytest Fixture to get a test database session.
@@ -75,7 +74,7 @@ def test_session() -> Session:
         conn.close()
 
 
-@pytest_asyncio.fixture(autouse=True)
+@pytest.fixture(autouse=True)
 def test_directory() -> pathlib.Path:
     """
     Overwrite the media directory with a temporary directory.
@@ -87,7 +86,7 @@ def test_directory() -> pathlib.Path:
         yield tmp_path
 
 
-@pytest_asyncio.fixture(autouse=True)
+@pytest.fixture(autouse=True)
 def test_config():
     """
     Create a test config based off the example config.

From 8e47d2a9a0e13659e15ea2c449068eac4246bf4d Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Thu, 27 Jan 2022 20:36:34 -0700
Subject: [PATCH 5/8] Removing spread_channel_downloads function in favor of
 DownloadManager managing next_download.

---
 modules/videos/__init__.py              |  7 ---
 modules/videos/channel/lib.py           | 27 +---------
 modules/videos/channel/test/test_api.py |  3 +-
 modules/videos/conftest.py              |  9 ++--
 modules/videos/models.py                | 13 ++++-
 modules/videos/test/test_downloader.py  | 53 +------------------
 modules/videos/test/test_lib.py         | 69 +------------------------
 7 files changed, 21 insertions(+), 160 deletions(-)

diff --git a/modules/videos/__init__.py b/modules/videos/__init__.py
index 5cd95bbb..8bfe14d6 100644
--- a/modules/videos/__init__.py
+++ b/modules/videos/__init__.py
@@ -1,8 +1 @@
 from modules.videos import downloader
-from wrolpi import before_startup
-
-
-@before_startup
-def startup_spread_channel_downloads():
-    from .channel import lib as channel_lib
-    channel_lib.spread_channel_downloads()
diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index 64e329ec..d8e49790 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -115,33 +115,11 @@ def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]
             yield dict(url=channel.url, frequency=frequency, next_download=next_download)
 
 
-def spread_channel_downloads():
-    """
-    Channels should be downloaded in a manner that is spread out over their frequency.  For example, three channels
-    with a frequency of a week should be downloaded on different days that week.
-    """
-    with get_db_session(commit=True) as session:
-        channels = list(session.query(Channel).filter(
-            Channel.url != None,
-            Channel.url != '',
-            Channel.download_frequency != None
-        ).order_by(Channel.link).all())  # noqa
-
-        url_next_download = _spread_by_frequency(channels)
-
-        for info in url_next_download:
-            url, frequency, next_download = info['url'], info['frequency'], info['next_download']
-            download = download_manager.get_or_create_download(url, session)
-            download.downloader = 'video_channel'
-            download.frequency = frequency
-            download.next_download = next_download
-
-
 @run_after(save_channels_config)
 def update_channel(data, link):
     with get_db_session(commit=True) as session:
         try:
-            channel = session.query(Channel).filter_by(link=link).one()
+            channel: Channel = session.query(Channel).filter_by(link=link).one()
         except NoResultFound:
             raise UnknownChannel()
 
@@ -184,13 +162,10 @@ def update_channel(data, link):
         # Apply the changes now that we've OK'd them
         channel.update(data)
 
-    spread_channel_downloads()
-
     return channel
 
 
 @run_after(save_channels_config)
-@run_after(spread_channel_downloads)
 def create_channel(data: dict, return_dict: bool = True) -> Union[Channel, dict]:
     """
     Create a new Channel.  Check for conflicts with existing Channels.
diff --git a/modules/videos/channel/test/test_api.py b/modules/videos/channel/test/test_api.py
index a610bb92..6e30d309 100644
--- a/modules/videos/channel/test/test_api.py
+++ b/modules/videos/channel/test/test_api.py
@@ -10,7 +10,7 @@ import mock
 import pytest
 
 from modules.videos.api import refresh_queue
-from modules.videos.channel.lib import spread_channel_downloads, delete_channel
+from modules.videos.channel.lib import delete_channel
 from modules.videos.common import get_no_channel_directory
 from modules.videos.lib import upsert_video
 from modules.videos.models import Channel, Video
@@ -380,7 +380,6 @@ class TestVideoAPI(TestAPI):
         location = response.headers['Location']
 
         # No downloads are scheduled.
-        spread_channel_downloads()
         with get_db_session() as session:
             downloads = list(download_manager.get_new_downloads(session))
             self.assertEqual(len(downloads), 0)
diff --git a/modules/videos/conftest.py b/modules/videos/conftest.py
index c4762c1c..b25edee4 100644
--- a/modules/videos/conftest.py
+++ b/modules/videos/conftest.py
@@ -7,11 +7,10 @@ import mock
 import pytest
 from PIL import Image
 
-from modules.videos.channel.lib import spread_channel_downloads
 from modules.videos.downloader import VideoDownloader, ChannelDownloader
 from modules.videos.models import Channel, Video
 from wrolpi.common import sanitize_link
-from wrolpi.downloader import DownloadFrequency, DownloadManager
+from wrolpi.downloader import DownloadFrequency, DownloadManager, Download
 from wrolpi.vars import PROJECT_DIR
 
 
@@ -57,7 +56,7 @@ def channel_factory(test_session, test_directory):
 
 
 @pytest.fixture
-def download_channel(test_session, test_directory) -> Channel:
+def download_channel(test_session, test_directory, video_download_manager) -> Channel:
     """
     Get a test Channel that has a download frequency.
     """
@@ -69,9 +68,9 @@ def download_channel(test_session, test_directory) -> Channel:
         download_frequency=DownloadFrequency.weekly,
         link='downloadchannel',
     )
-    test_session.add(channel)
+    download = Download(url=channel.url, downloader='video_channel', frequency=channel.download_frequency)
+    test_session.add_all([channel, download])
     test_session.commit()
-    spread_channel_downloads()
     return channel
 
 
diff --git a/modules/videos/models.py b/modules/videos/models.py
index b9ca0190..68ca22c1 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -11,7 +11,7 @@ from sqlalchemy.orm.collections import InstrumentedList
 from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directory
 from wrolpi.dates import now, TZDateTime
 from wrolpi.db import get_db_curs
-from wrolpi.downloader import Download
+from wrolpi.downloader import Download, download_manager
 from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory
 from wrolpi.media_path import MediaPathType, MediaPath
 
@@ -302,9 +302,20 @@ class Channel(ModelHelper, Base):
         session.delete(self)
 
     def update(self, data: dict):
+        """
+        Update the values of this Channel.  Will also update the Channel's Download, if it has one.
+        """
         for key, value in data.items():
             setattr(self, key, value)
 
+        if not self.url:
+            return
+        download = self.get_download()
+        if download:
+            session = Session.object_session(self)
+            download.frequency = self.download_frequency
+            download.next_download = download_manager.get_next_download(download, session)
+
     def config_view(self) -> dict:
         """
         Retrieve the data about this Channel that should be stored in a config file.
diff --git a/modules/videos/test/test_downloader.py b/modules/videos/test/test_downloader.py
index 7aa2d90b..60684bde 100644
--- a/modules/videos/test/test_downloader.py
+++ b/modules/videos/test/test_downloader.py
@@ -1,19 +1,17 @@
 import pathlib
 from copy import copy
-from datetime import datetime
 from unittest import mock
 
 import pytest
 from yt_dlp.utils import UnsupportedError
 
-from modules.videos.channel.lib import spread_channel_downloads, download_channel
+from modules.videos.channel.lib import download_channel
 from modules.videos.downloader import find_all_missing_videos, VideoDownloader, \
     ChannelDownloader, get_or_create_channel
 from modules.videos.models import Channel, Video
 from modules.videos.test.common import create_channel_structure
-from wrolpi.dates import local_timezone
 from wrolpi.db import get_db_session, get_db_context
-from wrolpi.downloader import DownloadManager, Download, DownloadFrequency
+from wrolpi.downloader import DownloadManager, Download
 from wrolpi.errors import InvalidDownload
 from wrolpi.test.common import wrap_test_db, TestAPI
 from wrolpi.test.test_downloader import HTTPDownloader
@@ -286,53 +284,6 @@ class TestVideosDownloaders(TestAPI):
             download = self.mgr.get_downloads(session)[0]
             assert download.status == 'failed'
 
-    @wrap_test_db
-    @mock.patch('modules.videos.channel.lib.today', lambda: local_timezone(datetime(2020, 1, 1, 0, 0, 0)))
-    def test_spread_channel_downloads(self):
-        with get_db_session(commit=True) as session:
-            channel1 = Channel(name='channel1', link='channel1', url='https://example.com/1',
-                               download_frequency=DownloadFrequency.weekly)
-            channel2 = Channel(name='channel2', link='channel2', url='https://example.com/2',
-                               download_frequency=DownloadFrequency.weekly)
-            session.add_all([channel1, channel2])
-
-        def check_frequencies(expected):
-            with get_db_session(commit=True) as session:
-                downloads = list(session.query(Download).all())
-                self.assertLength(downloads, expected)
-                for download in downloads:
-                    next_download = expected[download.url]
-                    self.assertEqual(next_download, str(download.next_download),
-                                     f'{download.url} frequency {next_download} != {download.next_download}')
-
-        # Spread the downloads out.
-        spread_channel_downloads()
-
-        expected = {
-            'https://example.com/1': '2020-01-08 00:00:00-07:00',
-            'https://example.com/2': '2020-01-11 12:00:00-07:00',
-        }
-        check_frequencies(expected)
-
-        with get_db_session(commit=True) as session:
-            channel3 = Channel(name='channel3', link='channel3', url='https://example.com/3',
-                               download_frequency=DownloadFrequency.daily)
-            channel4 = Channel(name='channel4', link='channel4', url='https://example.com/4',
-                               download_frequency=DownloadFrequency.weekly)
-            # 5 and 6 should be ignored.
-            channel5 = Channel(name='channel5', link='channel5', url='https://example.com/5')
-            channel6 = Channel(name='channel6', link='channel6', download_frequency=DownloadFrequency.daily)
-            session.add_all([channel3, channel4, channel5, channel6])
-
-        spread_channel_downloads()
-        expected = {
-            'https://example.com/1': '2020-01-08 00:00:00-07:00',
-            'https://example.com/2': '2020-01-10 08:00:00-07:00',
-            'https://example.com/3': '2020-01-02 00:00:00-07:00',
-            'https://example.com/4': '2020-01-12 16:00:00-07:00',
-        }
-        check_frequencies(expected)
-
     @wrap_test_db
     def test_get_or_create_channel(self):
         """
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index b16a66f5..bdf35d3b 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -7,80 +7,13 @@ from unittest import mock
 import pytest
 import sqlalchemy
 
-from modules.videos.channel import lib
 from modules.videos.common import apply_info_json
 from modules.videos.lib import validate_videos, parse_video_file_name, upsert_video
-from modules.videos.models import Channel, Video
+from modules.videos.models import Video
 from modules.videos.video.lib import video_search
 from wrolpi.dates import local_timezone
-from wrolpi.downloader import DownloadFrequency
 from wrolpi.vars import PROJECT_DIR
 
-daily, weekly, days30 = DownloadFrequency.daily, DownloadFrequency.weekly, DownloadFrequency.days30
-
-
-@pytest.mark.parametrize(
-    'channels,expected',
-    [
-        ([Channel(download_frequency=daily, url='https://example.com/1')],
-         [('https://example.com/1', daily, '2000-01-02 00:00:00-07:00')]
-         ),
-        # Several different download frequencies.
-        ([
-             Channel(download_frequency=daily, url='https://example.com/1'),
-             Channel(download_frequency=daily, url='https://example.com/2'),
-             Channel(download_frequency=daily, url='https://example.com/3'),
-             Channel(download_frequency=weekly, url='https://example.com/4'),
-             Channel(download_frequency=weekly, url='https://example.com/5'),
-             Channel(download_frequency=days30, url='https://example.com/6'),
-         ],
-         [
-             ('https://example.com/1', daily, '2000-01-02 00:00:00-07:00'),
-             ('https://example.com/2', daily, '2000-01-02 08:00:00-07:00'),
-             ('https://example.com/3', daily, '2000-01-02 16:00:00-07:00'),
-             ('https://example.com/4', weekly, '2000-01-08 00:00:00-07:00'),
-             ('https://example.com/5', weekly, '2000-01-11 12:00:00-07:00'),
-             ('https://example.com/6', days30, '2000-01-31 00:00:00-07:00'),
-         ]
-        ),
-        # Multiple downloads in a day.
-        ([Channel(download_frequency=daily, url=f'https://example.com/{i}') for i in range(1, 3)],
-         [
-             ('https://example.com/1', daily, '2000-01-02 00:00:00-07:00'),
-             ('https://example.com/2', daily, '2000-01-02 12:00:00-07:00'),
-         ]
-         ),
-        # More downloads in a week than days in a week.
-        ([Channel(download_frequency=weekly, url=f'https://example.com/{i}') for i in range(1, 9)],
-         [
-             ('https://example.com/1', weekly, '2000-01-08 00:00:00-07:00'),
-             ('https://example.com/2', weekly, '2000-01-08 21:00:00-07:00'),
-             ('https://example.com/3', weekly, '2000-01-09 18:00:00-07:00'),
-             ('https://example.com/4', weekly, '2000-01-10 15:00:00-07:00'),
-             ('https://example.com/5', weekly, '2000-01-11 12:00:00-07:00'),
-             ('https://example.com/6', weekly, '2000-01-12 09:00:00-07:00'),
-             ('https://example.com/7', weekly, '2000-01-13 06:00:00-07:00'),
-             ('https://example.com/8', weekly, '2000-01-14 03:00:00-07:00'),
-         ]
-         ),
-        # Any Channel without a URL AND frequency is ignored.
-        ([
-             Channel(download_frequency=daily, url='https://example.com/1'),
-             Channel(download_frequency=daily),
-             Channel(url='this should be ignored'),
-         ],
-         [('https://example.com/1', daily, '2000-01-02 00:00:00-07:00')]
-        ),
-    ]
-)
-@mock.patch('modules.videos.channel.lib.today', lambda: local_timezone(datetime(2000, 1, 1, 0, 0, 0)))
-def test_spread_by_frequency(channels, expected):
-    result = lib._spread_by_frequency(channels)
-    for result, (expected_url, expected_frequency, expected_dt) in zip(result, expected):
-        assert result['url'] == expected_url
-        assert result['frequency'] == expected_frequency
-        assert str(result['next_download']) == expected_dt
-
 
 def test_search_censored_videos(test_session, simple_channel):
     for i in map(str, range(50)):

From 8b0196ede3324cfe1fe7f25cb0a22a976f2e247b Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Thu, 27 Jan 2022 20:47:52 -0700
Subject: [PATCH 6/8] Using new methods in download_channel.

---
 modules/videos/channel/lib.py | 20 +++++++++++---------
 modules/videos/models.py      |  4 ++--
 2 files changed, 13 insertions(+), 11 deletions(-)

diff --git a/modules/videos/channel/lib.py b/modules/videos/channel/lib.py
index d8e49790..2bf3059c 100644
--- a/modules/videos/channel/lib.py
+++ b/modules/videos/channel/lib.py
@@ -2,6 +2,7 @@ from collections import defaultdict
 from datetime import timedelta
 from typing import List, Dict, Union, Generator
 
+from sqlalchemy.orm import Session
 from sqlalchemy.orm.exc import NoResultFound
 
 from wrolpi.common import sanitize_link, run_after, get_relative_to_media_directory, make_media_directory, logger
@@ -116,7 +117,8 @@ def _spread_by_frequency(channels: List[Channel]) -> Generator[Dict, None, None]
 
 
 @run_after(save_channels_config)
-def update_channel(data, link):
+def update_channel(data: dict, link: str) -> Channel:
+    """Update a Channel's DB record"""
     with get_db_session(commit=True) as session:
         try:
             channel: Channel = session.query(Channel).filter_by(link=link).one()
@@ -217,12 +219,12 @@ def delete_channel(link):
 
 
 def download_channel(link: str):
-    """
-    Create a Download record for a Channel's entire catalog.
-    """
+    """Create a Download record for a Channel's entire catalog.  Start downloading."""
     channel = get_channel(link=link, return_dict=False)
-    with get_db_session(commit=True) as session:
-        if not download_manager.get_download(session, channel.url):
-            raise InvalidDownload(f'Channel {channel.name} does not have a download!')
-
-        download_manager.create_download(channel.url, session, downloader='video_channel')
+    session = Session.object_session(channel)
+    download = channel.get_download()
+    if not download:
+        raise InvalidDownload(f'Channel {channel.name} does not have a download!')
+    download.renew(reset_attempts=True)
+    session.commit()
+    download_manager.start_downloads()
diff --git a/modules/videos/models.py b/modules/videos/models.py
index 68ca22c1..36a597e5 100644
--- a/modules/videos/models.py
+++ b/modules/videos/models.py
@@ -12,7 +12,7 @@ from wrolpi.common import Base, tsvector, ModelHelper, logger, get_media_directo
 from wrolpi.dates import now, TZDateTime
 from wrolpi.db import get_db_curs
 from wrolpi.downloader import Download, download_manager
-from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory
+from wrolpi.errors import UnknownVideo, UnknownFile, UnknownDirectory, InvalidDownload
 from wrolpi.media_path import MediaPathType, MediaPath
 
 logger = logger.getChild(__name__)
@@ -345,7 +345,7 @@ class Channel(ModelHelper, Base):
         Get the Download row for this Channel.  If there isn't a Download, return None.
         """
         if not self.url:
-            raise ValueError(f'Channel {self.name} does not have a URL to download!')
+            raise InvalidDownload(f'Channel {self.name} does not have a URL to download!')
 
         session = Session.object_session(self)
         download = session.query(Download).filter_by(url=self.url).one_or_none()

From 30b235a04e87340ff3cef70bf36f49d4bb687ad8 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Thu, 27 Jan 2022 21:06:30 -0700
Subject: [PATCH 7/8] Using greedy regex match on source_id.

---
 modules/videos/lib.py           | 2 +-
 modules/videos/test/test_lib.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index 053bf500..d39263d0 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -391,7 +391,7 @@ async def get_statistics():
     return ret
 
 
-NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.+?)_)?(.*)\.'
+NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.+)_)?(.*)\.'
                          r'(jpg|webp|flv|mp4|part|info\.json|description|webm|..\.srt|..\.vtt)')
 
 
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index bdf35d3b..84db1f38 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -170,6 +170,8 @@ def test_validate_video_exception(test_session, simple_channel, video_factory):
     ('foo .mp4', (None, None, None, 'foo')),
     ('NA_20000303_vp91w5_Bob&apos;s Pancakes.mp4', (None, '20000303', 'vp91w5', 'Bob&apos;s Pancakes')),
     ('NA_NA_vp91w5_Bob&apos;s Pancakes.mp4', (None, None, 'vp91w5', 'Bob&apos;s Pancakes')),
+    ('Learning Self-Reliance_20170529_p_MzsCFkUPU_Beekeeping 2017 Part 6 - Merging Hives.mp4',
+     ('Learning Self-Reliance', '20170529', 'p_MzsCFkUPU', 'Beekeeping 2017 Part 6 - Merging Hives'))
 ])
 def test_parse_video_file_name(file_name, expected):
     """

From 83b9e3fc8faf4d050c9f2de6c06ced16edfadc94 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Thu, 27 Jan 2022 21:09:22 -0700
Subject: [PATCH 8/8] Adding more video name parse tests.  Using limiting
 match.

---
 modules/videos/lib.py           | 2 +-
 modules/videos/test/test_lib.py | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/modules/videos/lib.py b/modules/videos/lib.py
index d39263d0..499bd484 100644
--- a/modules/videos/lib.py
+++ b/modules/videos/lib.py
@@ -391,7 +391,7 @@ async def get_statistics():
     return ret
 
 
-NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.+)_)?(.*)\.'
+NAME_PARSER = re.compile(r'(.*?)_((?:\d+?)|(?:NA))_(?:(.{5,15})_)?(.*)\.'
                          r'(jpg|webp|flv|mp4|part|info\.json|description|webm|..\.srt|..\.vtt)')
 
 
diff --git a/modules/videos/test/test_lib.py b/modules/videos/test/test_lib.py
index 84db1f38..694382bc 100644
--- a/modules/videos/test/test_lib.py
+++ b/modules/videos/test/test_lib.py
@@ -171,7 +171,11 @@ def test_validate_video_exception(test_session, simple_channel, video_factory):
     ('NA_20000303_vp91w5_Bob&apos;s Pancakes.mp4', (None, '20000303', 'vp91w5', 'Bob&apos;s Pancakes')),
     ('NA_NA_vp91w5_Bob&apos;s Pancakes.mp4', (None, None, 'vp91w5', 'Bob&apos;s Pancakes')),
     ('Learning Self-Reliance_20170529_p_MzsCFkUPU_Beekeeping 2017 Part 6 - Merging Hives.mp4',
-     ('Learning Self-Reliance', '20170529', 'p_MzsCFkUPU', 'Beekeeping 2017 Part 6 - Merging Hives'))
+     ('Learning Self-Reliance', '20170529', 'p_MzsCFkUPU', 'Beekeeping 2017 Part 6 - Merging Hives')),
+    ('Learning Self-Reliance_20170529_p_Mzs_Beekeeping 2017 Part 6 - Merging Hives.mp4',
+     ('Learning Self-Reliance', '20170529', 'p_Mzs', 'Beekeeping 2017 Part 6 - Merging Hives')),
+    ('Learning Self-Reliance_20170529_p_Beekeeping 2017 Part 6 - Merging Hives.mp4',
+     ('Learning Self-Reliance', '20170529', None, 'p_Beekeeping 2017 Part 6 - Merging Hives')),
 ])
 def test_parse_video_file_name(file_name, expected):
     """