Removing leading www. from archive domains.g

modules/archive/lib.py

@@ -13,10 +13,9 @@ from sqlalchemy.orm import Session
 
 from modules.archive.models import URL, Domain, Archive
 from wrolpi.common import get_media_directory, logger, chunks
-from wrolpi.dates import now, strptime_ms
+from wrolpi.dates import now, strptime_ms, strftime_ms
 from wrolpi.db import get_db_session, get_db_curs
 from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive
-from wrolpi.vars import DATETIME_FORMAT_MS
 
 logger = logger.getChild(__name__)
 
@@ -28,8 +27,18 @@ def get_archive_directory() -> pathlib.Path:
 
 
 def extract_domain(url):
+    """
+    Extract the domain from a URL.  Remove leading www.
+
+    >>> extract_domain('https://www.example.com/foo')
+    'example.com'
+    """
     parsed = urlparse(url)
-    return parsed.netloc
+    domain = parsed.netloc
+    if domain.startswith('www.'):
+        # Remove leading www.
+        domain = domain[4:]
+    return domain
 
 
 def get_domain_directory(url: str) -> pathlib.Path:
@@ -52,7 +61,7 @@ def get_new_archive_files(url: str):
     Create a list of archive files using a shared name schema.  Raise an error if any of them exist.
     """
     directory = get_domain_directory(url)
-    dt = now().strftime(DATETIME_FORMAT_MS)
+    dt = strftime_ms(now())
 
     singlefile_path = directory / f'{dt}.html'
     readability_path = directory / f'{dt}-readability.html'

modules/archive/test/test_lib.py

@@ -1,10 +1,11 @@
 import json
 import pathlib
+from datetime import datetime
 
 import mock
 
 from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, delete_url, \
-    _refresh_archives, get_domains
+    _refresh_archives, get_domains, get_new_archive_files
 from modules.archive.models import Archive, URL, Domain
 from wrolpi.common import get_media_directory
 from wrolpi.db import get_db_session
@@ -34,6 +35,23 @@ class TestArchive(TestAPI):
         super().setUp()
         (pathlib.Path(self.tmp_dir.name) / 'archive').mkdir(exist_ok=True)
 
+    @mock.patch('modules.archive.lib.now', lambda: datetime(2001, 1, 1))
+    def test_get_new_archive_files(self):
+        s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two'))
+        assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html')
+        assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html')
+        assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt')
+        assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
+        assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
+
+        s, r, t, j, c = get_new_archive_files('https://www.example.com/one')
+        # Leading www. is removed.
+        assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html')
+        assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html')
+        assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt')
+        assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
+        assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
+
     @wrap_test_db
     def test_new_archive(self):
         with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):