diff --git a/modules/archive/lib.py b/modules/archive/lib.py index a694598a..4c85aadd 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -13,10 +13,9 @@ from sqlalchemy.orm import Session from modules.archive.models import URL, Domain, Archive from wrolpi.common import get_media_directory, logger, chunks -from wrolpi.dates import now, strptime_ms +from wrolpi.dates import now, strptime_ms, strftime_ms from wrolpi.db import get_db_session, get_db_curs from wrolpi.errors import InvalidDomain, UnknownURL, PendingArchive, InvalidArchive -from wrolpi.vars import DATETIME_FORMAT_MS logger = logger.getChild(__name__) @@ -28,8 +27,18 @@ def get_archive_directory() -> pathlib.Path: def extract_domain(url): + """ + Extract the domain from a URL. Remove leading www. + + >>> extract_domain('https://www.example.com/foo') + 'example.com' + """ parsed = urlparse(url) - return parsed.netloc + domain = parsed.netloc + if domain.startswith('www.'): + # Remove leading www. + domain = domain[4:] + return domain def get_domain_directory(url: str) -> pathlib.Path: @@ -52,7 +61,7 @@ def get_new_archive_files(url: str): Create a list of archive files using a shared name schema. Raise an error if any of them exist. """ directory = get_domain_directory(url) - dt = now().strftime(DATETIME_FORMAT_MS) + dt = strftime_ms(now()) singlefile_path = directory / f'{dt}.html' readability_path = directory / f'{dt}-readability.html' diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 5a3cc959..ff68f8c0 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -1,10 +1,11 @@ import json import pathlib +from datetime import datetime import mock from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, delete_url, \ - _refresh_archives, get_domains + _refresh_archives, get_domains, get_new_archive_files from modules.archive.models import Archive, URL, Domain from wrolpi.common import get_media_directory from wrolpi.db import get_db_session @@ -34,6 +35,23 @@ class TestArchive(TestAPI): super().setUp() (pathlib.Path(self.tmp_dir.name) / 'archive').mkdir(exist_ok=True) + @mock.patch('modules.archive.lib.now', lambda: datetime(2001, 1, 1)) + def test_get_new_archive_files(self): + s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two')) + assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html') + assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html') + assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt') + assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json') + assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png') + + s, r, t, j, c = get_new_archive_files('https://www.example.com/one') + # Leading www. is removed. + assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html') + assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html') + assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt') + assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json') + assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png') + @wrap_test_db def test_new_archive(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):