From 284efedb80674542fa76ecabd82378da851bfafb Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 2 Feb 2022 15:13:47 -0700
Subject: [PATCH 1/5] Including title in archive file names.

---
 modules/archive/lib.py           |  74 +++++++++++-------
 modules/archive/test/test_lib.py | 129 +++++++++++++++++++++++++++----
 2 files changed, 156 insertions(+), 47 deletions(-)

diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 6f8921ff..5da81b64 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -6,7 +6,7 @@ import re
 import subprocess
 import tempfile
 from itertools import groupby
-from typing import Iterator
+from typing import Iterator, Optional
 
 import requests
 from bs4 import BeautifulSoup
@@ -15,7 +15,7 @@ from sqlalchemy.orm import Session
 
 from modules.archive.models import Domain, Archive
 from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
-from wrolpi.dates import now, strptime_ms, strftime_ms, Seconds
+from wrolpi.dates import now, strptime_ms, Seconds, strftime
 from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
 from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
 from wrolpi.vars import DOCKERIZED, PYTEST
@@ -47,18 +47,20 @@ def get_domain_directory(url: str) -> pathlib.Path:
     return directory
 
 
-def get_new_archive_files(url: str):
+def get_new_archive_files(url: str, title: Optional[str]):
     """
     Create a list of archive files using a shared name schema.  Raise an error if any of them exist.
     """
     directory = get_domain_directory(url)
-    dt = strftime_ms(now())
+    dt = now().strftime('%Y-%m-%d-%H-%M-%S')
 
-    singlefile_path = directory / f'{dt}.html'
-    readability_path = directory / f'{dt}-readability.html'
-    readability_txt_path = directory / f'{dt}-readability.txt'
-    readability_json_path = directory / f'{dt}-readability.json'
-    screenshot_path = directory / f'{dt}.png'
+    title = escape_file_name(title or 'NA')
+    prefix = f'{dt}_{title}'
+    singlefile_path = directory / f'{prefix}.html'
+    readability_path = directory / f'{prefix}.readability.html'
+    readability_txt_path = directory / f'{prefix}.readability.txt'
+    readability_json_path = directory / f'{prefix}.readability.json'
+    screenshot_path = directory / f'{prefix}.png'
 
     ret = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path)
 
@@ -196,8 +198,6 @@ def do_archive(url: str) -> Archive:
     record in the DB.  Create Domain/URL if missing.
     """
     logger.info(f'Archiving {url}')
-    singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
-        get_new_archive_files(url)
 
     if DOCKERIZED or PYTEST:
         # Perform the archive in the Archive docker container.  (Typically in the development environment).
@@ -206,6 +206,30 @@ def do_archive(url: str) -> Archive:
         # Perform the archive using locally installed executables.
         singlefile, readability, screenshot = local_archive(url)
 
+    # First try to get the title from Readability.
+    title = readability.get('title') if readability else None
+
+    if not title and singlefile:
+        # Try to get the title ourselves from the HTML.
+        title = get_title_from_html(singlefile, url=url)
+        if readability:
+            # Readability could not find title, lets use ours.
+            readability['title'] = title
+
+    singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \
+        get_new_archive_files(url, title)
+
+    if readability:
+        # Write the readability parts to their own files.  Write what is left after pops to the JSON file.
+        with readability_path.open('wt') as fh:
+            fh.write(readability.pop('content'))
+        with readability_txt_path.open('wt') as fh:
+            readability_txt = readability.pop('textContent')
+            fh.write(readability_txt)
+    else:
+        # No readability was returned, so there are no files.
+        readability_txt = readability_path = readability_txt_path = None
+
     # Store the single-file HTML in its own file.
     with singlefile_path.open('wt') as fh:
         fh.write(singlefile)
@@ -215,27 +239,8 @@ def do_archive(url: str) -> Archive:
     else:
         screenshot_path = None
 
-    # Store the Readability into separate files.  This allows the user to view text-only or html articles.
-    title = None
-    readability_txt = None
-    if readability:
-        title = readability.get('title')
-
-        # Write the readability parts to their own files.  Write what is left after pops to the JSON file.
-        with readability_path.open('wt') as fh:
-            fh.write(readability.pop('content'))
-        with readability_txt_path.open('wt') as fh:
-            readability_txt = readability.pop('textContent')
-            fh.write(readability_txt)
-    else:
-        readability_path = readability_txt_path = None
-
     # Always write a JSON file that contains at least the URL.
     readability = readability or {}
-    # Use the Readability title, or try and extract one from singlefile.
-    if not title and singlefile:
-        title = get_title_from_html(singlefile, url=url)
-        readability['title'] = title
     readability['url'] = url
     with readability_json_path.open('wt') as fh:
         fh.write(json.dumps(readability))
@@ -496,3 +501,12 @@ def search(search_str: str, domain: str, limit: int, offset: int):
     results = get_ranked_models(ranked_ids, Archive)
 
     return results, total
+
+
+# These characters are invalid in Windows or Linux.
+INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
+
+
+def escape_file_name(name: str) -> str:
+    """Replace any invalid characters in a file name with "_"."""
+    return INVALID_FILE_CHARS.sub('', name)
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index e868be84..c20ca5a5 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -3,7 +3,9 @@ import pathlib
 from datetime import datetime
 
 import mock
+import pytest
 
+from modules.archive import lib
 from modules.archive.lib import get_or_create_domain, _refresh_archives, \
     get_new_archive_files, delete_archive, do_archive, get_domains
 from modules.archive.models import Archive, Domain
@@ -34,23 +36,6 @@ class TestArchive(TestAPI):
         super().setUp()
         (pathlib.Path(self.tmp_dir.name) / 'archive').mkdir(exist_ok=True)
 
-    @mock.patch('modules.archive.lib.now', lambda: datetime(2001, 1, 1))
-    def test_get_new_archive_files(self):
-        s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two'))
-        assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html')
-        assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html')
-        assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt')
-        assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
-        assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
-
-        s, r, t, j, c = get_new_archive_files('https://www.example.com/one')
-        # Leading www. is removed.
-        assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html')
-        assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html')
-        assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt')
-        assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json')
-        assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png')
-
     @wrap_test_db
     def test_no_screenshot(self):
         with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)):
@@ -328,3 +313,113 @@ def test_get_title_from_html(test_session, fake_now):
     with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
         archive = do_archive('example.com')
         assert archive.title is None
+
+
+def test_get_new_archive_files(fake_now):
+    """Archive files have a specific format so they are sorted by datetime, and are near each other."""
+    fake_now(datetime(2001, 1, 1))
+    s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two', None))
+    assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_NA.html')
+    assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.html')
+    assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.txt')
+    assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.json')
+    assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_NA.png')
+
+    s, r, t, j, c = get_new_archive_files('https://www.example.com/one', 'Title')
+    # Leading www. is removed.
+    assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_Title.html')
+    assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.html')
+    assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.txt')
+    assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.json')
+    assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_Title.png')
+
+
+def test_title_in_filename(test_session, fake_now, test_directory):
+    """
+    The Archive files have the title in the path.
+    """
+    fake_now(datetime(2000, 1, 1))
+
+    with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()):
+        archive1 = do_archive('example.com')
+
+    assert archive1.title == 'ジにてこちら'
+
+    assert str(archive1.singlefile_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_ジにてこちら.html'
+    assert str(archive1.readability_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_ジにてこちら.readability.html'
+    assert str(archive1.readability_txt_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_ジにてこちら.readability.txt'
+    assert str(archive1.readability_json_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_ジにてこちら.readability.json'
+    assert str(archive1.screenshot_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_ジにてこちら.png'
+
+    def fake_request_archive(_):
+        singlefile = '<html>\ntest single-file\nジにてこちら\n</html>'  # no title in HTML
+        r = dict(
+            # No Title from Readability.
+            content=f'<html>test readability content</html>',
+            textContent='<html>test readability textContent</html>',
+        )
+        s = b'screenshot data'
+        return singlefile, r, s
+
+    with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
+        archive2 = do_archive('example.com')
+
+    assert str(archive2.singlefile_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_NA.html'
+    assert str(archive2.readability_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_NA.readability.html'
+    assert str(archive2.readability_txt_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_NA.readability.txt'
+    assert str(archive2.readability_json_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_NA.readability.json'
+    assert str(archive2.screenshot_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_NA.png'
+
+    def fake_request_archive(_):
+        singlefile = '<html>\ntest single-file\nジにてこちら\n<title>dangerous ;\\//_title</html></html>'
+        r = dict(
+            # No Title from Readability.
+            content=f'<html>test readability content</html>',
+            textContent='<html>test readability textContent</html>',
+        )
+        s = b'screenshot data'
+        return singlefile, r, s
+
+    with mock.patch('modules.archive.lib.request_archive', fake_request_archive):
+        archive3 = do_archive('example.com')
+
+    assert str(archive3.singlefile_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_dangerous ;_title.html'
+    assert str(archive3.readability_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_dangerous ;_title.readability.html'
+    assert str(archive3.readability_txt_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_dangerous ;_title.readability.txt'
+    assert str(archive3.readability_json_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_dangerous ;_title.readability.json'
+    assert str(archive3.screenshot_path.path.relative_to(test_directory)) == \
+           'archive/2000-01-01-00-00-00_dangerous ;_title.png'
+
+
+@pytest.mark.parametrize(
+    'name,expected', [
+        ('', ''),
+        ('foo', 'foo'),
+        ('foo\\', 'foo'),
+        ('foo/', 'foo'),
+        ('foo<', 'foo'),
+        ('foo>', 'foo'),
+        ('foo:', 'foo'),
+        ('foo|', 'foo'),
+        ('foo"', 'foo'),
+        ('foo?', 'foo'),
+        ('foo*', 'foo'),
+        ('foo&', 'foo&'),
+    ]
+)
+def test_escape_file_name(name, expected):
+    assert lib.escape_file_name(name) == expected

From c1baeabe1fbfc88e3131f4ad982c69c29c5bc447 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 2 Feb 2022 15:58:22 -0700
Subject: [PATCH 2/5] Archive files with title in the name can be found.

---
 modules/archive/lib.py           |  45 ++++++----
 modules/archive/test/test_lib.py | 150 +++++++++++++++----------------
 wrolpi/common.py                 |   9 ++
 wrolpi/test/test_common.py       |  23 ++++-
 4 files changed, 131 insertions(+), 96 deletions(-)

diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 5da81b64..84d2531c 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -5,6 +5,7 @@ import pathlib
 import re
 import subprocess
 import tempfile
+from datetime import datetime
 from itertools import groupby
 from typing import Iterator, Optional
 
@@ -14,8 +15,8 @@ from selenium import webdriver
 from sqlalchemy.orm import Session
 
 from modules.archive.models import Domain, Archive
-from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir
-from wrolpi.dates import now, strptime_ms, Seconds, strftime
+from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir, escape_file_name, walk
+from wrolpi.dates import now, Seconds, local_timezone
 from wrolpi.db import get_db_session, get_db_curs, get_ranked_models
 from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive
 from wrolpi.vars import DOCKERIZED, PYTEST
@@ -57,9 +58,9 @@ def get_new_archive_files(url: str, title: Optional[str]):
     title = escape_file_name(title or 'NA')
     prefix = f'{dt}_{title}'
     singlefile_path = directory / f'{prefix}.html'
-    readability_path = directory / f'{prefix}.readability.html'
-    readability_txt_path = directory / f'{prefix}.readability.txt'
-    readability_json_path = directory / f'{prefix}.readability.json'
+    readability_path = directory / f'{prefix}-readability.html'
+    readability_txt_path = directory / f'{prefix}-readability.txt'
+    readability_json_path = directory / f'{prefix}-readability.json'
     screenshot_path = directory / f'{prefix}.png'
 
     ret = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path)
@@ -309,6 +310,13 @@ def delete_archive(archive_id: int):
         archive.delete()
 
 
+def archive_strptime(dt: str) -> datetime:
+    try:
+        return local_timezone(datetime.strptime(dt, '%Y-%m-%d-%H-%M-%S'))
+    except ValueError:
+        return local_timezone(datetime.strptime(dt, '%Y-%m-%d %H:%M:%S'))
+
+
 def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
     """
     Group archive files by their timestamp.
@@ -316,10 +324,10 @@ def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
     # groupby requires the files to be sorted.
     files = sorted(files)
     # Group archive files by their datetime at the beginning of the file.
-    groups = groupby(files, key=lambda i: i.name[:26])
+    groups = groupby(files, key=lambda i: i.name[:19])
     for dt, files in groups:
         try:
-            dt = strptime_ms(dt)
+            dt = archive_strptime(dt)
         except ValueError:
             logger.info(f'Ignoring invalid archives of {dt=}')
             continue
@@ -350,15 +358,22 @@ def group_archive_files(files: Iterator[pathlib.Path]) -> groupby:
 
 
 ARCHIVE_MATCHER = re.compile(r'\d{4}-\d\d-\d\d (\d\d:){2}\d\d\.\d{6}.*$')
+NEW_ARCHIVE_MATCHER = re.compile(r'\d{4}-(\d{2}-){4}\d\d_.*$')
 ARCHIVE_SUFFIXES = {'.txt', '.html', '.json', '.png', '.jpg', '.jpeg'}
 
 
 def is_archive_file(path: pathlib.Path) -> bool:
     """
-    Archive files are expected to start with the following: %Y-%m-%d %H:%M:%S.%f
+    Archive files are expected to start with the following: %Y-%m-%d-%H-%M-%S
     they must have one of the following suffixes: .txt, .html, .json, .png, .jpg, .jpeg
     """
-    return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(ARCHIVE_MATCHER.match(path.name))
+    if not path.is_file():
+        return False
+    if path.suffix.lower() not in ARCHIVE_SUFFIXES:
+        return False
+    if bool(ARCHIVE_MATCHER.match(path.name)) or bool(NEW_ARCHIVE_MATCHER.match(path.name)):
+        return True
+    return True
 
 
 def _refresh_archives():
@@ -371,7 +386,7 @@ def _refresh_archives():
     singlefile_paths = set()
     for domain_directory in filter(lambda i: i.is_dir(), archive_directory.iterdir()):
         logger.debug(f'Refreshing directory: {domain_directory}')
-        archives_files = filter(is_archive_file, domain_directory.iterdir())
+        archives_files = filter(is_archive_file, walk(domain_directory))
         archive_groups = group_archive_files(archives_files)
         archive_count = 0
         for chunk in chunks(archive_groups, 20):
@@ -419,6 +434,7 @@ def upsert_archive(dt: str, files, session: Session):
     """
     singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = files
     # Extract the URL from the JSON.  Fail if this is not possible.
+    print(singlefile_path, readability_json_path)
     try:
         with readability_json_path.open() as fh:
             json_contents = json.load(fh)
@@ -501,12 +517,3 @@ def search(search_str: str, domain: str, limit: int, offset: int):
     results = get_ranked_models(ranked_ids, Archive)
 
     return results, total
-
-
-# These characters are invalid in Windows or Linux.
-INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
-
-
-def escape_file_name(name: str) -> str:
-    """Replace any invalid characters in a file name with "_"."""
-    return INVALID_FILE_CHARS.sub('', name)
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index c20ca5a5..9f4f7f84 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -3,17 +3,15 @@ import pathlib
 from datetime import datetime
 
 import mock
-import pytest
 
-from modules.archive import lib
 from modules.archive.lib import get_or_create_domain, _refresh_archives, \
-    get_new_archive_files, delete_archive, do_archive, get_domains
+    get_new_archive_files, delete_archive, do_archive, get_domains, get_archive_directory, group_archive_files
 from modules.archive.models import Archive, Domain
-from wrolpi.common import get_media_directory
+from wrolpi.dates import local_timezone
 from wrolpi.db import get_db_session
 from wrolpi.media_path import MediaPath
 from wrolpi.root_api import CustomJSONEncoder
-from wrolpi.test.common import TestAPI, wrap_test_db, wrap_media_directory
+from wrolpi.test.common import TestAPI, wrap_test_db
 
 
 def make_fake_request_archive(readability=True, screenshot=True, title=True):
@@ -88,42 +86,6 @@ class TestArchive(TestAPI):
             except ValueError as e:
                 self.assertIn('relative', str(e), f'Relative path error was not raised')
 
-    @wrap_test_db
-    def test_refresh_archives(self):
-        with wrap_media_directory():
-            archive_directory = get_media_directory() / 'archive'
-            archive_directory.mkdir()
-
-            # Make some test files to refresh.
-            example = archive_directory / 'example.com'
-            example.mkdir()
-            (example / '2021-10-05 16:20:10.346823.html').touch()
-            (example / '2021-10-05 16:20:10.346823.png').touch()
-            (example / '2021-10-05 16:20:10.346823-readability.txt').touch()
-            (example / '2021-10-05 16:20:10.346823-readability.json').touch()
-            (example / '2021-10-05 16:20:10.346823-readability.html').touch()
-            with (example / '2021-10-05 16:20:10.346823-readability.json').open('wt') as fh:
-                json.dump({'url': 'foo'}, fh)
-
-            # These should log an error because they are missing the singlefile path.
-            (example / '2021-10-05 16:20:10.346824-readability.html').touch()
-            (example / '2021-10-05 16:20:10.346825-readability.json').touch()
-            # These are not archives and should be ignored.
-            (example / '2021-10-05 16:20:10.346826-something.html').touch()
-            (example / 'random file').touch()
-            (example / '2021-10-05 16:20:10.346827-something.html').mkdir()
-            (example / '2021-10-05 16:20:10 invalid date').touch()
-
-            # This single archive is found.
-            _refresh_archives()
-            with get_db_session() as session:
-                self.assertEqual(session.query(Archive).count(), 1)
-
-            # Running the refresh does not result in a new archive.
-            _refresh_archives()
-            with get_db_session() as session:
-                self.assertEqual(session.query(Archive).count(), 1)
-
 
 def test_archive_refresh_deleted_archive(test_session, archive_directory, archive_factory):
     """
@@ -176,9 +138,7 @@ def test_archive_refresh_deleted_archive(test_session, archive_directory, archiv
 
 
 def test_refresh_archives_fills_contents(test_session, archive_factory, test_client):
-    """
-    Refreshing archives fills in any missing contents.
-    """
+    """Refreshing archives fills in any missing contents."""
     archive1 = archive_factory('example.com', 'https://example.com/one')
     archive2 = archive_factory('example.com', 'https://example.com/one')
     archive3 = archive_factory('example.org')
@@ -320,17 +280,17 @@ def test_get_new_archive_files(fake_now):
     fake_now(datetime(2001, 1, 1))
     s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two', None))
     assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_NA.html')
-    assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.html')
-    assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.txt')
-    assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.json')
+    assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_NA-readability.html')
+    assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_NA-readability.txt')
+    assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_NA-readability.json')
     assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_NA.png')
 
     s, r, t, j, c = get_new_archive_files('https://www.example.com/one', 'Title')
     # Leading www. is removed.
     assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_Title.html')
-    assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.html')
-    assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.txt')
-    assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.json')
+    assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_Title-readability.html')
+    assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_Title-readability.txt')
+    assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_Title-readability.json')
     assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_Title.png')
 
 
@@ -348,11 +308,11 @@ def test_title_in_filename(test_session, fake_now, test_directory):
     assert str(archive1.singlefile_path.path.relative_to(test_directory)) == \
            'archive/2000-01-01-00-00-00_ジにてこちら.html'
     assert str(archive1.readability_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_ジにてこちら.readability.html'
+           'archive/2000-01-01-00-00-00_ジにてこちら-readability.html'
     assert str(archive1.readability_txt_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_ジにてこちら.readability.txt'
+           'archive/2000-01-01-00-00-00_ジにてこちら-readability.txt'
     assert str(archive1.readability_json_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_ジにてこちら.readability.json'
+           'archive/2000-01-01-00-00-00_ジにてこちら-readability.json'
     assert str(archive1.screenshot_path.path.relative_to(test_directory)) == \
            'archive/2000-01-01-00-00-00_ジにてこちら.png'
 
@@ -372,11 +332,11 @@ def test_title_in_filename(test_session, fake_now, test_directory):
     assert str(archive2.singlefile_path.path.relative_to(test_directory)) == \
            'archive/2000-01-01-00-00-00_NA.html'
     assert str(archive2.readability_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_NA.readability.html'
+           'archive/2000-01-01-00-00-00_NA-readability.html'
     assert str(archive2.readability_txt_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_NA.readability.txt'
+           'archive/2000-01-01-00-00-00_NA-readability.txt'
     assert str(archive2.readability_json_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_NA.readability.json'
+           'archive/2000-01-01-00-00-00_NA-readability.json'
     assert str(archive2.screenshot_path.path.relative_to(test_directory)) == \
            'archive/2000-01-01-00-00-00_NA.png'
 
@@ -396,30 +356,68 @@ def test_title_in_filename(test_session, fake_now, test_directory):
     assert str(archive3.singlefile_path.path.relative_to(test_directory)) == \
            'archive/2000-01-01-00-00-00_dangerous ;_title.html'
     assert str(archive3.readability_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_dangerous ;_title.readability.html'
+           'archive/2000-01-01-00-00-00_dangerous ;_title-readability.html'
     assert str(archive3.readability_txt_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_dangerous ;_title.readability.txt'
+           'archive/2000-01-01-00-00-00_dangerous ;_title-readability.txt'
     assert str(archive3.readability_json_path.path.relative_to(test_directory)) == \
-           'archive/2000-01-01-00-00-00_dangerous ;_title.readability.json'
+           'archive/2000-01-01-00-00-00_dangerous ;_title-readability.json'
     assert str(archive3.screenshot_path.path.relative_to(test_directory)) == \
            'archive/2000-01-01-00-00-00_dangerous ;_title.png'
 
 
-@pytest.mark.parametrize(
-    'name,expected', [
-        ('', ''),
-        ('foo', 'foo'),
-        ('foo\\', 'foo'),
-        ('foo/', 'foo'),
-        ('foo<', 'foo'),
-        ('foo>', 'foo'),
-        ('foo:', 'foo'),
-        ('foo|', 'foo'),
-        ('foo"', 'foo'),
-        ('foo?', 'foo'),
-        ('foo*', 'foo'),
-        ('foo&', 'foo&'),
+def test_refresh_archives(test_session, test_directory):
+    """Archives can be found and put in the database.  A single Archive will have multiple files."""
+    archive_directory = get_archive_directory()
+
+    # Make some test files to refresh.
+    example = archive_directory / 'example.com'
+    example.mkdir()
+    (example / '2021-10-05 16:20:10.346823.html').touch()
+    (example / '2021-10-05 16:20:10.346823.png').touch()
+    (example / '2021-10-05 16:20:10.346823-readability.txt').touch()
+    (example / '2021-10-05 16:20:10.346823-readability.json').touch()
+    (example / '2021-10-05 16:20:10.346823-readability.html').touch()
+    with (example / '2021-10-05 16:20:10.346823-readability.json').open('wt') as fh:
+        json.dump({'url': 'foo'}, fh)
+
+    # These should log an error because they are missing the singlefile path.
+    (example / '2021-10-05 16:20:10-readability.html').touch()
+    (example / '2021-10-05 16:20:11-readability.json').touch()
+    (example / 'random file').touch()
+
+    # This single archive is found.
+    _refresh_archives()
+    with get_db_session() as session:
+        assert session.query(Archive).count() == 1
+
+    # Running the refresh does not result in a new archive.
+    _refresh_archives()
+    with get_db_session() as session:
+        assert session.query(Archive).count() == 1
+
+    # Archives file format was changed, lets check the new formats are found.
+    (example / '2021-10-05-16-20-10_The Title.html').touch()
+    (example / '2021-10-05-16-20-10_The Title-readability.json').write_text(json.dumps({'url': 'bar'}))
+    _refresh_archives()
+    with get_db_session() as session:
+        assert session.query(Archive).count() == 2
+
+
+def test_group_archive_files(test_directory):
+    files = [
+        pathlib.Path('2021-10-05 16:20:10.346823.html'),
+        pathlib.Path('2021-10-05 16:20:10.346823-readability.json'),
+        pathlib.Path('2000-01-01-00-00-00_Title.html'),
+        pathlib.Path('2000-01-01-00-00-00_Title-readability.json'),
+    ]
+    assert list(group_archive_files(files)) == [
+        (local_timezone(datetime(2000, 1, 1, 0, 0, 0)),
+         (
+             pathlib.Path('2000-01-01-00-00-00_Title.html'), None, None,
+             pathlib.Path('2000-01-01-00-00-00_Title-readability.json'), None,
+         )),
+        (local_timezone(datetime(2021, 10, 5, 16, 20, 10)),
+         (
+             pathlib.Path('2021-10-05 16:20:10.346823.html'), None, None,
+             pathlib.Path('2021-10-05 16:20:10.346823-readability.json'), None)),
     ]
-)
-def test_escape_file_name(name, expected):
-    assert lib.escape_file_name(name) == expected
diff --git a/wrolpi/common.py b/wrolpi/common.py
index 9056e473..320e2796 100644
--- a/wrolpi/common.py
+++ b/wrolpi/common.py
@@ -646,3 +646,12 @@ def walk(path: Path) -> Generator[Path, None, None]:
         yield path
         if path.is_dir():
             yield from walk(path)
+
+
+# These characters are invalid in Windows or Linux.
+INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]')
+
+
+def escape_file_name(name: str) -> str:
+    """Replace any invalid characters in a file name with "_"."""
+    return INVALID_FILE_CHARS.sub('', name)
diff --git a/wrolpi/test/test_common.py b/wrolpi/test/test_common.py
index afc3455e..5b37a51c 100644
--- a/wrolpi/test/test_common.py
+++ b/wrolpi/test/test_common.py
@@ -7,7 +7,8 @@ from decimal import Decimal
 
 import pytest
 
-from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag
+from wrolpi.common import combine_dicts, insert_parameter, date_range, api_param_limiter, chdir, zig_zag, \
+    escape_file_name
 from wrolpi.dates import set_timezone, now
 from wrolpi.errors import InvalidTimezone
 from wrolpi.test.common import build_test_directories, wrap_media_directory
@@ -315,3 +316,23 @@ def test_zig_zag(low, high, expected):
         result = next(zagger)
         assert result == i
         assert low <= result < high
+
+
+@pytest.mark.parametrize(
+    'name,expected', [
+        ('', ''),
+        ('foo', 'foo'),
+        ('foo\\', 'foo'),
+        ('foo/', 'foo'),
+        ('foo<', 'foo'),
+        ('foo>', 'foo'),
+        ('foo:', 'foo'),
+        ('foo|', 'foo'),
+        ('foo"', 'foo'),
+        ('foo?', 'foo'),
+        ('foo*', 'foo'),
+        ('foo&', 'foo&'),
+    ]
+)
+def test_escape_file_name(name, expected):
+    assert escape_file_name(name) == expected

From 2946255fc6471dc226072243304b9e7897cad362 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 2 Feb 2022 16:08:18 -0700
Subject: [PATCH 3/5] Default to false to avoid non-archive files.

---
 modules/archive/lib.py           | 2 +-
 modules/archive/test/test_lib.py | 2 ++
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 84d2531c..8e8fed56 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -373,7 +373,7 @@ def is_archive_file(path: pathlib.Path) -> bool:
         return False
     if bool(ARCHIVE_MATCHER.match(path.name)) or bool(NEW_ARCHIVE_MATCHER.match(path.name)):
         return True
-    return True
+    return False
 
 
 def _refresh_archives():
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index 9f4f7f84..3c7c9bc2 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -384,6 +384,8 @@ def test_refresh_archives(test_session, test_directory):
     (example / '2021-10-05 16:20:10-readability.html').touch()
     (example / '2021-10-05 16:20:11-readability.json').touch()
     (example / 'random file').touch()
+    (example / '2021-10-05 16:20:10.346827-something.html').mkdir()
+    (example / '2021-10-05 16:20:10 no extension').touch()
 
     # This single archive is found.
     _refresh_archives()

From a6a3284b49cab3a79de1f2d1594e88b0f0d10afb Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 2 Feb 2022 16:09:39 -0700
Subject: [PATCH 4/5] Removing accidental print.

---
 modules/archive/lib.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 8e8fed56..1c497214 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -434,7 +434,6 @@ def upsert_archive(dt: str, files, session: Session):
     """
     singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = files
     # Extract the URL from the JSON.  Fail if this is not possible.
-    print(singlefile_path, readability_json_path)
     try:
         with readability_json_path.open() as fh:
             json_contents = json.load(fh)

From 829f5eea08d1faf367b6061aac3dcd6f99488a33 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 2 Feb 2022 16:10:59 -0700
Subject: [PATCH 5/5] Fixing archive counting.

---
 modules/archive/lib.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 1c497214..36c1051f 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -390,14 +390,14 @@ def _refresh_archives():
         archive_groups = group_archive_files(archives_files)
         archive_count = 0
         for chunk in chunks(archive_groups, 20):
-            archive_count += 1
             with get_db_session(commit=True) as session:
                 for dt, files in chunk:
+                    archive_count += 1
                     singlefile_paths.add(str(files[0]))
                     upsert_archive(dt, files, session)
 
         if archive_count:
-            logger.info(f'Inserted/updated {archive_count} archives')
+            logger.info(f'Inserted/updated {archive_count} archives in {domain_directory}')
 
     singlefile_paths = list(singlefile_paths)
     with get_db_curs(commit=True) as curs: