From c5538222ef62a38f78eec3151a1ddeff179944f7 Mon Sep 17 00:00:00 2001
From: Roland <roland@learningselfreliance.com>
Date: Wed, 2 Nov 2022 14:37:57 -0600
Subject: [PATCH] Handling when a Singlefile is found that was not created by
 WROLPi.

---
 modules/archive/conftest.py      | 26 +++++++++++
 modules/archive/lib.py           | 20 +++++++-
 modules/archive/models.py        | 80 ++++++++++++++++++--------------
 modules/archive/test/test_lib.py | 17 +++++++
 4 files changed, 107 insertions(+), 36 deletions(-)

diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py
index ddc3e987..6c2e8f37 100644
--- a/modules/archive/conftest.py
+++ b/modules/archive/conftest.py
@@ -1,3 +1,4 @@
+import copy
 import datetime
 import json
 import pathlib
@@ -84,3 +85,28 @@ def archive_factory(test_session, archive_directory, make_files_structure):
         return archive
 
     return _
+
+
+SINGLEFILE_CONTENTS = '''<html><!--
+ Page saved with SingleFile 
+ url: https://example.com 
+ saved date: Thu May 12 2022 00:38:02 GMT+0000 (Coordinated Universal Time)
+--><head><meta charset="utf-8">
+
+<meta name="msapplication-TileColor" content="#2d89ef">
+<meta name="theme-color" content="#000000">
+
+<title>the title</title>
+
+<body>
+some body contents
+</body>
+
+</html>
+'''
+
+
+@pytest.fixture
+def singlefile_contents() -> str:
+    """Return a short HTML string that contains an example Singlefile file."""
+    return copy.deepcopy(SINGLEFILE_CONTENTS)
diff --git a/modules/archive/lib.py b/modules/archive/lib.py
index 2e7a6077..453828ea 100644
--- a/modules/archive/lib.py
+++ b/modules/archive/lib.py
@@ -488,13 +488,29 @@ def is_archive_file(path: pathlib.Path) -> bool:
     return path.is_file() and path.suffix.lower() in ARCHIVE_SUFFIXES and bool(ARCHIVE_MATCHER.match(path.name))
 
 
+SINGLEFILE_HEADER = '''<!--
+ Page saved with SingleFile'''
+
+
 def is_singlefile_file(path: pathlib.Path) -> bool:
     """
     Archive singlefile files are expected to start with the following: %Y-%m-%d-%H-%M-%S
     they must end with .html, but never with .readability.html.
     """
-    return path.is_file() and path.suffix.lower() == '.html' and not path.name.endswith('.readability.html') \
-           and bool(ARCHIVE_MATCHER.match(path.name))
+    if not path.is_file() or \
+            path.suffix.lower() != '.html' or \
+            path.name.lower().endswith('.readability.html') or \
+            path.stat().st_size == 0:
+        return False
+    if ARCHIVE_MATCHER.match(path.name):
+        # A file name matching exactly the Archive format from WROLPi should always be an Archive.
+        return True
+    with path.open('rt') as fh:
+        # Lastly, try to read the header of this HTML file.
+        header = fh.read(500)
+        if SINGLEFILE_HEADER in header:
+            return True
+    return False
 
 
 def get_domains():
diff --git a/modules/archive/models.py b/modules/archive/models.py
index e66a650f..17041046 100644
--- a/modules/archive/models.py
+++ b/modules/archive/models.py
@@ -18,6 +18,11 @@ from wrolpi.media_path import MediaPathType
 
 logger = logger.getChild(__name__)
 
+__all__ = ['Archive', 'Domain']
+
+MATCH_URL = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
+MATCH_DATE = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
+
 
 class Archive(Base, ModelHelper):
     __tablename__ = 'archive'
@@ -144,39 +149,37 @@ class Archive(Base, ModelHelper):
         """
         Read the start of the singlefile (if any) and decode the Archive information.
         """
-        match_url = re.compile(r'^\s+?url:\s+?(http.*)', re.MULTILINE)
-        match_date = re.compile(r'^\s+?saved date:\s+?(.*)', re.MULTILINE)
+        if not self.singlefile_path or not self.singlefile_file:
+            return
+        path = self.singlefile_file.path if self.singlefile_file else self.singlefile_path
 
-        if self.singlefile_path:
-            with self.singlefile_path.open('rt') as fh:
-                head = fh.read(500)
-                if 'Page saved with SingleFile' not in head:
-                    return
-                try:
-                    if match := match_url.findall(head):
-                        self.url = match[0].strip()
-                except Exception as e:
-                    logger.error(f'Could not get URL from singlefile {self.singlefile_path}', exc_info=e)
-                try:
-                    if match := match_date.findall(head):
-                        dt = match[0].strip()
-                        dt = ' '.join(dt.split(' ')[:5])
-                        # SingleFile uses GMT.
-                        dt = datetime.datetime.strptime(
-                            dt,
-                            '%a %b %d %Y %H:%M:%S'  # Fri Jun 17 2022 19:24:52
-                        ).replace(tzinfo=pytz.timezone('GMT'))
-                        self.archive_datetime = dt
-                except Exception as e:
-                    logger.error(f'Could not get archive date from singlefile {self.singlefile_path}', exc_info=e)
+        with path.open('rt') as fh:
+            head = fh.read(500)
+            if 'Page saved with SingleFile' not in head:
+                return
+            try:
+                if match := MATCH_URL.findall(head):
+                    self.url = match[0].strip()
+            except Exception as e:
+                logger.error(f'Could not get URL from singlefile {path}', exc_info=e)
+            try:
+                if match := MATCH_DATE.findall(head):
+                    dt = match[0].strip()
+                    dt = ' '.join(dt.split(' ')[:5])
+                    # SingleFile uses GMT.
+                    dt = datetime.datetime.strptime(
+                        dt,
+                        '%a %b %d %Y %H:%M:%S'  # Fri Jun 17 2022 19:24:52
+                    ).replace(tzinfo=pytz.timezone('GMT'))
+                    self.archive_datetime = dt
+            except Exception as e:
+                logger.error(f'Could not get archive date from singlefile {path}', exc_info=e)
 
     def read_readability_data(self):
         """Read the Readability JSON file, apply its contents to this record."""
-        from modules.archive.lib import get_or_create_domain, get_title_from_html
-
         readability_json_file = self.readability_json_file
         if not readability_json_file or not readability_json_file.path.is_file():
-            logger.warning(f'Archive must have a readability json file: {self.singlefile_file.path}')
+            logger.warning(f'{self.singlefile_file.path} does not have an info json file')
             return
 
         try:
@@ -187,21 +190,30 @@ class Archive(Base, ModelHelper):
         except Exception as e:
             raise InvalidArchive() from e
 
-        domain = None
-        if url:
-            domain = get_or_create_domain(Session.object_session(self), url)
-
-        if not title:
-            title = get_title_from_html(self.singlefile_path.read_text())
-
         self.url = url
         self.title = title
+
+    def apply_domain(self):
+        """Get the domain from the URL."""
+        from modules.archive.lib import get_or_create_domain
+        domain = None
+        if self.url:
+            domain = get_or_create_domain(Session.object_session(self), self.url)
+        # Clear domain if the URL is missing.
         self.domain_id = domain.id if domain else None
 
+    def apply_title(self):
+        """Get the title from the Singlefile, if its missing."""
+        from modules.archive.lib import get_title_from_html
+        if self.singlefile_path and not self.title:
+            self.title = get_title_from_html(self.singlefile_path.read_text())
+
     def validate(self):
         try:
             self.read_readability_data()
             self.read_singlefile_data()
+            self.apply_domain()
+            self.apply_title()
             self.validated = True
         except Exception as e:
             logger.warning(f'Unable to validate {self}', exc_info=e)
diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py
index f54194da..e9d2b180 100644
--- a/modules/archive/test/test_lib.py
+++ b/modules/archive/test/test_lib.py
@@ -565,6 +565,23 @@ def test_is_archive_file(name, expected, test_directory):
     assert lib.is_archive_file(path) == expected
 
 
+@pytest.mark.parametrize(
+    'name,expected', [
+        ('foo', False),
+        ('2000-01-01-00-00-00_Some NA.html', True),
+        ('2000-01-01-00-00-00_Some NA.readability.html', False),
+        ('2000-01-01-00-00-00_ジにてこちら.html', True),
+        ('2000-01-01-00-00-00_Some Title.readability.txt', False),
+        ('foo.txt', False),
+        ('foo.html', True),
+    ]
+)
+def test_is_singlefile_file(name, expected, make_files_structure, singlefile_contents):
+    """A Singlefile may not be created by WROLPi, an HTML file created by Singlefile can also be an Archive."""
+    path, = make_files_structure({name: singlefile_contents})
+    assert lib.is_singlefile_file(path) == expected
+
+
 @skip_circleci
 def test_migrate_archive_files(test_session, archive_directory, archive_factory):
     """Test that migration of all old Archive formatted files are migrated."""