Indexing Archives after downloading.

modules/archive/lib.py

@@ -104,10 +104,8 @@ def get_new_archive_files(url: str, title: Optional[str]) -> ArchiveFiles:
 ARCHIVE_TIMEOUT = Seconds.minute * 10  # Wait at most 10 minutes for response.
 
 
-async def request_archive(url: str):
-    """
-    Send a request to the archive service to archive the URL.
-    """
+async def request_archive(url: str) -> Tuple[str, Optional[str], Optional[str]]:
+    """Send a request to the archive service to archive the URL."""
     logger.info(f'Sending archive request to archive service: {url}')
 
     data = {'url': url}
@@ -133,13 +131,10 @@ async def request_archive(url: str):
     if not readability:
         logger.info(f'Failed to get readability for {url=}')
 
-    if not singlefile:
-        logger.info(f'Failed to get singlefile for {url=}')
-    else:
-        # Decode and decompress.
-        singlefile = base64.b64decode(singlefile)
-        singlefile = gzip.decompress(singlefile)
-        singlefile = singlefile.decode()
+    # Decode and decompress.
+    singlefile = base64.b64decode(singlefile)
+    singlefile = gzip.decompress(singlefile)
+    singlefile = singlefile.decode()
 
     if not screenshot:
         logger.info(f'Failed to get screenshot for {url=}')
@@ -235,7 +230,7 @@ def local_archive(url: str):
 async def do_archive(url: str) -> Archive:
     """
     Perform the real archive request to the archiving service.  Store the resulting data into files.  Create an Archive
-    record in the DB.  Create Domain/URL if missing.
+    record in the DB.  Create Domain if missing.
     """
     logger.info(f'Archiving {url}')
 
@@ -267,14 +262,13 @@ async def do_archive(url: str) -> Archive:
             fh.write(readability_txt)
     else:
         # No readability was returned, so there are no files.
-        readability_txt = archive_files.readability_txt = archive_files.readability = None
+        archive_files.readability_txt = archive_files.readability = None
 
     # Store the single-file HTML in its own file.
-    with archive_files.singlefile.open('wt') as fh:
-        fh.write(singlefile)
+    archive_files.singlefile.write_text(singlefile)
+
     if screenshot:
-        with archive_files.screenshot.open('wb') as fh:
-            fh.write(screenshot)
+        archive_files.screenshot.write_bytes(screenshot)
     else:
         archive_files.screenshot = None
 
@@ -284,16 +278,36 @@ async def do_archive(url: str) -> Archive:
     with archive_files.readability_json.open('wt') as fh:
         fh.write(json.dumps(readability))
 
+    # Create any File models that we can, index them all.
+    singlefile_file = File(path=archive_files.singlefile, model='archive')
+    singlefile_file.do_index()
+    readability_file = None
+    if archive_files.readability:
+        readability_file = File(path=archive_files.readability, associated=True)
+        readability_file.do_index()
+    readability_json_file = None
+    if archive_files.readability_json:
+        readability_json_file = File(path=archive_files.readability_json, associated=True)
+        readability_json_file.do_index()
+    readability_txt_file = None
+    if archive_files.readability_txt:
+        readability_txt_file = File(path=archive_files.readability_txt, associated=True)
+        readability_txt_file.do_index()
+    screenshot_file = None
+    if archive_files.screenshot:
+        screenshot_file = File(path=archive_files.screenshot, associated=True)
+        screenshot_file.do_index()
+
     with get_db_session(commit=True) as session:
         domain = get_or_create_domain(session, url)
         archive = Archive(
             title=title,
             archive_datetime=now(),
-            singlefile_file=File(path=archive_files.singlefile, model='archive'),
-            readability_file=File(path=archive_files.readability) if archive_files.readability else None,
-            readability_json_file=File(path=archive_files.readability_json) if archive_files.readability_json else None,
-            readability_txt_file=File(path=archive_files.readability_txt) if archive_files.readability_txt else None,
-            screenshot_file=File(path=archive_files.screenshot) if archive_files.screenshot else None,
+            singlefile_file=singlefile_file,
+            readability_file=readability_file,
+            readability_json_file=readability_json_file,
+            readability_txt_file=readability_txt_file,
+            screenshot_file=screenshot_file,
             url=url,
             domain_id=domain.id,
         )

modules/archive/test/test_lib.py

@@ -13,7 +13,6 @@ from modules.archive.models import Archive, Domain
 from wrolpi.dates import local_timezone
 from wrolpi.db import get_db_session
 from wrolpi.files.models import File
-from wrolpi.media_path import MediaPath
 from wrolpi.root_api import CustomJSONEncoder
 from wrolpi.test.common import skip_circleci
 
@@ -587,3 +586,27 @@ def test_archive_order(test_session, test_directory, archive_factory):
 
     assert archive2 > archive1
     assert archive1 < archive2
+
+
+@pytest.mark.asyncio
+async def test_archive_download_index(test_session, test_directory, image_file):
+    """An Archive is indexed when it is downloaded."""
+    with mock.patch('modules.archive.lib.request_archive') as mock_request_archive:
+        singlefile = '<html><title>the singlefile</title></html>'
+        readability = dict(
+            content='<html>the readability</html>',
+            textContent='the readability',
+        )
+        mock_request_archive.return_value = (singlefile, readability, image_file.read_bytes())
+        archive = await lib.do_archive('https://example.com')
+
+    assert isinstance(archive, lib.Archive), 'Did not get an archive'
+    assert archive.singlefile_path and archive.singlefile_path.exists() and \
+           archive.singlefile_path.read_text() == singlefile, 'Singlefile was not stored'
+    assert archive.readability_path and archive.readability_path.exists()
+    assert archive.readability_json_path
+    assert archive.readability_txt_path and archive.readability_txt_path.exists()
+    assert archive.readability_txt_file.d_text == '{the,readability}', 'Readability text was not indexed'
+    assert archive.title == 'the singlefile', 'Did not get the title from the singlefile'
+    assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \
+        'Did not store the screenshot'

wrolpi/files/models.py

@@ -106,8 +106,10 @@ class File(ModelHelper, Base):
 
         try:
             if force_index or self.indexed is not True:
+                # Get the indexer on a separate line for debugging.
+                indexer = self.indexer
                 # Only read the contents of the file once.
-                self.a_text, self.b_text, self.c_text, self.d_text = self.indexer.create_index(self)
+                self.a_text, self.b_text, self.c_text, self.d_text = indexer.create_index(self)
                 self.a_text = truncate_object_bytes(self.a_text, MAX_TEXT_FILE_BYTES)
                 self.b_text = truncate_object_bytes(self.b_text, MAX_TEXT_FILE_BYTES)
                 self.c_text = truncate_object_bytes(self.c_text, MAX_TEXT_FILE_BYTES)