Handling when archives have been manually deleted.

modules/archive/conftest.py

@@ -0,0 +1,85 @@
+import datetime
+import json
+import pathlib
+from uuid import uuid4
+
+import pytest
+
+from modules.archive.models import Archive, URL, Domain
+from wrolpi.dates import strftime_ms
+
+
+@pytest.fixture
+def archive_directory(test_directory) -> pathlib.Path:
+    path = test_directory / 'archive'
+    path.mkdir()
+    return path
+
+
+@pytest.fixture
+def archive_factory(test_session, archive_directory):
+    def time_generator():
+        timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0)
+        while True:
+            timestamp += datetime.timedelta(seconds=1)
+            yield timestamp
+
+    now = time_generator()
+
+    def _(domain: str = None, url: str = None) -> Archive:
+        title = str(uuid4())
+        url = url or f'https://example.com/{title}'
+        domain_dir = domain or title
+        domain_dir = archive_directory / domain_dir
+        domain_dir.mkdir(exist_ok=True)
+
+        timestamp = strftime_ms(next(now))
+
+        singlefile_path = domain_dir / f'{timestamp}.html'
+        screenshot_path = domain_dir / f'{timestamp}.png'
+        readability_path = domain_dir / f'{timestamp}-readability.html'
+        readability_txt_path = domain_dir / f'{timestamp}-readability.txt'
+        readability_json_path = domain_dir / f'{timestamp}-readability.json'
+
+        with readability_json_path.open('wt') as fh:
+            json.dump({'url': url}, fh)
+
+        if domain:
+            domain = test_session.query(Domain).filter_by(domain=domain).one_or_none()
+
+        if not domain:
+            domain = Domain(
+                domain=domain,
+                directory=domain_dir,
+            )
+            test_session.add(domain)
+            test_session.commit()
+
+        url_ = test_session.query(URL).filter_by(url=url).one_or_none()
+        if not url_:
+            url_ = URL(
+                url=url,
+                domain_id=domain.id,
+            )
+            test_session.add(url_)
+            test_session.commit()
+
+        archive = Archive(
+            readability_json_path=readability_json_path,
+            readability_path=readability_path,
+            readability_txt_path=readability_txt_path,
+            screenshot_path=screenshot_path,
+            singlefile_path=singlefile_path,
+            status='complete',
+            title=title,
+            url_id=url_.id,
+        )
+        test_session.add(archive)
+        test_session.commit()
+
+        for path in archive.my_paths():
+            path.touch()
+
+        return archive
+
+    return _

modules/archive/lib.py

@@ -367,9 +367,12 @@ def is_archive_file(path: pathlib.Path) -> bool:
 
 def _refresh_archives():
     """
-    Search the Archives directory for archive files, update the database if new files are found.
+    Search the Archives directory for archive files, update the database if new files are found.  Remove any orphan
+    URLs or Domains.
     """
     archive_directory = get_archive_directory()
+
+    singlefile_paths = set()
     for domain_directory in filter(lambda i: i.is_dir(), archive_directory.iterdir()):
         logger.debug(f'Refreshing directory: {domain_directory}')
         archives_files = filter(is_archive_file, domain_directory.iterdir())
@@ -379,19 +382,24 @@ def _refresh_archives():
             archive_count += 1
             with get_db_session(commit=True) as session:
                 for dt, files in chunk:
-                    print('_refresh_archives', dt)
+                    singlefile_paths.add(str(files[0]))
                     upsert_archive(dt, files, session)
 
         if archive_count:
             logger.info(f'Inserted/updated {archive_count} archives')
 
-    cleanup_domains_urls()
+    with get_db_session(commit=True) as session:
+        # Check that each URLs most recent Archive exists, if it does not use the next most recent.
+        urls = session.query(URL).all()
+        for url in urls:
+            if not url.latest or not url.latest.singlefile_path or not url.latest.singlefile_path.path.exists():
+                url.update_latest()
+                session.commit()
 
+    singlefile_paths = list(singlefile_paths)
+    with get_db_curs(commit=True) as curs:
+        curs.execute('DELETE FROM archive WHERE singlefile_path != ALL(%s)', (singlefile_paths,))
 
-def cleanup_domains_urls():
-    """
-    Delete any URLs/Domains without Archives.
-    """
     with get_db_curs(commit=True) as curs:
         stmt = '''
             DELETE FROM url WHERE id NOT IN (

modules/archive/models.py

@@ -1,5 +1,7 @@
+from datetime import datetime
 from typing import Generator
 
+import pytz
 from sqlalchemy import Column, Integer, String, ForeignKey
 from sqlalchemy.orm import relationship
 from sqlalchemy.orm.collections import InstrumentedList
@@ -75,6 +77,17 @@ class URL(Base, ModelHelper):
         d['archives'] = [i.dict() for i in self.archives]
         return d
 
+    def update_latest(self):
+        """
+        Set `latest` to the most recent Archive of this URL whose `singlefile` exists.
+        """
+        self.latest_id = None
+        latest_datetime = datetime(2, 1, 1, 0, 0, 0).astimezone(tz=pytz.UTC)
+        for archive in self.archives:
+            if archive.singlefile_path and archive.singlefile_path.path.exists() and \
+                    archive.archive_datetime >= latest_datetime:
+                self.latest_id = archive.id
+
 
 class Domain(Base, ModelHelper):
     __tablename__ = 'domains'  # plural to avoid conflict

modules/archive/test/test_lib.py

@@ -5,7 +5,7 @@ from datetime import datetime
 import mock
 
 from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, delete_url, \
-    _refresh_archives, get_domains, get_new_archive_files
+    _refresh_archives, get_new_archive_files
 from modules.archive.models import Archive, URL, Domain
 from wrolpi.common import get_media_directory
 from wrolpi.db import get_db_session
@@ -320,29 +320,50 @@ class TestArchive(TestAPI):
                 self.assertEqual(session.query(URL).count(), 1)
                 self.assertEqual(session.query(Archive).count(), 1)
 
-    @wrap_test_db
-    def test_refresh_archives_delete_empty(self):
-        """
-        Domains or URLs with no archives should be deleted during refresh.
-        """
-        with wrap_media_directory():
-            (get_media_directory() / 'archive').mkdir()
-            with get_db_session(commit=True) as session:
-                # These should not be deleted.
-                domain, url = get_or_create_domain_and_url(session, 'https://example.com')
-                archive = Archive(singlefile_path='foo', url_id=url.id)
-                session.add(archive)
 
-                # These should be deleted.
-                domain = Domain(domain='bar')
-                session.add(domain)
-                session.flush()
-                session.add(URL(url='bar', domain_id=domain.id))
+def test_archive_refresh_deleted_archive(test_session, archive_directory, archive_factory):
+    archive1 = archive_factory('example.com', 'https://example.com/1')
+    archive2 = archive_factory('example.com', 'https://example.com/1')
+    archive3 = archive_factory('example.com')
+    archive4 = archive_factory('example.org')
+    archive5 = archive_factory()
 
-            self.assertEqual(len(get_urls()), 2)
-            self.assertEqual(len(get_domains()), 2)
+    # Empty directories should be ignored.
+    empty = archive_directory / 'empty'
+    empty.mkdir()
 
-            _refresh_archives()
+    def check_counts(archive_count, url_count, domain_count):
+        assert test_session.query(Archive).count() == archive_count
+        assert test_session.query(URL).count() == url_count
+        assert test_session.query(Domain).count() == domain_count
 
-            self.assertEqual(len(get_urls()), 1)
-            self.assertEqual(len(get_domains()), 1)
+    # All 3 archives are already in the DB.
+    check_counts(archive_count=5, url_count=4, domain_count=5)
+    _refresh_archives()
+    check_counts(archive_count=5, url_count=4, domain_count=4)
+
+    # Delete archive2's files, it's the latest for 'https://example.com/1'
+    for path in archive2.my_paths():
+        path.unlink()
+    _refresh_archives()
+    check_counts(archive_count=4, url_count=4, domain_count=4)
+
+    # Delete archive1's files, now the URL is empty.
+    for path in archive1.my_paths():
+        path.unlink()
+    _refresh_archives()
+    check_counts(archive_count=3, url_count=3, domain_count=3)
+
+    # Delete archive3, now there is now example.com domain
+    for path in archive3.my_paths():
+        path.unlink()
+    _refresh_archives()
+    check_counts(archive_count=2, url_count=2, domain_count=2)
+
+    # Delete all the rest of the archives
+    for path in archive4.my_paths():
+        path.unlink()
+    for path in archive5.my_paths():
+        path.unlink()
+    _refresh_archives()
+    check_counts(archive_count=0, url_count=0, domain_count=0)

wrolpi/errors.py

@@ -298,5 +298,5 @@ API_ERRORS = {
         'code': next(error_codes),
         'message': 'Download experienced an error which cannot be recovered.  Download will be deleted.',
         'status': HTTPStatus.INTERNAL_SERVER_ERROR,
-    }
+    },
 }