diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py new file mode 100644 index 00000000..52dc6aab --- /dev/null +++ b/modules/archive/conftest.py @@ -0,0 +1,85 @@ +import datetime +import json +import pathlib +from uuid import uuid4 + +import pytest + +from modules.archive.models import Archive, URL, Domain +from wrolpi.dates import strftime_ms + + +@pytest.fixture +def archive_directory(test_directory) -> pathlib.Path: + path = test_directory / 'archive' + path.mkdir() + return path + + +@pytest.fixture +def archive_factory(test_session, archive_directory): + def time_generator(): + timestamp = datetime.datetime(2000, 1, 1, 0, 0, 0) + while True: + timestamp += datetime.timedelta(seconds=1) + yield timestamp + + now = time_generator() + + def _(domain: str = None, url: str = None) -> Archive: + title = str(uuid4()) + url = url or f'https://example.com/{title}' + domain_dir = domain or title + domain_dir = archive_directory / domain_dir + domain_dir.mkdir(exist_ok=True) + + timestamp = strftime_ms(next(now)) + + singlefile_path = domain_dir / f'{timestamp}.html' + screenshot_path = domain_dir / f'{timestamp}.png' + readability_path = domain_dir / f'{timestamp}-readability.html' + readability_txt_path = domain_dir / f'{timestamp}-readability.txt' + readability_json_path = domain_dir / f'{timestamp}-readability.json' + + with readability_json_path.open('wt') as fh: + json.dump({'url': url}, fh) + + if domain: + domain = test_session.query(Domain).filter_by(domain=domain).one_or_none() + + if not domain: + domain = Domain( + domain=domain, + directory=domain_dir, + ) + test_session.add(domain) + test_session.commit() + + url_ = test_session.query(URL).filter_by(url=url).one_or_none() + if not url_: + url_ = URL( + url=url, + domain_id=domain.id, + ) + test_session.add(url_) + test_session.commit() + + archive = Archive( + readability_json_path=readability_json_path, + readability_path=readability_path, + readability_txt_path=readability_txt_path, + screenshot_path=screenshot_path, + singlefile_path=singlefile_path, + status='complete', + title=title, + url_id=url_.id, + ) + test_session.add(archive) + test_session.commit() + + for path in archive.my_paths(): + path.touch() + + return archive + + return _ diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 8c4bbc11..847fc8f4 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -367,9 +367,12 @@ def is_archive_file(path: pathlib.Path) -> bool: def _refresh_archives(): """ - Search the Archives directory for archive files, update the database if new files are found. + Search the Archives directory for archive files, update the database if new files are found. Remove any orphan + URLs or Domains. """ archive_directory = get_archive_directory() + + singlefile_paths = set() for domain_directory in filter(lambda i: i.is_dir(), archive_directory.iterdir()): logger.debug(f'Refreshing directory: {domain_directory}') archives_files = filter(is_archive_file, domain_directory.iterdir()) @@ -379,19 +382,24 @@ def _refresh_archives(): archive_count += 1 with get_db_session(commit=True) as session: for dt, files in chunk: - print('_refresh_archives', dt) + singlefile_paths.add(str(files[0])) upsert_archive(dt, files, session) if archive_count: logger.info(f'Inserted/updated {archive_count} archives') - cleanup_domains_urls() + with get_db_session(commit=True) as session: + # Check that each URLs most recent Archive exists, if it does not use the next most recent. + urls = session.query(URL).all() + for url in urls: + if not url.latest or not url.latest.singlefile_path or not url.latest.singlefile_path.path.exists(): + url.update_latest() + session.commit() + singlefile_paths = list(singlefile_paths) + with get_db_curs(commit=True) as curs: + curs.execute('DELETE FROM archive WHERE singlefile_path != ALL(%s)', (singlefile_paths,)) -def cleanup_domains_urls(): - """ - Delete any URLs/Domains without Archives. - """ with get_db_curs(commit=True) as curs: stmt = ''' DELETE FROM url WHERE id NOT IN ( diff --git a/modules/archive/models.py b/modules/archive/models.py index e6fb550d..5d5ea12b 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -1,5 +1,7 @@ +from datetime import datetime from typing import Generator +import pytz from sqlalchemy import Column, Integer, String, ForeignKey from sqlalchemy.orm import relationship from sqlalchemy.orm.collections import InstrumentedList @@ -75,6 +77,17 @@ class URL(Base, ModelHelper): d['archives'] = [i.dict() for i in self.archives] return d + def update_latest(self): + """ + Set `latest` to the most recent Archive of this URL whose `singlefile` exists. + """ + self.latest_id = None + latest_datetime = datetime(2, 1, 1, 0, 0, 0).astimezone(tz=pytz.UTC) + for archive in self.archives: + if archive.singlefile_path and archive.singlefile_path.path.exists() and \ + archive.archive_datetime >= latest_datetime: + self.latest_id = archive.id + class Domain(Base, ModelHelper): __tablename__ = 'domains' # plural to avoid conflict diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index d0a779fb..e8598871 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -5,7 +5,7 @@ from datetime import datetime import mock from modules.archive.lib import new_archive, get_or_create_domain_and_url, get_urls, get_url_count, delete_url, \ - _refresh_archives, get_domains, get_new_archive_files + _refresh_archives, get_new_archive_files from modules.archive.models import Archive, URL, Domain from wrolpi.common import get_media_directory from wrolpi.db import get_db_session @@ -320,29 +320,50 @@ class TestArchive(TestAPI): self.assertEqual(session.query(URL).count(), 1) self.assertEqual(session.query(Archive).count(), 1) - @wrap_test_db - def test_refresh_archives_delete_empty(self): - """ - Domains or URLs with no archives should be deleted during refresh. - """ - with wrap_media_directory(): - (get_media_directory() / 'archive').mkdir() - with get_db_session(commit=True) as session: - # These should not be deleted. - domain, url = get_or_create_domain_and_url(session, 'https://example.com') - archive = Archive(singlefile_path='foo', url_id=url.id) - session.add(archive) - # These should be deleted. - domain = Domain(domain='bar') - session.add(domain) - session.flush() - session.add(URL(url='bar', domain_id=domain.id)) +def test_archive_refresh_deleted_archive(test_session, archive_directory, archive_factory): + archive1 = archive_factory('example.com', 'https://example.com/1') + archive2 = archive_factory('example.com', 'https://example.com/1') + archive3 = archive_factory('example.com') + archive4 = archive_factory('example.org') + archive5 = archive_factory() - self.assertEqual(len(get_urls()), 2) - self.assertEqual(len(get_domains()), 2) + # Empty directories should be ignored. + empty = archive_directory / 'empty' + empty.mkdir() - _refresh_archives() + def check_counts(archive_count, url_count, domain_count): + assert test_session.query(Archive).count() == archive_count + assert test_session.query(URL).count() == url_count + assert test_session.query(Domain).count() == domain_count - self.assertEqual(len(get_urls()), 1) - self.assertEqual(len(get_domains()), 1) + # All 3 archives are already in the DB. + check_counts(archive_count=5, url_count=4, domain_count=5) + _refresh_archives() + check_counts(archive_count=5, url_count=4, domain_count=4) + + # Delete archive2's files, it's the latest for 'https://example.com/1' + for path in archive2.my_paths(): + path.unlink() + _refresh_archives() + check_counts(archive_count=4, url_count=4, domain_count=4) + + # Delete archive1's files, now the URL is empty. + for path in archive1.my_paths(): + path.unlink() + _refresh_archives() + check_counts(archive_count=3, url_count=3, domain_count=3) + + # Delete archive3, now there is now example.com domain + for path in archive3.my_paths(): + path.unlink() + _refresh_archives() + check_counts(archive_count=2, url_count=2, domain_count=2) + + # Delete all the rest of the archives + for path in archive4.my_paths(): + path.unlink() + for path in archive5.my_paths(): + path.unlink() + _refresh_archives() + check_counts(archive_count=0, url_count=0, domain_count=0) diff --git a/wrolpi/errors.py b/wrolpi/errors.py index eda916f0..7f6396ef 100644 --- a/wrolpi/errors.py +++ b/wrolpi/errors.py @@ -298,5 +298,5 @@ API_ERRORS = { 'code': next(error_codes), 'message': 'Download experienced an error which cannot be recovered. Download will be deleted.', 'status': HTTPStatus.INTERNAL_SERVER_ERROR, - } + }, }