import json import pathlib from datetime import datetime, timedelta from http import HTTPStatus import mock import pytest from modules.archive import lib from modules.archive.lib import get_or_create_domain, get_new_archive_files, delete_archives, do_archive, get_domains, \ group_archive_files, ArchiveFiles from modules.archive.models import Archive, Domain from wrolpi.dates import local_timezone from wrolpi.db import get_db_session from wrolpi.files.lib import refresh_files from wrolpi.files.models import File from wrolpi.root_api import CustomJSONEncoder from wrolpi.test.common import skip_circleci def make_fake_request_archive(readability=True, screenshot=True, title=True): async def fake_request_archive(_): singlefile = '\ntest single-file\nジにてこちら\nsome title' r = dict( content=f'test readability content', textContent='test readability textContent', title='ジにてこちら' if title else None, ) if readability else None s = b'screenshot data' if screenshot else None return singlefile, r, s return fake_request_archive @pytest.mark.asyncio async def test_no_screenshot(test_session): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)): archive = await do_archive('https://example.com') assert isinstance(archive.singlefile_path, pathlib.Path) assert isinstance(archive.readability_path, pathlib.Path) # Screenshot was empty assert archive.screenshot_path is None @pytest.mark.asyncio async def test_no_readability(test_session): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(readability=False)): archive = await do_archive('https://example.com') assert isinstance(archive.singlefile_path, pathlib.Path) assert isinstance(archive.screenshot_path, pathlib.Path) # Readability empty assert archive.readability_path is None assert archive.readability_txt_path is None @pytest.mark.asyncio async def test_dict(test_session): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): d = (await do_archive('https://example.com')).dict() assert isinstance(d, dict) json.dumps(d, cls=CustomJSONEncoder) @pytest.mark.asyncio async def test_relationships(test_session, test_directory): with get_db_session(commit=True) as session: url = 'https://wrolpi.org:443' domain = get_or_create_domain(session, url) archive = Archive( singlefile_file=File(path=test_directory / 'wrolpi.org:443/foo'), title='bar', url=url, domain_id=domain.id, ) session.add(archive) session.flush() assert archive.domain == domain def test_archive_refresh_deleted_archive(test_client, test_session, archive_directory, archive_factory): """Archives/Domains should be deleted when archive files are deleted.""" archive1 = archive_factory('example.com', 'https://example.com/1') archive2 = archive_factory('example.com', 'https://example.com/1') archive3 = archive_factory('example.com') archive4 = archive_factory('example.org') archive5 = archive_factory() test_session.commit() # Empty directories should be ignored. (archive_directory / 'empty').mkdir() def check_counts(archive_count, domain_count): assert test_session.query(Archive).count() == archive_count, 'Archive count does not match' assert test_session.query(Domain).count() == domain_count, 'Domain count does not match' # All 5 archives are already in the DB. check_counts(archive_count=5, domain_count=2) test_client.post('/api/files/refresh') check_counts(archive_count=5, domain_count=1) # Delete archive2's files, it's the latest for 'https://example.com/1' for path in archive2.my_paths(): path.unlink() test_client.post('/api/files/refresh') check_counts(archive_count=4, domain_count=1) # Delete archive1's files, now the URL is empty. for path in archive1.my_paths(): path.unlink() test_client.post('/api/files/refresh') check_counts(archive_count=3, domain_count=0) # Delete archive3, now there is now example.com domain for path in archive3.my_paths(): path.unlink() test_client.post('/api/files/refresh') check_counts(archive_count=2, domain_count=0) # Delete all the rest of the archives for path in archive4.my_paths(): path.unlink() for path in archive5.my_paths(): path.unlink() test_client.post('/api/files/refresh') check_counts(archive_count=0, domain_count=0) def test_fills_contents_with_refresh(test_client, test_session, archive_factory): """Refreshing archives fills in any missing contents.""" archive1 = archive_factory('example.com', 'https://example.com/one') archive2 = archive_factory('example.com', 'https://example.com/one') # Title can be found in archive3's singlefile archive3 = archive_factory('example.org', singlefile_contents=' some stufflast title other stuff') archive4 = archive_factory('example.org') # has no contents # Clear the title, this will be filled in. archive1.title = archive2.title = archive3.title = archive4.title = None contents_title_archive = [ ('foo bar qux', 'my archive', archive1), ('foo baz qux qux', 'other archive', archive2), ('baz qux qux qux', None, archive3), ] for contents, json_title, archive in contents_title_archive: with archive.readability_txt_path.open('wt') as fh: fh.write(contents) if json_title: archive.readability_json_path.write_text(json.dumps({'title': json_title, 'url': archive.url})) test_session.commit() # Contents are empty. Archives are all missing the title. assert not archive1.singlefile_file.d_text assert not archive2.singlefile_file.d_text assert not archive3.singlefile_file.d_text assert not archive4.singlefile_file.d_text # Fill the contents. test_client.post('/api/files/refresh') # The archives will be renamed with their title. archive1, archive2, archive3, archive4 = test_session.query(Archive).order_by(Archive.id) assert archive1.singlefile_file.title assert archive2.singlefile_file.title assert archive3.singlefile_file.title assert not archive4.singlefile_file.d_text # Missing title is filled. assert archive1.title == 'my archive' assert archive2.title == 'other archive' assert archive3.title == 'last title' # from json def test_delete_archive(test_session, archive_factory): """Archives can be deleted.""" archive1 = archive_factory('example.com', 'https://example.com/1') archive2 = archive_factory('example.com', 'https://example.com/1') archive3 = archive_factory('example.com', 'https://example.com/1') test_session.commit() # Files are real. assert archive1.my_paths() and all(i.is_file() for i in archive1.my_paths()) assert archive2.my_paths() and all(i.is_file() for i in archive2.my_paths()) assert archive3.my_paths() and all(i.is_file() for i in archive3.my_paths()) assert test_session.query(Archive).count() == 3 assert test_session.query(File).count() == 15 # Delete the oldest. delete_archives(archive1.id, archive3.id) assert test_session.query(Archive).count() == 1 assert test_session.query(File).count() == 5 # Files were deleted. assert archive1.my_paths() and not any(i.is_file() for i in archive1.my_paths()) assert archive3.my_paths() and not any(i.is_file() for i in archive3.my_paths()) # Archive2 is untouched assert archive2.my_paths() and all(i.is_file() for i in archive2.my_paths()) # Delete the last archive. The Domain should also be deleted. delete_archives(archive2.id) assert test_session.query(Archive).count() == 0 domain = test_session.query(Domain).one_or_none() assert domain is None # All Files were deleted. assert test_session.query(File).count() == 0 assert archive2.my_paths() and not any(i.is_file() for i in archive2.my_paths()) def test_get_domains(test_session, archive_factory): """ `get_domains` gets only Domains with Archives. """ archive1 = archive_factory('example.com') archive2 = archive_factory('example.com') archive3 = archive_factory('example.org') test_session.commit() assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org'] archive2.delete() test_session.commit() assert [i['domain'] for i in get_domains()] == ['example.com', 'example.org'] archive1.delete() test_session.commit() assert [i['domain'] for i in get_domains()] == ['example.org'] archive3.delete() test_session.commit() assert [i['domain'] for i in get_domains()] == [] @pytest.mark.asycio async def test_new_archive(test_session, fake_now): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): fake_now(datetime(2000, 1, 1)) archive1 = await do_archive('https://example.com') # Everything is filled out. assert isinstance(archive1, Archive) assert archive1.archive_datetime is not None assert isinstance(archive1.singlefile_path, pathlib.Path) assert isinstance(archive1.readability_path, pathlib.Path) assert isinstance(archive1.readability_txt_path, pathlib.Path) assert isinstance(archive1.screenshot_path, pathlib.Path) assert archive1.title == 'ジにてこちら' assert archive1.url is not None assert archive1.domain is not None # The actual files were dumped and read correctly. with open(archive1.singlefile_path) as fh: assert fh.read() == '\ntest single-file\nジにてこちら\nsome title' with open(archive1.readability_path) as fh: assert fh.read() == 'test readability content' with open(archive1.readability_txt_path) as fh: assert fh.read() == 'test readability textContent' with open(archive1.readability_json_path) as fh: assert json.load(fh) == {'title': 'ジにてこちら', 'url': 'https://example.com'} fake_now(datetime(2000, 1, 2)) archive2 = await do_archive('https://example.com') # Domain is reused. assert archive1.domain == archive2.domain @pytest.mark.asyncio async def test_get_title_from_html(test_session, fake_now): fake_now(datetime(2000, 1, 1)) with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): archive = await do_archive('https://example.com') assert archive.title == 'ジにてこちら' async def fake_request_archive(_): singlefile = '\ntest single-file\nジにてこちら\nsome title' r = dict( content=f'test readability content', textContent='test readability textContent', ) s = b'screenshot data' return singlefile, r, s fake_now(datetime(2000, 1, 2)) with mock.patch('modules.archive.lib.request_archive', fake_request_archive): archive = await do_archive('https://example.com') assert archive.title == 'some title' async def fake_request_archive(_): singlefile = '' r = dict( content=f'missing a title', textContent='', ) s = b'screenshot data' return singlefile, r, s fake_now(datetime(2000, 1, 3)) with mock.patch('modules.archive.lib.request_archive', fake_request_archive): archive = await do_archive('https://example.com') assert archive.title is None @skip_circleci def test_get_new_archive_files(fake_now): """Archive files have a specific format so they are sorted by datetime, and are near each other.""" fake_now(datetime(2001, 1, 1)) archive_files = get_new_archive_files('https://example.com/two', None) assert str(archive_files.singlefile).endswith('archive/example.com/2001-01-01-00-00-00_NA.html') assert str(archive_files.readability).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.html') assert str(archive_files.readability_txt).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.txt') assert str(archive_files.readability_json).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.json') assert str(archive_files.screenshot).endswith('archive/example.com/2001-01-01-00-00-00_NA.png') archive_files = get_new_archive_files('https://www.example.com/one', 'Title') # Leading www. is removed. assert str(archive_files.singlefile).endswith('archive/example.com/2001-01-01-00-00-00_Title.html') assert str(archive_files.readability).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.html') assert str(archive_files.readability_txt).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.txt') assert str(archive_files.readability_json).endswith( 'archive/example.com/2001-01-01-00-00-00_Title.readability.json') assert str(archive_files.screenshot).endswith('archive/example.com/2001-01-01-00-00-00_Title.png') @skip_circleci @pytest.mark.asyncio async def test_title_in_filename(test_session, fake_now, test_directory): """ The Archive files have the title in the path. """ fake_now(datetime(2000, 1, 1)) with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): archive1 = await do_archive('https://example.com') assert archive1.title == 'ジにてこちら' assert str(archive1.singlefile_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_ジにてこちら.html' assert str(archive1.readability_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_ジにてこちら.readability.html' assert str(archive1.readability_txt_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_ジにてこちら.readability.txt' assert str(archive1.readability_json_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_ジにてこちら.readability.json' assert str(archive1.screenshot_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_ジにてこちら.png' async def fake_request_archive(_): singlefile = '\ntest single-file\nジにてこちら\n' # no title in HTML r = dict( # No Title from Readability. content=f'test readability content', textContent='test readability textContent', ) s = b'screenshot data' return singlefile, r, s with mock.patch('modules.archive.lib.request_archive', fake_request_archive): archive2 = await do_archive('https://example.com') assert str(archive2.singlefile_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_NA.html' assert str(archive2.readability_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_NA.readability.html' assert str(archive2.readability_txt_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_NA.readability.txt' assert str(archive2.readability_json_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_NA.readability.json' assert str(archive2.screenshot_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_NA.png' async def fake_request_archive(_): singlefile = '\ntest single-file\nジにてこちら\ndangerous ;\\//_title</html></html>' r = dict( # No Title from Readability. content=f'<html>test readability content</html>', textContent='<html>test readability textContent</html>', ) s = b'screenshot data' return singlefile, r, s with mock.patch('modules.archive.lib.request_archive', fake_request_archive): archive3 = await do_archive('https://example.com') assert str(archive3.singlefile_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.html' assert str(archive3.readability_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.readability.html' assert str(archive3.readability_txt_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.readability.txt' assert str(archive3.readability_json_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.readability.json' assert str(archive3.screenshot_path.relative_to(test_directory)) == \ 'archive/example.com/2000-01-01-00-00-00_dangerous ;_title.png' @skip_circleci def test_refresh_archives(test_session, test_directory, test_client, make_files_structure): """ Archives can be found and put in the database. A single Archive will have multiple files. """ # The start of a typical singlefile html file. singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!-- Page saved with SingleFile url: https://example.com saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time) --><head><meta charset="utf-8">''' make_files_structure({ # These are all for an individual Archive. 'archive/example.com/2021-10-05 16:20:10.346823.html': singlefile_contents, # renames to 2021-10-05-16-20-10_NA 'archive/example.com/2021-10-05 16:20:10.346823.png': None, 'archive/example.com/2021-10-05 16:20:10.346823-readability.txt': 'article text contents', 'archive/example.com/2021-10-05 16:20:10.346823-readability.json': '{"url": "https://example.com"}', 'archive/example.com/2021-10-05 16:20:10.346823-readability.html': '<html></html>', # These should log an error because they are missing the singlefile path. 'archive/example.com/2021-10-05 16:20:10.readability.html': '<html></html>', 'archive/example.com/2021-10-05 16:20:10.readability.json': None, # Bad files should also be ignored. 'archive/example.com/random file': 'hello', 'archive/example.com/2021-10-05 16:20:10 no extension': None, }) # The single archive is found. test_client.post('/api/files/refresh') assert test_session.query(Archive).count() == 1 # Running the refresh does not result in a new archive. test_client.post('/api/files/refresh') assert test_session.query(Archive).count() == 1 # Archives file format was changed, lets check the new formats are found. make_files_structure({ 'archive/example.com/2021-10-05-16-20-11_The Title.html': '<html></html>', 'archive/example.com/2021-10-05-16-20-11_The Title.readability.json': '{"url": "https://example.com"}', }) test_client.post('/api/files/refresh') # The old formatted archive above is renamed. assert (test_directory / 'archive/example.com/2021-10-05-16-20-10_NA.html').is_file() assert test_session.query(Archive).count() == 2 content = json.dumps({'search_str': 'text', 'model': 'archive'}) request, response = test_client.post('/api/files/search', content=content) assert response.status_code == HTTPStatus.OK assert response.json['files'], 'No files matched "text"' assert response.json['files'][0]['model'] == 'archive', 'Returned file was not an archive' assert response.json['files'][0].get('path') == 'archive/example.com/2021-10-05-16-20-10_NA.html', \ 'Could not find readability text file containing "text"' assert response.json['files'][0]['archive']['archive_datetime'], 'Archive has no datetime' @pytest.mark.asyncio async def test_refresh_archives_index(test_session, test_directory, test_client, make_files_structure): """Archives are indexed using ArchiveIndexer and archive_modeler.""" # The start of a typical singlefile html file. singlefile_contents = '''<!DOCTYPE html> <html lang="en"><!-- Page saved with SingleFile url: https://example.com saved date: Mon May 16 2022 23:51:35 GMT+0000 (Coordinated Universal Time) --><head><meta charset="utf-8">''' singlefile, *_ = make_files_structure({ 'archive/example.com/2021-10-05-16-20-10_NA.html': singlefile_contents, 'archive/example.com/2021-10-05-16-20-10_NA.png': None, 'archive/example.com/2021-10-05-16-20-10_NA.readability.txt': 'article text contents', 'archive/example.com/2021-10-05-16-20-10_NA.readability.json': '{"url": "https://example.com", "title": "the title"}', 'archive/example.com/2021-10-05-16-20-10_NA.readability.html': '<html></html>', }) await refresh_files() archive: Archive = test_session.query(Archive).one() assert archive.singlefile_path == singlefile assert archive.singlefile_file.a_text == 'the title' assert archive.singlefile_file.d_text == 'article text contents' # The associated files are not indexed. for name in ('screenshot_file', 'readability_file', 'readability_json_file', 'readability_txt_file'): file = getattr(archive, name) assert file.associated is True, 'The associated file is not marked' assert file.indexed is False # file.a_text is the file name. assert not file.b_text assert not file.c_text assert not file.d_text def test_refresh_archives_invalid_file(test_session, test_directory, test_client, make_files_structure): """ Invalid Archives should be removed during refresh. """ *_, bogus_path = make_files_structure({ '2022-09-04-16-20-11_The Title.html': '<html></html>', '2022-09-04-16-20-11_The Title.readability.json': '{"url": "https://example.com"}', 'bogus file': None, }) test_client.post('/api/files/refresh') assert test_session.query(File).count() == 3 # Three files total. assert test_session.query(Archive).count() == 1 # Only one real archive. # Create an invalid Archive, it should be removed during refresh. test_session.add(Archive(singlefile_path=bogus_path)) # Claim the bogus file as an Archive, this should be removed. bogus_file: File = test_session.query(File).filter_by(path=bogus_path).one() bogus_file.model = 'archive' test_session.commit() assert test_session.query(File).count() == 3 assert test_session.query(Archive).count() == 2 test_client.post('/api/files/refresh') assert test_session.query(File).count() == 3, 'More files in DB than we created' assert test_session.query(Archive).count() == 1, 'The bogus Archive was not removed' assert bogus_file.model is None, 'Model was not removed' def test_group_archive_files(test_directory): """Archive files should be grouped together when they are the same Archive.""" files = [ pathlib.Path('2021-10-05 16:20:10.346823.html'), pathlib.Path('2021-10-05 16:20:10.346823.readability.json'), pathlib.Path('2000-01-01-00-00-00_Title.html'), pathlib.Path('2000-01-01-00-00-00_Title.readability.json'), pathlib.Path('not an archive'), pathlib.Path('2000-01-01-00-00-01_Missing a singlefile.readability.json'), ] group1 = ArchiveFiles( singlefile=pathlib.Path('2000-01-01-00-00-00_Title.html'), readability_json=pathlib.Path('2000-01-01-00-00-00_Title.readability.json') ) group2 = ArchiveFiles( singlefile=pathlib.Path('2021-10-05 16:20:10.346823.html'), readability_json=pathlib.Path('2021-10-05 16:20:10.346823.readability.json'), ) assert list(group_archive_files(files)) == [ (local_timezone(datetime(2000, 1, 1, 0, 0, 0)), group1), (local_timezone(datetime(2021, 10, 5, 16, 20, 10)), group2), ] @pytest.mark.parametrize( 'name,expected', [ ('foo', False), ('2000-01-01-00-00-00_Some Title.html', True), ('2000-01-01-00-00-00_Some Title.readability.json', True), ('2000-01-01-00-00-00_Some Title.readability.html', True), ('2000-01-01-00-00-00_Some Title.readability.txt', True), ('2000-01-01-00-00-00_Some Title.png', True), ('2000-01-01-00-00-00_Some Title.jpg', True), ('2000-01-01-00-00-00_Some Title.jpeg', True), ('2000-01-01-00-00-00_Some NA.html', True), ('2000-01-01-00-00-00_NA.readability.json', True), ('2000-01-01-00-00-00_NA.readability.html', True), ('2000-01-01-00-00-00_NA.readability.txt', True), ('2000-01-01-00-00-00_NA.png', True), ('2000-01-01-00-00-00_NA.jpg', True), ('2000-01-01-00-00-00_NA.jpeg', True), ('2000-01-01-00-00-00_ジにてこちら.html', True), ('2000-01-01-00-00-00no underscore.html', False), ] ) def test_is_archive_file(name, expected, test_directory): path = test_directory / name path.touch() assert lib.is_archive_file(path) == expected @skip_circleci def test_migrate_archive_files(test_session, archive_directory, archive_factory): """Test that migration of all old Archive formatted files are migrated.""" # Create some updated archives that should be ignored. archive1 = archive_factory('example.com', title='the title') archive2 = archive_factory('example.com') archive3 = archive_factory('example.org') test_session.commit() assert archive1.singlefile_path.is_file() assert archive2.singlefile_path.is_file() assert archive3.singlefile_path.is_file() example_com_directory = archive_directory / 'example.com' example_org_directory = archive_directory / 'example.org' # Archive4 (example_com_directory / '2021-10-05 16:20:10.346823.html').touch() path = example_com_directory / '2021-10-05 16:20:10.346823.readability.json' path.write_text(json.dumps({'title': 'my title'})) # Archive5 (example_com_directory / '2000-01-01 00:00:00.000000.html').touch() path = example_com_directory / '2000-01-01 00:00:00.000000.readability.json' path.write_text(json.dumps({'title': 'invalid#*: title'})) # Archive6 (example_org_directory / '2000-01-01 00:00:00.000000.html').touch() path = example_org_directory / '2000-01-01 00:00:00.000000.readability.json' path.write_text(json.dumps({'title': 'A' * 500})) # Really long file name. # Migration forms a plan, then performs it. plan = lib.migrate_archive_files() for old, new in plan: # All files are moved. assert not old.is_file() assert new.is_file() # Check that only the outdated files are moved. plan = [(i.relative_to(archive_directory), j.relative_to(archive_directory)) for i, j in plan] assert plan == [ (pathlib.Path('example.com/2000-01-01 00:00:00.000000.html'), pathlib.Path('example.com/2000-01-01-00-00-00_invalid# title.html')), (pathlib.Path('example.com/2000-01-01 00:00:00.000000.readability.json'), pathlib.Path('example.com/2000-01-01-00-00-00_invalid# title.readability.json')), (pathlib.Path('example.com/2021-10-05 16:20:10.346823.html'), pathlib.Path('example.com/2021-10-05-16-20-10_my title.html')), (pathlib.Path('example.com/2021-10-05 16:20:10.346823.readability.json'), pathlib.Path('example.com/2021-10-05-16-20-10_my title.readability.json')), (pathlib.Path('example.org/2000-01-01 00:00:00.000000.html'), pathlib.Path('example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.html')), (pathlib.Path('example.org/2000-01-01 00:00:00.000000.readability.json'), pathlib.Path( 'example.org/2000-01-01-00-00-00_AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA.readability.json')), ] # New Archive files still exist. assert all(i.is_file() for i in archive1.my_paths()) assert all(i.is_file() for i in archive2.my_paths()) assert all(i.is_file() for i in archive3.my_paths()) def test_archive_files(test_directory): af = lib.ArchiveFiles( singlefile=test_directory / 'archive/singlefile.html', ) assert repr(af) == \ "<ArchiveFiles singlefile='singlefile.html' readability=None readability_txt=None readability_json=None" \ " screenshot=None>" def test_archive_order(test_session, test_directory, archive_factory): archive1 = archive_factory() archive2 = archive_factory() archive2.archive_datetime += timedelta(seconds=10) test_session.commit() assert archive2 > archive1 assert archive1 < archive2 @pytest.mark.asyncio async def test_archive_download_index(test_session, test_directory, image_file): """An Archive is indexed when it is downloaded.""" with mock.patch('modules.archive.lib.request_archive') as mock_request_archive: singlefile = '<html><title>the singlefile' readability = dict( content='the readability', textContent='the readability', ) mock_request_archive.return_value = (singlefile, readability, image_file.read_bytes()) archive = await lib.do_archive('https://example.com') assert isinstance(archive, lib.Archive), 'Did not get an archive' assert archive.singlefile_path and archive.singlefile_path.exists() and \ archive.singlefile_path.read_text() == singlefile, 'Singlefile was not stored' assert archive.readability_path and archive.readability_path.exists() assert archive.readability_json_path assert archive.readability_txt_path and archive.readability_txt_path.exists() assert archive.readability_txt_file.d_text == '{the,readability}', 'Readability text was not indexed' assert archive.title == 'the singlefile', 'Did not get the title from the singlefile' assert archive.screenshot_path and archive.screenshot_file and archive.screenshot_path.is_file(), \ 'Did not store the screenshot' assert not archive.singlefile_file.associated assert archive.readability_txt_file.associated \ and archive.readability_json_file.associated \ and archive.readability_file.associated \ and archive.screenshot_file.associated, 'Archive files must be associated'