diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 6f8921ff..5da81b64 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -6,7 +6,7 @@ import re import subprocess import tempfile from itertools import groupby -from typing import Iterator +from typing import Iterator, Optional import requests from bs4 import BeautifulSoup @@ -15,7 +15,7 @@ from sqlalchemy.orm import Session from modules.archive.models import Domain, Archive from wrolpi.common import get_media_directory, logger, chunks, extract_domain, chdir -from wrolpi.dates import now, strptime_ms, strftime_ms, Seconds +from wrolpi.dates import now, strptime_ms, Seconds, strftime from wrolpi.db import get_db_session, get_db_curs, get_ranked_models from wrolpi.errors import InvalidDomain, UnknownURL, InvalidArchive from wrolpi.vars import DOCKERIZED, PYTEST @@ -47,18 +47,20 @@ def get_domain_directory(url: str) -> pathlib.Path: return directory -def get_new_archive_files(url: str): +def get_new_archive_files(url: str, title: Optional[str]): """ Create a list of archive files using a shared name schema. Raise an error if any of them exist. """ directory = get_domain_directory(url) - dt = strftime_ms(now()) + dt = now().strftime('%Y-%m-%d-%H-%M-%S') - singlefile_path = directory / f'{dt}.html' - readability_path = directory / f'{dt}-readability.html' - readability_txt_path = directory / f'{dt}-readability.txt' - readability_json_path = directory / f'{dt}-readability.json' - screenshot_path = directory / f'{dt}.png' + title = escape_file_name(title or 'NA') + prefix = f'{dt}_{title}' + singlefile_path = directory / f'{prefix}.html' + readability_path = directory / f'{prefix}.readability.html' + readability_txt_path = directory / f'{prefix}.readability.txt' + readability_json_path = directory / f'{prefix}.readability.json' + screenshot_path = directory / f'{prefix}.png' ret = (singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path) @@ -196,8 +198,6 @@ def do_archive(url: str) -> Archive: record in the DB. Create Domain/URL if missing. """ logger.info(f'Archiving {url}') - singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \ - get_new_archive_files(url) if DOCKERIZED or PYTEST: # Perform the archive in the Archive docker container. (Typically in the development environment). @@ -206,6 +206,30 @@ def do_archive(url: str) -> Archive: # Perform the archive using locally installed executables. singlefile, readability, screenshot = local_archive(url) + # First try to get the title from Readability. + title = readability.get('title') if readability else None + + if not title and singlefile: + # Try to get the title ourselves from the HTML. + title = get_title_from_html(singlefile, url=url) + if readability: + # Readability could not find title, lets use ours. + readability['title'] = title + + singlefile_path, readability_path, readability_txt_path, readability_json_path, screenshot_path = \ + get_new_archive_files(url, title) + + if readability: + # Write the readability parts to their own files. Write what is left after pops to the JSON file. + with readability_path.open('wt') as fh: + fh.write(readability.pop('content')) + with readability_txt_path.open('wt') as fh: + readability_txt = readability.pop('textContent') + fh.write(readability_txt) + else: + # No readability was returned, so there are no files. + readability_txt = readability_path = readability_txt_path = None + # Store the single-file HTML in its own file. with singlefile_path.open('wt') as fh: fh.write(singlefile) @@ -215,27 +239,8 @@ def do_archive(url: str) -> Archive: else: screenshot_path = None - # Store the Readability into separate files. This allows the user to view text-only or html articles. - title = None - readability_txt = None - if readability: - title = readability.get('title') - - # Write the readability parts to their own files. Write what is left after pops to the JSON file. - with readability_path.open('wt') as fh: - fh.write(readability.pop('content')) - with readability_txt_path.open('wt') as fh: - readability_txt = readability.pop('textContent') - fh.write(readability_txt) - else: - readability_path = readability_txt_path = None - # Always write a JSON file that contains at least the URL. readability = readability or {} - # Use the Readability title, or try and extract one from singlefile. - if not title and singlefile: - title = get_title_from_html(singlefile, url=url) - readability['title'] = title readability['url'] = url with readability_json_path.open('wt') as fh: fh.write(json.dumps(readability)) @@ -496,3 +501,12 @@ def search(search_str: str, domain: str, limit: int, offset: int): results = get_ranked_models(ranked_ids, Archive) return results, total + + +# These characters are invalid in Windows or Linux. +INVALID_FILE_CHARS = re.compile(r'[/<>:\|"\\\?\*]') + + +def escape_file_name(name: str) -> str: + """Replace any invalid characters in a file name with "_".""" + return INVALID_FILE_CHARS.sub('', name) diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index e868be84..c20ca5a5 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -3,7 +3,9 @@ import pathlib from datetime import datetime import mock +import pytest +from modules.archive import lib from modules.archive.lib import get_or_create_domain, _refresh_archives, \ get_new_archive_files, delete_archive, do_archive, get_domains from modules.archive.models import Archive, Domain @@ -34,23 +36,6 @@ class TestArchive(TestAPI): super().setUp() (pathlib.Path(self.tmp_dir.name) / 'archive').mkdir(exist_ok=True) - @mock.patch('modules.archive.lib.now', lambda: datetime(2001, 1, 1)) - def test_get_new_archive_files(self): - s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two')) - assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html') - assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html') - assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt') - assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json') - assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png') - - s, r, t, j, c = get_new_archive_files('https://www.example.com/one') - # Leading www. is removed. - assert str(s).endswith('archive/example.com/2001-01-01 00:00:00.000000.html') - assert str(r).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.html') - assert str(t).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.txt') - assert str(j).endswith('archive/example.com/2001-01-01 00:00:00.000000-readability.json') - assert str(c).endswith('archive/example.com/2001-01-01 00:00:00.000000.png') - @wrap_test_db def test_no_screenshot(self): with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive(screenshot=False)): @@ -328,3 +313,113 @@ def test_get_title_from_html(test_session, fake_now): with mock.patch('modules.archive.lib.request_archive', fake_request_archive): archive = do_archive('example.com') assert archive.title is None + + +def test_get_new_archive_files(fake_now): + """Archive files have a specific format so they are sorted by datetime, and are near each other.""" + fake_now(datetime(2001, 1, 1)) + s, r, t, j, c = map(str, get_new_archive_files('https://example.com/two', None)) + assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_NA.html') + assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.html') + assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.txt') + assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_NA.readability.json') + assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_NA.png') + + s, r, t, j, c = get_new_archive_files('https://www.example.com/one', 'Title') + # Leading www. is removed. + assert str(s).endswith('archive/example.com/2001-01-01-00-00-00_Title.html') + assert str(r).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.html') + assert str(t).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.txt') + assert str(j).endswith('archive/example.com/2001-01-01-00-00-00_Title.readability.json') + assert str(c).endswith('archive/example.com/2001-01-01-00-00-00_Title.png') + + +def test_title_in_filename(test_session, fake_now, test_directory): + """ + The Archive files have the title in the path. + """ + fake_now(datetime(2000, 1, 1)) + + with mock.patch('modules.archive.lib.request_archive', make_fake_request_archive()): + archive1 = do_archive('example.com') + + assert archive1.title == 'ジにてこちら' + + assert str(archive1.singlefile_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_ジにてこちら.html' + assert str(archive1.readability_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_ジにてこちら.readability.html' + assert str(archive1.readability_txt_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_ジにてこちら.readability.txt' + assert str(archive1.readability_json_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_ジにてこちら.readability.json' + assert str(archive1.screenshot_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_ジにてこちら.png' + + def fake_request_archive(_): + singlefile = '\ntest single-file\nジにてこちら\n' # no title in HTML + r = dict( + # No Title from Readability. + content=f'test readability content', + textContent='test readability textContent', + ) + s = b'screenshot data' + return singlefile, r, s + + with mock.patch('modules.archive.lib.request_archive', fake_request_archive): + archive2 = do_archive('example.com') + + assert str(archive2.singlefile_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_NA.html' + assert str(archive2.readability_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_NA.readability.html' + assert str(archive2.readability_txt_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_NA.readability.txt' + assert str(archive2.readability_json_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_NA.readability.json' + assert str(archive2.screenshot_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_NA.png' + + def fake_request_archive(_): + singlefile = '\ntest single-file\nジにてこちら\ndangerous ;\\//_title</html></html>' + r = dict( + # No Title from Readability. + content=f'<html>test readability content</html>', + textContent='<html>test readability textContent</html>', + ) + s = b'screenshot data' + return singlefile, r, s + + with mock.patch('modules.archive.lib.request_archive', fake_request_archive): + archive3 = do_archive('example.com') + + assert str(archive3.singlefile_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_dangerous ;_title.html' + assert str(archive3.readability_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_dangerous ;_title.readability.html' + assert str(archive3.readability_txt_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_dangerous ;_title.readability.txt' + assert str(archive3.readability_json_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_dangerous ;_title.readability.json' + assert str(archive3.screenshot_path.path.relative_to(test_directory)) == \ + 'archive/2000-01-01-00-00-00_dangerous ;_title.png' + + +@pytest.mark.parametrize( + 'name,expected', [ + ('', ''), + ('foo', 'foo'), + ('foo\\', 'foo'), + ('foo/', 'foo'), + ('foo<', 'foo'), + ('foo>', 'foo'), + ('foo:', 'foo'), + ('foo|', 'foo'), + ('foo"', 'foo'), + ('foo?', 'foo'), + ('foo*', 'foo'), + ('foo&', 'foo&'), + ] +) +def test_escape_file_name(name, expected): + assert lib.escape_file_name(name) == expected