diff --git a/modules/archive/test/test_compressed_singlefile.py b/modules/archive/test/test_compressed_singlefile.py index b8bcdc80..29ab71ef 100644 --- a/modules/archive/test/test_compressed_singlefile.py +++ b/modules/archive/test/test_compressed_singlefile.py @@ -196,6 +196,23 @@ async def test_model_archive_compressed(async_client, test_session, test_directo assert archive.collection.name == 'example.com' +def test_html_indexer_compressed(make_files_structure, compressed_singlefile_factory): + """The generic HTML indexer is binary-safe: it indexes the page inside a compressed + singlefile instead of crashing on the binary ZIP tail. + + The archive modeler claims WROLPi's own singlefiles; this indexer handles foreign HTML + files, e.g. a SingleFileZ file saved by the browser extension and dropped into the media + directory.""" + from wrolpi.files.indexers import HTMLIndexer + + path, = make_files_structure(['foreign.html']) + path.write_bytes(compressed_singlefile_factory(title='foreign singlefilez')) + + title, _, _, words = HTMLIndexer.create_index(path) + assert title == 'foreign singlefilez' + assert 'test compressed singlefile' in words + + def test_compress_singlefile_setting_flow(test_session): """compress_singlefile survives DownloadRequest settings validation, and the RSSDownloader passes it on to the child archive downloads.""" diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py index 05a6e2b1..abaeec75 100644 --- a/wrolpi/files/indexers.py +++ b/wrolpi/files/indexers.py @@ -121,11 +121,13 @@ class HTMLIndexer(Indexer, ABC): @classmethod def create_index(cls, path: pathlib.Path): - from modules.archive.lib import parse_article_html_metadata + from modules.archive.lib import parse_article_html_metadata, read_singlefile_html a = cls.get_title(path) - contents = path.read_text() + # Read bytes; a compressed (SingleFileZ) singlefile has a binary ZIP tail, and its real + # page (index.html) is inside the ZIP. Other HTML files are read unchanged. + contents = read_singlefile_html(path) metadata = parse_article_html_metadata(contents) text = extract_html_text(contents) words = split_lines_by_length(text)