From fb23fe1f6165245d9e2db65cd63f6e5791a83b12 Mon Sep 17 00:00:00 2001 From: Roland Date: Fri, 12 Jun 2026 17:24:02 -0600 Subject: [PATCH] HTML indexer is binary-safe; it indexes the page inside a compressed singlefile. The archive modeler claims WROLPi's own singlefiles, but a foreign compressed (SingleFileZ) HTML file (e.g. saved by the browser extension and dropped into the media directory) falls through to the generic HTML indexer, which crashed reading the binary ZIP tail as text. The indexer now reads bytes and indexes the index.html inside the ZIP. Co-Authored-By: Claude Fable 5 --- .../archive/test/test_compressed_singlefile.py | 17 +++++++++++++++++ wrolpi/files/indexers.py | 6 ++++-- 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/modules/archive/test/test_compressed_singlefile.py b/modules/archive/test/test_compressed_singlefile.py index b8bcdc80..29ab71ef 100644 --- a/modules/archive/test/test_compressed_singlefile.py +++ b/modules/archive/test/test_compressed_singlefile.py @@ -196,6 +196,23 @@ async def test_model_archive_compressed(async_client, test_session, test_directo assert archive.collection.name == 'example.com' +def test_html_indexer_compressed(make_files_structure, compressed_singlefile_factory): + """The generic HTML indexer is binary-safe: it indexes the page inside a compressed + singlefile instead of crashing on the binary ZIP tail. + + The archive modeler claims WROLPi's own singlefiles; this indexer handles foreign HTML + files, e.g. a SingleFileZ file saved by the browser extension and dropped into the media + directory.""" + from wrolpi.files.indexers import HTMLIndexer + + path, = make_files_structure(['foreign.html']) + path.write_bytes(compressed_singlefile_factory(title='foreign singlefilez')) + + title, _, _, words = HTMLIndexer.create_index(path) + assert title == 'foreign singlefilez' + assert 'test compressed singlefile' in words + + def test_compress_singlefile_setting_flow(test_session): """compress_singlefile survives DownloadRequest settings validation, and the RSSDownloader passes it on to the child archive downloads.""" diff --git a/wrolpi/files/indexers.py b/wrolpi/files/indexers.py index 05a6e2b1..abaeec75 100644 --- a/wrolpi/files/indexers.py +++ b/wrolpi/files/indexers.py @@ -121,11 +121,13 @@ class HTMLIndexer(Indexer, ABC): @classmethod def create_index(cls, path: pathlib.Path): - from modules.archive.lib import parse_article_html_metadata + from modules.archive.lib import parse_article_html_metadata, read_singlefile_html a = cls.get_title(path) - contents = path.read_text() + # Read bytes; a compressed (SingleFileZ) singlefile has a binary ZIP tail, and its real + # page (index.html) is inside the ZIP. Other HTML files are read unchanged. + contents = read_singlefile_html(path) metadata = parse_article_html_metadata(contents) text = extract_html_text(contents) words = split_lines_by_length(text)