mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-22 16:06:19 -05:00
HTML indexer is binary-safe; it indexes the page inside a compressed singlefile.
The archive modeler claims WROLPi's own singlefiles, but a foreign compressed (SingleFileZ) HTML file (e.g. saved by the browser extension and dropped into the media directory) falls through to the generic HTML indexer, which crashed reading the binary ZIP tail as text. The indexer now reads bytes and indexes the index.html inside the ZIP. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
1acb148c90
commit
fb23fe1f61
2 changed files with 21 additions and 2 deletions
|
|
@ -196,6 +196,23 @@ async def test_model_archive_compressed(async_client, test_session, test_directo
|
|||
assert archive.collection.name == 'example.com'
|
||||
|
||||
|
||||
def test_html_indexer_compressed(make_files_structure, compressed_singlefile_factory):
|
||||
"""The generic HTML indexer is binary-safe: it indexes the page inside a compressed
|
||||
singlefile instead of crashing on the binary ZIP tail.
|
||||
|
||||
The archive modeler claims WROLPi's own singlefiles; this indexer handles foreign HTML
|
||||
files, e.g. a SingleFileZ file saved by the browser extension and dropped into the media
|
||||
directory."""
|
||||
from wrolpi.files.indexers import HTMLIndexer
|
||||
|
||||
path, = make_files_structure(['foreign.html'])
|
||||
path.write_bytes(compressed_singlefile_factory(title='foreign singlefilez'))
|
||||
|
||||
title, _, _, words = HTMLIndexer.create_index(path)
|
||||
assert title == 'foreign singlefilez'
|
||||
assert 'test compressed singlefile' in words
|
||||
|
||||
|
||||
def test_compress_singlefile_setting_flow(test_session):
|
||||
"""compress_singlefile survives DownloadRequest settings validation, and the RSSDownloader
|
||||
passes it on to the child archive downloads."""
|
||||
|
|
|
|||
|
|
@ -121,11 +121,13 @@ class HTMLIndexer(Indexer, ABC):
|
|||
|
||||
@classmethod
|
||||
def create_index(cls, path: pathlib.Path):
|
||||
from modules.archive.lib import parse_article_html_metadata
|
||||
from modules.archive.lib import parse_article_html_metadata, read_singlefile_html
|
||||
|
||||
a = cls.get_title(path)
|
||||
|
||||
contents = path.read_text()
|
||||
# Read bytes; a compressed (SingleFileZ) singlefile has a binary ZIP tail, and its real
|
||||
# page (index.html) is inside the ZIP. Other HTML files are read unchanged.
|
||||
contents = read_singlefile_html(path)
|
||||
metadata = parse_article_html_metadata(contents)
|
||||
text = extract_html_text(contents)
|
||||
words = split_lines_by_length(text)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue