Readability extraction no longer takes minutes on image-heavy pages

SingleFile inlines every image as a base64 data URI, and readability-extractor
(jsdom) slows to minutes on a page carrying megabytes of them, so these pages
hit the readability timeout and were saved without readability text.

Replace each base64 payload with a short placeholder before running
readability-extractor, then restore the payloads in its output.  Applied in
both the archive container and the native (Raspberry Pi/Debian) path.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-15 18:04:18 -06:00
parent c6613dfee0
commit 7adef5d43e
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
3 changed files with 99 additions and 3 deletions

View file

@ -11,6 +11,7 @@ import logging
import os
import os.path
import pathlib
import re
import signal
import subprocess
import sys
@ -187,6 +188,31 @@ async def call_single_file(url, compress: bool = False) -> (bytes, bytes):
return singlefile, page_html
# The prefix and payload of a base64 data URI. Keep in sync with modules/archive/lib.py.
BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)')
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI(\d+)X')
def strip_data_uris(html: bytes) -> (bytes, list):
"""Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a
page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads."""
payloads = list()
def replace(match):
payloads.append(match.group(2))
return match.group(1) + f'WROLPIDATAURI{len(payloads) - 1}X'.encode()
return BASE64_DATA_URI.sub(replace, html), payloads
def restore_data_uris(readability: dict, payloads: list) -> dict:
"""Put the payloads removed by `strip_data_uris` back into the readability output."""
for key, value in readability.items():
if isinstance(value, str):
readability[key] = DATA_URI_PLACEHOLDER.sub(lambda m: payloads[int(m.group(1))].decode(), value)
return readability
async def extract_readability(path: str, url: str) -> dict:
"""
Call the CLI command for readability-extractor.
@ -194,6 +220,9 @@ async def extract_readability(path: str, url: str) -> dict:
See https://github.com/ArchiveBox/readability-extractor
"""
logger.info(f'readability for {url}')
html, data_uri_payloads = strip_data_uris(pathlib.Path(path).read_bytes())
path = pathlib.Path(path).with_name('readability-input.html')
path.write_bytes(html)
cmd = f'readability-extractor {path} "{url}"'
logger.debug(f'readability cmd: {cmd}')
stdout, stderr, return_code = await check_output(cmd, timeout=3 * 60)
@ -203,7 +232,7 @@ async def extract_readability(path: str, url: str) -> dict:
stderr = stderr.decode() if stderr else None
raise RuntimeError(f'Failed to extract readability. {stderr}') from e
logger.debug(f'done readability for {url}')
return output
return restore_data_uris(output, data_uri_payloads)
async def take_screenshot(url: str) -> bytes:

View file

@ -1558,10 +1558,36 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) ->
]
# The prefix and payload of a base64 data URI. Keep in sync with docker/archive/main.py.
BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)')
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI(\d+)X')
def strip_data_uris(html: bytes) -> Tuple[bytes, List[bytes]]:
"""Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a
page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads."""
payloads = list()
def replace(match):
payloads.append(match.group(2))
return match.group(1) + f'WROLPIDATAURI{len(payloads) - 1}X'.encode()
return BASE64_DATA_URI.sub(replace, html), payloads
def restore_data_uris(readability: dict, payloads: List[bytes]) -> dict:
"""Put the payloads removed by `strip_data_uris` back into the readability output."""
for key, value in readability.items():
if isinstance(value, str):
readability[key] = DATA_URI_PLACEHOLDER.sub(lambda m: payloads[int(m.group(1))].decode(), value)
return readability
async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
"""Extract the readability dict from the provided HTML."""
html, data_uri_payloads = strip_data_uris(html.encode() if isinstance(html, str) else html)
with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file:
singlefile_file.write(html.encode() if isinstance(html, str) else html)
singlefile_file.write(html)
singlefile_file.flush()
os.fsync(singlefile_file.fileno())
@ -1585,7 +1611,7 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
e = ChildProcessError(stderr)
raise RuntimeError(f'Failed to extract readability from {url}') from e
logger.debug(f'done readability for {url}')
return readability
return restore_data_uris(readability, data_uri_payloads)
else:
logger.error(f'Failed to extract readability for {url}')
e = ChildProcessError(stdout if stdout else 'No stdout')

View file

@ -1,5 +1,6 @@
import json
import pathlib
import re
import tempfile
from datetime import datetime, timedelta
from http import HTTPStatus
@ -19,6 +20,7 @@ from modules.archive.lib import get_or_create_domain_collection, get_new_archive
delete_archives_by_file_group_ids, model_archive_result, get_domains
from modules.archive.models import Archive
from wrolpi.api_utils import CustomJSONEncoder
from wrolpi.cmd import CommandResult
from wrolpi.collections import Collection
from wrolpi.common import get_wrolpi_config
from wrolpi.db import get_db_session
@ -1452,3 +1454,42 @@ def test_search_archives_count_is_driven_by_the_search_index(test_directory, tes
(statement, params), = counts
plan = [row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {statement}'), params).fetchall()]
assert plan[0].startswith('SCAN fts VIRTUAL TABLE'), plan
@pytest.mark.asyncio
async def test_html_to_readability_data_uris(test_directory):
"""readability-extractor never receives inline base64 data, which makes it take minutes on a page of
inline images; the data is restored in the readability output."""
png = 'iVBORw0KGgo' + 'A' * 5000 + '=='
webp = 'UklGRiQAAABXRUJQ' + 'B' * 5000
html = f'<html><head><style>.x {{ background: url(data:image/png;base64,{png}) }}</style></head>' \
f'<body><article><p>Some text</p>' \
f'<img src="data:image/png;base64,{png}">' \
f'<img src=\'data:image/webp;base64,{webp}\'>' \
f'<img src="data:image/svg+xml,%3Csvg%3E%3C/svg%3E">' \
f'</article></body></html>'
received = list()
async def fake_run_command(cmd, *args, **kwargs):
# Echo the images back like readability-extractor does in `content`.
page = pathlib.Path(cmd[3]).read_text()
received.append(page)
content = ''.join(re.findall(r'<img[^>]*>', page))
stdout = json.dumps({'title': 'Title', 'content': content, 'textContent': 'Some text'}).encode()
return CommandResult(return_code=0, cancelled=False, stdout=stdout, stderr=b'', elapsed=0)
with mock.patch('modules.archive.lib.run_command', fake_run_command):
readability = await lib.html_to_readability(html, 'https://example.com')
# readability-extractor got the page without the base64 payloads.
assert png not in received[0] and webp not in received[0]
assert 'Some text' in received[0]
assert 'data:image/svg+xml,%3Csvg%3E%3C/svg%3E' in received[0], 'non-base64 data URI should be untouched'
# The images were restored in the output.
assert readability['content'] == f'<img src="data:image/png;base64,{png}">' \
f'<img src=\'data:image/webp;base64,{webp}\'>' \
f'<img src="data:image/svg+xml,%3Csvg%3E%3C/svg%3E">'
assert readability['textContent'] == 'Some text'
assert readability['title'] == 'Title'