mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Readability extraction no longer takes minutes on image-heavy pages (#838)
* Readability extraction no longer takes minutes on image-heavy pages SingleFile inlines every image as a base64 data URI, and readability-extractor (jsdom) slows to minutes on a page carrying megabytes of them, so these pages hit the readability timeout and were saved without readability text. Replace each base64 payload with a short placeholder before running readability-extractor, then restore the payloads in its output. Applied in both the archive container and the native (Raspberry Pi/Debian) path. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Data URI placeholders cannot be forged by page text The placeholder was a fixed string, so a page containing placeholder-like text could index past the payloads (IndexError, failing the archive on the native path) or repeat a placeholder to copy a large image into the output many times. Each extraction now uses a random token, and text that is not a known placeholder is left as-is. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * Data URI placeholders survive Readability's lazy-image cleanup Readability's _fixLazyImages removes an <img> src whose base64 payload is shorter than 133 characters when another attribute names an image file (alt="cat.png", title, a leftover data-src), treating it as a lazy-load placeholder. The ~50 character placeholder triggered this, so those images lost their inline data. The token is now 160 hex characters. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance
Roland
parent
06e35e1095
commit
1f07b5f944
3 changed files with 148 additions and 4 deletions
|
|
@ -11,6 +11,8 @@ import logging
|
|||
import os
|
||||
import os.path
|
||||
import pathlib
|
||||
import re
|
||||
import secrets
|
||||
import signal
|
||||
import subprocess
|
||||
import sys
|
||||
|
|
@ -187,6 +189,37 @@ async def call_single_file(url, compress: bool = False) -> (bytes, bytes):
|
|||
return singlefile, page_html
|
||||
|
||||
|
||||
# The prefix and payload of a base64 data URI. Keep in sync with modules/archive/lib.py.
|
||||
BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)')
|
||||
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X')
|
||||
|
||||
|
||||
def strip_data_uris(html: bytes) -> (bytes, dict):
|
||||
"""Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a
|
||||
page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by
|
||||
placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one."""
|
||||
# At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another
|
||||
# attribute names an image file (e.g. alt="cat.png").
|
||||
token = secrets.token_hex(80)
|
||||
payloads = dict()
|
||||
|
||||
def replace(match):
|
||||
placeholder = f'WROLPIDATAURI{token}N{len(payloads)}X'
|
||||
payloads[placeholder] = match.group(2)
|
||||
return match.group(1) + placeholder.encode()
|
||||
|
||||
return BASE64_DATA_URI.sub(replace, html), payloads
|
||||
|
||||
|
||||
def restore_data_uris(readability: dict, payloads: dict) -> dict:
|
||||
"""Put the payloads removed by `strip_data_uris` back into the readability output."""
|
||||
for key, value in readability.items():
|
||||
if isinstance(value, str):
|
||||
readability[key] = DATA_URI_PLACEHOLDER.sub(
|
||||
lambda m: payloads[m.group(0)].decode() if m.group(0) in payloads else m.group(0), value)
|
||||
return readability
|
||||
|
||||
|
||||
async def extract_readability(path: str, url: str) -> dict:
|
||||
"""
|
||||
Call the CLI command for readability-extractor.
|
||||
|
|
@ -194,6 +227,9 @@ async def extract_readability(path: str, url: str) -> dict:
|
|||
See https://github.com/ArchiveBox/readability-extractor
|
||||
"""
|
||||
logger.info(f'readability for {url}')
|
||||
html, data_uri_payloads = strip_data_uris(pathlib.Path(path).read_bytes())
|
||||
path = pathlib.Path(path).with_name('readability-input.html')
|
||||
path.write_bytes(html)
|
||||
cmd = f'readability-extractor {path} "{url}"'
|
||||
logger.debug(f'readability cmd: {cmd}')
|
||||
stdout, stderr, return_code = await check_output(cmd, timeout=3 * 60)
|
||||
|
|
@ -203,7 +239,7 @@ async def extract_readability(path: str, url: str) -> dict:
|
|||
stderr = stderr.decode() if stderr else None
|
||||
raise RuntimeError(f'Failed to extract readability. {stderr}') from e
|
||||
logger.debug(f'done readability for {url}')
|
||||
return output
|
||||
return restore_data_uris(output, data_uri_payloads)
|
||||
|
||||
|
||||
async def take_screenshot(url: str) -> bytes:
|
||||
|
|
|
|||
|
|
@ -5,13 +5,14 @@ import json
|
|||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import secrets
|
||||
import shlex
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from json import JSONDecodeError
|
||||
from typing import Optional, Tuple, List, Union
|
||||
from typing import Optional, Tuple, List, Union, Dict
|
||||
from urllib.parse import urlparse
|
||||
|
||||
import pytz
|
||||
|
|
@ -1558,10 +1559,42 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) ->
|
|||
]
|
||||
|
||||
|
||||
# The prefix and payload of a base64 data URI. Keep in sync with docker/archive/main.py.
|
||||
BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)')
|
||||
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X')
|
||||
|
||||
|
||||
def strip_data_uris(html: bytes) -> Tuple[bytes, Dict[str, bytes]]:
|
||||
"""Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a
|
||||
page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by
|
||||
placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one."""
|
||||
# At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another
|
||||
# attribute names an image file (e.g. alt="cat.png").
|
||||
token = secrets.token_hex(80)
|
||||
payloads = dict()
|
||||
|
||||
def replace(match):
|
||||
placeholder = f'WROLPIDATAURI{token}N{len(payloads)}X'
|
||||
payloads[placeholder] = match.group(2)
|
||||
return match.group(1) + placeholder.encode()
|
||||
|
||||
return BASE64_DATA_URI.sub(replace, html), payloads
|
||||
|
||||
|
||||
def restore_data_uris(readability: dict, payloads: Dict[str, bytes]) -> dict:
|
||||
"""Put the payloads removed by `strip_data_uris` back into the readability output."""
|
||||
for key, value in readability.items():
|
||||
if isinstance(value, str):
|
||||
readability[key] = DATA_URI_PLACEHOLDER.sub(
|
||||
lambda m: payloads[m.group(0)].decode() if m.group(0) in payloads else m.group(0), value)
|
||||
return readability
|
||||
|
||||
|
||||
async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
|
||||
"""Extract the readability dict from the provided HTML."""
|
||||
html, data_uri_payloads = strip_data_uris(html.encode() if isinstance(html, str) else html)
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file:
|
||||
singlefile_file.write(html.encode() if isinstance(html, str) else html)
|
||||
singlefile_file.write(html)
|
||||
singlefile_file.flush()
|
||||
os.fsync(singlefile_file.fileno())
|
||||
|
||||
|
|
@ -1585,7 +1618,7 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120):
|
|||
e = ChildProcessError(stderr)
|
||||
raise RuntimeError(f'Failed to extract readability from {url}') from e
|
||||
logger.debug(f'done readability for {url}')
|
||||
return readability
|
||||
return restore_data_uris(readability, data_uri_payloads)
|
||||
else:
|
||||
logger.error(f'Failed to extract readability for {url}')
|
||||
e = ChildProcessError(stdout if stdout else 'No stdout')
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
import pathlib
|
||||
import re
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta
|
||||
from http import HTTPStatus
|
||||
|
|
@ -19,6 +20,7 @@ from modules.archive.lib import get_or_create_domain_collection, get_new_archive
|
|||
delete_archives_by_file_group_ids, model_archive_result, get_domains
|
||||
from modules.archive.models import Archive
|
||||
from wrolpi.api_utils import CustomJSONEncoder
|
||||
from wrolpi.cmd import CommandResult
|
||||
from wrolpi.collections import Collection
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
from wrolpi.db import get_db_session
|
||||
|
|
@ -1452,3 +1454,76 @@ def test_search_archives_count_is_driven_by_the_search_index(test_directory, tes
|
|||
(statement, params), = counts
|
||||
plan = [row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {statement}'), params).fetchall()]
|
||||
assert plan[0].startswith('SCAN fts VIRTUAL TABLE'), plan
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_html_to_readability_data_uris(test_directory):
|
||||
"""readability-extractor never receives inline base64 data, which makes it take minutes on a page of
|
||||
inline images; the data is restored in the readability output."""
|
||||
png = 'iVBORw0KGgo' + 'A' * 5000 + '=='
|
||||
webp = 'UklGRiQAAABXRUJQ' + 'B' * 5000
|
||||
html = f'<html><head><style>.x {{ background: url(data:image/png;base64,{png}) }}</style></head>' \
|
||||
f'<body><article><p>Some text</p>' \
|
||||
f'<img src="data:image/png;base64,{png}">' \
|
||||
f'<img src=\'data:image/webp;base64,{webp}\'>' \
|
||||
f'<img src="data:image/svg+xml,%3Csvg%3E%3C/svg%3E">' \
|
||||
f'</article></body></html>'
|
||||
|
||||
received = list()
|
||||
|
||||
async def fake_run_command(cmd, *args, **kwargs):
|
||||
# Echo the images back like readability-extractor does in `content`.
|
||||
page = pathlib.Path(cmd[3]).read_text()
|
||||
received.append(page)
|
||||
content = ''.join(re.findall(r'<img[^>]*>', page))
|
||||
stdout = json.dumps({'title': 'Title', 'content': content, 'textContent': 'Some text'}).encode()
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=stdout, stderr=b'', elapsed=0)
|
||||
|
||||
with mock.patch('modules.archive.lib.run_command', fake_run_command):
|
||||
readability = await lib.html_to_readability(html, 'https://example.com')
|
||||
|
||||
# readability-extractor got the page without the base64 payloads.
|
||||
assert png not in received[0] and webp not in received[0]
|
||||
assert 'Some text' in received[0]
|
||||
assert 'data:image/svg+xml,%3Csvg%3E%3C/svg%3E' in received[0], 'non-base64 data URI should be untouched'
|
||||
|
||||
# The images were restored in the output.
|
||||
assert readability['content'] == f'<img src="data:image/png;base64,{png}">' \
|
||||
f'<img src=\'data:image/webp;base64,{webp}\'>' \
|
||||
f'<img src="data:image/svg+xml,%3Csvg%3E%3C/svg%3E">'
|
||||
assert readability['textContent'] == 'Some text'
|
||||
assert readability['title'] == 'Title'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_html_to_readability_forged_placeholder():
|
||||
"""Placeholder-like text already in a page is not treated as a placeholder; it cannot index past the
|
||||
payloads or duplicate an image into the output."""
|
||||
png = 'iVBORw0KGgo' + 'A' * 5000
|
||||
html = f'<p>WROLPIDATAURI0X WROLPIDATAURI99X</p><img src="data:image/png;base64,{png}">'
|
||||
|
||||
async def fake_run_command(cmd, *args, **kwargs):
|
||||
page = pathlib.Path(cmd[3]).read_text()
|
||||
stdout = json.dumps({'content': page, 'textContent': 'WROLPIDATAURI0X WROLPIDATAURI99X'}).encode()
|
||||
return CommandResult(return_code=0, cancelled=False, stdout=stdout, stderr=b'', elapsed=0)
|
||||
|
||||
with mock.patch('modules.archive.lib.run_command', fake_run_command):
|
||||
readability = await lib.html_to_readability(html, 'https://example.com')
|
||||
|
||||
assert readability['content'] == html
|
||||
assert readability['content'].count(png) == 1
|
||||
assert readability['textContent'] == 'WROLPIDATAURI0X WROLPIDATAURI99X'
|
||||
|
||||
|
||||
def test_strip_data_uris_placeholder_length():
|
||||
"""Readability's _fixLazyImages removes an <img> `src` whose base64 payload is shorter than 133 characters
|
||||
when another attribute names an image file (e.g. alt="cat.png"), so a placeholder must be at least that
|
||||
long or the image is lost before it can be restored."""
|
||||
png = 'iVBORw0KGgo' + 'A' * 4000
|
||||
html, payloads = lib.strip_data_uris(f'<img src="data:image/png;base64,{png}" alt="cat.png">'.encode())
|
||||
|
||||
(placeholder,) = payloads
|
||||
assert len(placeholder) >= 133
|
||||
assert html == f'<img src="data:image/png;base64,{placeholder}" alt="cat.png">'.encode()
|
||||
assert lib.restore_data_uris({'content': html.decode()}, payloads)['content'] \
|
||||
== f'<img src="data:image/png;base64,{png}" alt="cat.png">'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue