From b273e7135f91926beffce26e19a764b0155e68d9 Mon Sep 17 00:00:00 2001 From: Roland Date: Thu, 15 Oct 2026 18:29:18 -0600 Subject: [PATCH] Data URI placeholders survive Readability's lazy-image cleanup Readability's _fixLazyImages removes an src whose base64 payload is shorter than 133 characters when another attribute names an image file (alt="cat.png", title, a leftover data-src), treating it as a lazy-load placeholder. The ~50 character placeholder triggered this, so those images lost their inline data. The token is now 160 hex characters. Co-Authored-By: Claude Opus 5.5 --- docker/archive/main.py | 6 ++++-- modules/archive/lib.py | 6 ++++-- modules/archive/test/test_lib.py | 14 ++++++++++++++ 3 files changed, 22 insertions(+), 4 deletions(-) diff --git a/docker/archive/main.py b/docker/archive/main.py index 74574340..ca9f892c 100644 --- a/docker/archive/main.py +++ b/docker/archive/main.py @@ -191,14 +191,16 @@ async def call_single_file(url, compress: bool = False) -> (bytes, bytes): # The prefix and payload of a base64 data URI. Keep in sync with modules/archive/lib.py. BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)') -DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{32}N\d+X') +DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X') def strip_data_uris(html: bytes) -> (bytes, dict): """Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one.""" - token = secrets.token_hex(16) + # At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another + # attribute names an image file (e.g. alt="cat.png"). + token = secrets.token_hex(80) payloads = dict() def replace(match): diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 55cc5a2a..4833091b 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -1561,14 +1561,16 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) -> # The prefix and payload of a base64 data URI. Keep in sync with docker/archive/main.py. BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)') -DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{32}N\d+X') +DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X') def strip_data_uris(html: bytes) -> Tuple[bytes, Dict[str, bytes]]: """Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one.""" - token = secrets.token_hex(16) + # At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another + # attribute names an image file (e.g. alt="cat.png"). + token = secrets.token_hex(80) payloads = dict() def replace(match): diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index c912b752..0a49dee3 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -1513,3 +1513,17 @@ async def test_html_to_readability_forged_placeholder(): assert readability['content'] == html assert readability['content'].count(png) == 1 assert readability['textContent'] == 'WROLPIDATAURI0X WROLPIDATAURI99X' + + +def test_strip_data_uris_placeholder_length(): + """Readability's _fixLazyImages removes an `src` whose base64 payload is shorter than 133 characters + when another attribute names an image file (e.g. alt="cat.png"), so a placeholder must be at least that + long or the image is lost before it can be restored.""" + png = 'iVBORw0KGgo' + 'A' * 4000 + html, payloads = lib.strip_data_uris(f'cat.png'.encode()) + + (placeholder,) = payloads + assert len(placeholder) >= 133 + assert html == f'cat.png'.encode() + assert lib.restore_data_uris({'content': html.decode()}, payloads)['content'] \ + == f'cat.png'