mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 07:26:18 -05:00
Data URI placeholders survive Readability's lazy-image cleanup
Readability's _fixLazyImages removes an <img> src whose base64 payload is shorter than 133 characters when another attribute names an image file (alt="cat.png", title, a leftover data-src), treating it as a lazy-load placeholder. The ~50 character placeholder triggered this, so those images lost their inline data. The token is now 160 hex characters. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
5802f245c0
commit
b273e7135f
3 changed files with 22 additions and 4 deletions
|
|
@ -191,14 +191,16 @@ async def call_single_file(url, compress: bool = False) -> (bytes, bytes):
|
|||
|
||||
# The prefix and payload of a base64 data URI. Keep in sync with modules/archive/lib.py.
|
||||
BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)')
|
||||
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{32}N\d+X')
|
||||
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X')
|
||||
|
||||
|
||||
def strip_data_uris(html: bytes) -> (bytes, dict):
|
||||
"""Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a
|
||||
page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by
|
||||
placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one."""
|
||||
token = secrets.token_hex(16)
|
||||
# At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another
|
||||
# attribute names an image file (e.g. alt="cat.png").
|
||||
token = secrets.token_hex(80)
|
||||
payloads = dict()
|
||||
|
||||
def replace(match):
|
||||
|
|
|
|||
|
|
@ -1561,14 +1561,16 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) ->
|
|||
|
||||
# The prefix and payload of a base64 data URI. Keep in sync with docker/archive/main.py.
|
||||
BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)')
|
||||
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{32}N\d+X')
|
||||
DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X')
|
||||
|
||||
|
||||
def strip_data_uris(html: bytes) -> Tuple[bytes, Dict[str, bytes]]:
|
||||
"""Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a
|
||||
page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by
|
||||
placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one."""
|
||||
token = secrets.token_hex(16)
|
||||
# At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another
|
||||
# attribute names an image file (e.g. alt="cat.png").
|
||||
token = secrets.token_hex(80)
|
||||
payloads = dict()
|
||||
|
||||
def replace(match):
|
||||
|
|
|
|||
|
|
@ -1513,3 +1513,17 @@ async def test_html_to_readability_forged_placeholder():
|
|||
assert readability['content'] == html
|
||||
assert readability['content'].count(png) == 1
|
||||
assert readability['textContent'] == 'WROLPIDATAURI0X WROLPIDATAURI99X'
|
||||
|
||||
|
||||
def test_strip_data_uris_placeholder_length():
|
||||
"""Readability's _fixLazyImages removes an <img> `src` whose base64 payload is shorter than 133 characters
|
||||
when another attribute names an image file (e.g. alt="cat.png"), so a placeholder must be at least that
|
||||
long or the image is lost before it can be restored."""
|
||||
png = 'iVBORw0KGgo' + 'A' * 4000
|
||||
html, payloads = lib.strip_data_uris(f'<img src="data:image/png;base64,{png}" alt="cat.png">'.encode())
|
||||
|
||||
(placeholder,) = payloads
|
||||
assert len(placeholder) >= 133
|
||||
assert html == f'<img src="data:image/png;base64,{placeholder}" alt="cat.png">'.encode()
|
||||
assert lib.restore_data_uris({'content': html.decode()}, payloads)['content'] \
|
||||
== f'<img src="data:image/png;base64,{png}" alt="cat.png">'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue