diff --git a/docker/archive/main.py b/docker/archive/main.py index 18bb48aa..ca9f892c 100644 --- a/docker/archive/main.py +++ b/docker/archive/main.py @@ -11,6 +11,8 @@ import logging import os import os.path import pathlib +import re +import secrets import signal import subprocess import sys @@ -187,6 +189,37 @@ async def call_single_file(url, compress: bool = False) -> (bytes, bytes): return singlefile, page_html +# The prefix and payload of a base64 data URI. Keep in sync with modules/archive/lib.py. +BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)') +DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X') + + +def strip_data_uris(html: bytes) -> (bytes, dict): + """Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a + page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by + placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one.""" + # At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another + # attribute names an image file (e.g. alt="cat.png"). + token = secrets.token_hex(80) + payloads = dict() + + def replace(match): + placeholder = f'WROLPIDATAURI{token}N{len(payloads)}X' + payloads[placeholder] = match.group(2) + return match.group(1) + placeholder.encode() + + return BASE64_DATA_URI.sub(replace, html), payloads + + +def restore_data_uris(readability: dict, payloads: dict) -> dict: + """Put the payloads removed by `strip_data_uris` back into the readability output.""" + for key, value in readability.items(): + if isinstance(value, str): + readability[key] = DATA_URI_PLACEHOLDER.sub( + lambda m: payloads[m.group(0)].decode() if m.group(0) in payloads else m.group(0), value) + return readability + + async def extract_readability(path: str, url: str) -> dict: """ Call the CLI command for readability-extractor. @@ -194,6 +227,9 @@ async def extract_readability(path: str, url: str) -> dict: See https://github.com/ArchiveBox/readability-extractor """ logger.info(f'readability for {url}') + html, data_uri_payloads = strip_data_uris(pathlib.Path(path).read_bytes()) + path = pathlib.Path(path).with_name('readability-input.html') + path.write_bytes(html) cmd = f'readability-extractor {path} "{url}"' logger.debug(f'readability cmd: {cmd}') stdout, stderr, return_code = await check_output(cmd, timeout=3 * 60) @@ -203,7 +239,7 @@ async def extract_readability(path: str, url: str) -> dict: stderr = stderr.decode() if stderr else None raise RuntimeError(f'Failed to extract readability. {stderr}') from e logger.debug(f'done readability for {url}') - return output + return restore_data_uris(output, data_uri_payloads) async def take_screenshot(url: str) -> bytes: diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 78e542ed..4833091b 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -5,13 +5,14 @@ import json import os import pathlib import re +import secrets import shlex import tempfile import zipfile from dataclasses import dataclass, field from datetime import datetime from json import JSONDecodeError -from typing import Optional, Tuple, List, Union +from typing import Optional, Tuple, List, Union, Dict from urllib.parse import urlparse import pytz @@ -1558,10 +1559,42 @@ async def search_domains_by_name(session: Session, name: str, limit: int = 5) -> ] +# The prefix and payload of a base64 data URI. Keep in sync with docker/archive/main.py. +BASE64_DATA_URI = re.compile(rb'(data:[^,;\s"\'()]*(?:;[^,;\s"\'()]*)*;base64,)([A-Za-z0-9+/]+=*)') +DATA_URI_PLACEHOLDER = re.compile(r'WROLPIDATAURI[0-9a-f]{160}N\d+X') + + +def strip_data_uris(html: bytes) -> Tuple[bytes, Dict[str, bytes]]: + """Replace each base64 data URI payload with a placeholder; readability-extractor takes minutes on a + page of large inline images (SingleFile inlines every image). Returns the HTML and the payloads by + placeholder. The placeholder holds a random token, so text in the page cannot be mistaken for one.""" + # At least 133 characters: Readability drops a shorter base64 `src` as a lazy-load placeholder when another + # attribute names an image file (e.g. alt="cat.png"). + token = secrets.token_hex(80) + payloads = dict() + + def replace(match): + placeholder = f'WROLPIDATAURI{token}N{len(payloads)}X' + payloads[placeholder] = match.group(2) + return match.group(1) + placeholder.encode() + + return BASE64_DATA_URI.sub(replace, html), payloads + + +def restore_data_uris(readability: dict, payloads: Dict[str, bytes]) -> dict: + """Put the payloads removed by `strip_data_uris` back into the readability output.""" + for key, value in readability.items(): + if isinstance(value, str): + readability[key] = DATA_URI_PLACEHOLDER.sub( + lambda m: payloads[m.group(0)].decode() if m.group(0) in payloads else m.group(0), value) + return readability + + async def html_to_readability(html: str | bytes, url: str, timeout: int = 120): """Extract the readability dict from the provided HTML.""" + html, data_uri_payloads = strip_data_uris(html.encode() if isinstance(html, str) else html) with tempfile.NamedTemporaryFile('wb', suffix='.html') as singlefile_file: - singlefile_file.write(html.encode() if isinstance(html, str) else html) + singlefile_file.write(html) singlefile_file.flush() os.fsync(singlefile_file.fileno()) @@ -1585,7 +1618,7 @@ async def html_to_readability(html: str | bytes, url: str, timeout: int = 120): e = ChildProcessError(stderr) raise RuntimeError(f'Failed to extract readability from {url}') from e logger.debug(f'done readability for {url}') - return readability + return restore_data_uris(readability, data_uri_payloads) else: logger.error(f'Failed to extract readability for {url}') e = ChildProcessError(stdout if stdout else 'No stdout') diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 77c1e934..0a49dee3 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -1,5 +1,6 @@ import json import pathlib +import re import tempfile from datetime import datetime, timedelta from http import HTTPStatus @@ -19,6 +20,7 @@ from modules.archive.lib import get_or_create_domain_collection, get_new_archive delete_archives_by_file_group_ids, model_archive_result, get_domains from modules.archive.models import Archive from wrolpi.api_utils import CustomJSONEncoder +from wrolpi.cmd import CommandResult from wrolpi.collections import Collection from wrolpi.common import get_wrolpi_config from wrolpi.db import get_db_session @@ -1452,3 +1454,76 @@ def test_search_archives_count_is_driven_by_the_search_index(test_directory, tes (statement, params), = counts plan = [row[3] for row in test_session.execute(text(f'EXPLAIN QUERY PLAN {statement}'), params).fetchall()] assert plan[0].startswith('SCAN fts VIRTUAL TABLE'), plan + + +@pytest.mark.asyncio +async def test_html_to_readability_data_uris(test_directory): + """readability-extractor never receives inline base64 data, which makes it take minutes on a page of + inline images; the data is restored in the readability output.""" + png = 'iVBORw0KGgo' + 'A' * 5000 + '==' + webp = 'UklGRiQAAABXRUJQ' + 'B' * 5000 + html = f'' \ + f'

Some text

' \ + f'' \ + f'' \ + f'' \ + f'
' + + received = list() + + async def fake_run_command(cmd, *args, **kwargs): + # Echo the images back like readability-extractor does in `content`. + page = pathlib.Path(cmd[3]).read_text() + received.append(page) + content = ''.join(re.findall(r']*>', page)) + stdout = json.dumps({'title': 'Title', 'content': content, 'textContent': 'Some text'}).encode() + return CommandResult(return_code=0, cancelled=False, stdout=stdout, stderr=b'', elapsed=0) + + with mock.patch('modules.archive.lib.run_command', fake_run_command): + readability = await lib.html_to_readability(html, 'https://example.com') + + # readability-extractor got the page without the base64 payloads. + assert png not in received[0] and webp not in received[0] + assert 'Some text' in received[0] + assert 'data:image/svg+xml,%3Csvg%3E%3C/svg%3E' in received[0], 'non-base64 data URI should be untouched' + + # The images were restored in the output. + assert readability['content'] == f'' \ + f'' \ + f'' + assert readability['textContent'] == 'Some text' + assert readability['title'] == 'Title' + + +@pytest.mark.asyncio +async def test_html_to_readability_forged_placeholder(): + """Placeholder-like text already in a page is not treated as a placeholder; it cannot index past the + payloads or duplicate an image into the output.""" + png = 'iVBORw0KGgo' + 'A' * 5000 + html = f'

WROLPIDATAURI0X WROLPIDATAURI99X

' + + async def fake_run_command(cmd, *args, **kwargs): + page = pathlib.Path(cmd[3]).read_text() + stdout = json.dumps({'content': page, 'textContent': 'WROLPIDATAURI0X WROLPIDATAURI99X'}).encode() + return CommandResult(return_code=0, cancelled=False, stdout=stdout, stderr=b'', elapsed=0) + + with mock.patch('modules.archive.lib.run_command', fake_run_command): + readability = await lib.html_to_readability(html, 'https://example.com') + + assert readability['content'] == html + assert readability['content'].count(png) == 1 + assert readability['textContent'] == 'WROLPIDATAURI0X WROLPIDATAURI99X' + + +def test_strip_data_uris_placeholder_length(): + """Readability's _fixLazyImages removes an `src` whose base64 payload is shorter than 133 characters + when another attribute names an image file (e.g. alt="cat.png"), so a placeholder must be at least that + long or the image is lost before it can be restored.""" + png = 'iVBORw0KGgo' + 'A' * 4000 + html, payloads = lib.strip_data_uris(f'cat.png'.encode()) + + (placeholder,) = payloads + assert len(placeholder) >= 133 + assert html == f'cat.png'.encode() + assert lib.restore_data_uris({'content': html.decode()}, payloads)['content'] \ + == f'cat.png'