rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-25 18:40:06 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Scrape: honor <meta> charset when decoding pages (#800)

Browse source 
aiohttp's response.text() ignores <meta> charset declarations and assumes
UTF-8 when the Content-Type header has no charset, so legacy Latin-1 pages
crashed the Scrape downloader with UnicodeDecodeError. Decode with
UnicodeDammit instead: header charset first, then the page's declared
charset, then detection.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-10-08 19:39:28 -06:00 • committed by Roland
parent 5ec85c3e04
commit 22b8b9c878
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
2 changed files with 23 additions and 1 deletions
Download patch file Download diff file Expand all files Collapse all files

17
wrolpi/files/test/test_scrape_downloader.py
View file

@ -261,3 +261,20 @@ def test_finalize_returns_success_with_downloads_and_settings(test_session, test
assert result.settings['suffix'] == '.pdf' # original setting preserved
assert result.error == 'Reached max page count.'
assert result.location == f'/files?folders={executed.destination}'
@pytest.mark.asyncio
@pytest.mark.parametrize('encoding,charset_tag', [
('iso-8859-1', '<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1">'),
('iso-8859-1', '<meta charset="ISO-8859-1">'),
('utf-8', ''),
])
async def test_fetch_html_decodes_declared_charset(test_directory, simple_file_server, encoding, charset_tag):
"""fetch_html honors a <meta> charset when the Content-Type header has none, so non-ASCII links survive."""
html = f'<html><head>{charset_tag}</head><body><a href="zurück.txt">Zurück</a></body></html>'
(test_directory / 'page.html').write_bytes(html.encode(encoding))
host, port = simple_file_server.server_address
content = await ScrapeHTMLDownloader.fetch_html(f'http://127.0.0.1:{port}/page.html')
assert '<a href="zurück.txt">Zurück</a>' in content

7
wrolpi/scrape_downloader.py
View file

@ -4,6 +4,7 @@ from dataclasses import dataclass
from typing import List
from urllib.parse import urlparse
from bs4.dammit import UnicodeDammit
from sqlalchemy.orm import Session
from wrolpi.common import logger, get_html_soup, aiohttp_get
@ -64,8 +65,12 @@ class ScrapeHTMLDownloader(Downloader):
@staticmethod
async def fetch_html(url: str) -> str:
"""Fetch and decode a page. A charset in the Content-Type header wins, then the page's
<meta> charset; aiohttp alone ignores <meta> and assumes UTF-8."""
async with aiohttp_get(url, timeout=60 * 5) as response:
return await response.text()
body = await response.read()
known_encodings = [response.charset] if response.charset else []
return UnicodeDammit(body, known_definite_encodings=known_encodings, is_html=True).unicode_markup
def prepare_download(self, session: Session, download: Download) -> PreparedScrape:
"""Validate settings and ensure the destination directory exists.