From 22b8b9c8781a2e83eb5fb9e61b6514d5438d574e Mon Sep 17 00:00:00 2001
From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com>
Date: Thu, 8 Oct 2026 19:39:28 -0600
Subject: [PATCH] Scrape: honor <meta> charset when decoding pages (#800)

aiohttp's response.text() ignores <meta> charset declarations and assumes
UTF-8 when the Content-Type header has no charset, so legacy Latin-1 pages
crashed the Scrape downloader with UnicodeDecodeError. Decode with
UnicodeDammit instead: header charset first, then the page's declared
charset, then detection.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
---
 wrolpi/files/test/test_scrape_downloader.py | 17 +++++++++++++++++
 wrolpi/scrape_downloader.py                 |  7 ++++++-
 2 files changed, 23 insertions(+), 1 deletion(-)

diff --git a/wrolpi/files/test/test_scrape_downloader.py b/wrolpi/files/test/test_scrape_downloader.py
index 402583d3..33e48151 100644
--- a/wrolpi/files/test/test_scrape_downloader.py
+++ b/wrolpi/files/test/test_scrape_downloader.py
@@ -261,3 +261,20 @@ def test_finalize_returns_success_with_downloads_and_settings(test_session, test
     assert result.settings['suffix'] == '.pdf'    # original setting preserved
     assert result.error == 'Reached max page count.'
     assert result.location == f'/files?folders={executed.destination}'
+
+
+@pytest.mark.asyncio
+@pytest.mark.parametrize('encoding,charset_tag', [
+    ('iso-8859-1', '<meta http-equiv="content-type" content="text/html; charset=ISO-8859-1">'),
+    ('iso-8859-1', '<meta charset="ISO-8859-1">'),
+    ('utf-8', ''),
+])
+async def test_fetch_html_decodes_declared_charset(test_directory, simple_file_server, encoding, charset_tag):
+    """fetch_html honors a <meta> charset when the Content-Type header has none, so non-ASCII links survive."""
+    html = f'<html><head>{charset_tag}</head><body><a href="zurück.txt">Zurück</a></body></html>'
+    (test_directory / 'page.html').write_bytes(html.encode(encoding))
+    host, port = simple_file_server.server_address
+
+    content = await ScrapeHTMLDownloader.fetch_html(f'http://127.0.0.1:{port}/page.html')
+
+    assert '<a href="zurück.txt">Zurück</a>' in content
diff --git a/wrolpi/scrape_downloader.py b/wrolpi/scrape_downloader.py
index dbb0039b..0c74b8d3 100644
--- a/wrolpi/scrape_downloader.py
+++ b/wrolpi/scrape_downloader.py
@@ -4,6 +4,7 @@ from dataclasses import dataclass
 from typing import List
 from urllib.parse import urlparse
 
+from bs4.dammit import UnicodeDammit
 from sqlalchemy.orm import Session
 
 from wrolpi.common import logger, get_html_soup, aiohttp_get
@@ -64,8 +65,12 @@ class ScrapeHTMLDownloader(Downloader):
 
     @staticmethod
     async def fetch_html(url: str) -> str:
+        """Fetch and decode a page.  A charset in the Content-Type header wins, then the page's
+        <meta> charset; aiohttp alone ignores <meta> and assumes UTF-8."""
         async with aiohttp_get(url, timeout=60 * 5) as response:
-            return await response.text()
+            body = await response.read()
+            known_encodings = [response.charset] if response.charset else []
+        return UnicodeDammit(body, known_definite_encodings=known_encodings, is_html=True).unicode_markup
 
     def prepare_download(self, session: Session, download: Download) -> PreparedScrape:
         """Validate settings and ensure the destination directory exists.