diff --git a/wrolpi/files/test/test_scrape_downloader.py b/wrolpi/files/test/test_scrape_downloader.py index 402583d3..33e48151 100644 --- a/wrolpi/files/test/test_scrape_downloader.py +++ b/wrolpi/files/test/test_scrape_downloader.py @@ -261,3 +261,20 @@ def test_finalize_returns_success_with_downloads_and_settings(test_session, test assert result.settings['suffix'] == '.pdf' # original setting preserved assert result.error == 'Reached max page count.' assert result.location == f'/files?folders={executed.destination}' + + +@pytest.mark.asyncio +@pytest.mark.parametrize('encoding,charset_tag', [ + ('iso-8859-1', ''), + ('iso-8859-1', ''), + ('utf-8', ''), +]) +async def test_fetch_html_decodes_declared_charset(test_directory, simple_file_server, encoding, charset_tag): + """fetch_html honors a charset when the Content-Type header has none, so non-ASCII links survive.""" + html = f'
{charset_tag}Zurück' + (test_directory / 'page.html').write_bytes(html.encode(encoding)) + host, port = simple_file_server.server_address + + content = await ScrapeHTMLDownloader.fetch_html(f'http://127.0.0.1:{port}/page.html') + + assert 'Zurück' in content diff --git a/wrolpi/scrape_downloader.py b/wrolpi/scrape_downloader.py index dbb0039b..0c74b8d3 100644 --- a/wrolpi/scrape_downloader.py +++ b/wrolpi/scrape_downloader.py @@ -4,6 +4,7 @@ from dataclasses import dataclass from typing import List from urllib.parse import urlparse +from bs4.dammit import UnicodeDammit from sqlalchemy.orm import Session from wrolpi.common import logger, get_html_soup, aiohttp_get @@ -64,8 +65,12 @@ class ScrapeHTMLDownloader(Downloader): @staticmethod async def fetch_html(url: str) -> str: + """Fetch and decode a page. A charset in the Content-Type header wins, then the page's + charset; aiohttp alone ignores and assumes UTF-8.""" async with aiohttp_get(url, timeout=60 * 5) as response: - return await response.text() + body = await response.read() + known_encodings = [response.charset] if response.charset else [] + return UnicodeDammit(body, known_definite_encodings=known_encodings, is_html=True).unicode_markup def prepare_download(self, session: Session, download: Download) -> PreparedScrape: """Validate settings and ensure the destination directory exists.