diff --git a/modules/zim/downloader.py b/modules/zim/downloader.py index e47d3674..ce896495 100644 --- a/modules/zim/downloader.py +++ b/modules/zim/downloader.py @@ -2,20 +2,38 @@ from http import HTTPStatus from typing import List from modules.zim import lib -from wrolpi.common import logger, aiohttp_get, background_task, get_html_soup +from wrolpi.common import logger, aiohttp_get, get_html_soup from wrolpi.downloader import Downloader, Download, DownloadResult __all__ = ['KiwixCatalogDownloader', 'KiwixZimDownloader', 'kiwix_zim_downloader', 'kiwix_catalog_downloader'] -from wrolpi.files.lib import refresh_files, upsert_file -from wrolpi.vars import PYTEST +from wrolpi.files.lib import upsert_file +from wrolpi.vars import DOWNLOAD_USER_AGENT logger = logger.getChild(__name__) +# Headers to use when issuing Kiwix HTTP requests. +headers = { + 'accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', + 'accept-language': 'en-US,en;q=0.9', + 'dnt': '1', + 'priority': 'u=0, i', + 'sec-ch-ua': '"Chromium";v="129", "Not=A?Brand";v="8"', + 'sec-ch-ua-mobile': '?0', + 'sec-ch-ua-platform': '"Windows"', + 'sec-fetch-dest': 'document', + 'sec-fetch-mode': 'navigate', + 'sec-fetch-site': 'none', + 'sec-fetch-user': '?1', + 'sec-gpc': '1', + 'upgrade-insecure-requests': '1', + 'user-agent': DOWNLOAD_USER_AGENT, +} + async def fetch_hrefs(url: str) -> List[str]: try: - async with aiohttp_get(url, timeout=20) as response: + async with aiohttp_get(url, headers=headers, timeout=20) as response: status = response.status content = await response.content.read() if status != HTTPStatus.OK: diff --git a/modules/zim/kiwix.py b/modules/zim/kiwix.py index f2dd8ba8..12bad6a6 100644 --- a/modules/zim/kiwix.py +++ b/modules/zim/kiwix.py @@ -38,53 +38,56 @@ WIKIBOOKS_LANGAUGES = ['af', 'ar', 'az', 'ba', 'be', 'bg', 'bn', 'bs', 'ca', 'cs 'ms', 'nb', 'ne', 'nl', 'oc', 'pa', 'pl', 'pt', 'ro', 'ru', 'sa', 'si', 'sk', 'sl', 'sq', 'sr', 'sv', 'ta', 'te', 'tg', 'th', 'tl', 'tr', 'tt', 'uk', 'ur', 'vi', 'zh'] +MB = 1048576 +GB = MB * 1024 + KIWIX_CATALOG = [ { 'name': 'Wikibooks (with images)', 'languages': WIKIBOOKS_LANGAUGES, - 'size': 4.3 * 1073741824, + 'size': 4.3 * GB, }, { 'name': 'Wikibooks (no images)', 'languages': WIKIBOOKS_LANGAUGES, - 'size': 3.4 * 1073741824, + 'size': 3.4 * GB, }, { 'name': 'Wikipedia (with images)', 'languages': WIKIPEDIA_LANGUAGES, - 'size': 94 * 1073741824, + 'size': 102 * GB, }, { 'name': 'Wikipedia (mini)', 'languages': WIKIPEDIA_LANGUAGES, - 'size': 54 * 1073741824, + 'size': 6.9 * GB, }, { 'name': 'Wikipedia (no images)', 'languages': WIKIPEDIA_LANGUAGES, - 'size': 50 * 1073741824, + 'size': 53 * GB, }, { 'name': 'Wikiversity (with images)', 'languages': ['ar', 'cs', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hi', 'it', 'ja', 'ko', 'pt', 'ru', 'sl', 'sv', 'zh'], - 'size': 2.3 * 1073741824, + 'size': 2.3 * GB, }, { 'name': 'Wikiversity (no images)', 'languages': ['ar', 'cs', 'de', 'el', 'en', 'es', 'fi', 'fr', 'hi', 'it', 'ja', 'ko', 'pt', 'ru', 'sl', 'sv', 'zh'], - 'size': 1.8 * 1073741824, + 'size': 1.8 * GB, }, { 'name': 'Wikitionary (no images)', 'languages': WIKITIONARY_LANGUAGES, - 'size': 7.3 * 1073741824, + 'size': 7.3 * GB, }, { 'name': 'Wikitionary (with images)', 'languages': WIKITIONARY_LANGUAGES, - 'size': 8.0 * 1073741824, + 'size': 8.0 * GB, }, { 'name': 'Wikisource', @@ -93,32 +96,32 @@ KIWIX_CATALOG = [ 'is', 'it', 'ja', 'kn', 'ko', 'la', 'li', 'lt', 'mk', 'ml', 'mr', 'nan', 'nb', 'nl', 'or', 'pa', 'pl', 'pms', 'pt', 'ro', 'ru', 'sa', 'sah', 'sk', 'sl', 'sr', 'sv', 'ta', 'te', 'th', 'tr', 'uk', 'vec', 'vi', 'yi', 'zh', 'zh-min-nan'], - 'size': 16 * 1073741824, + 'size': 16 * GB, }, { 'name': 'Raspberry Pi (Stack Exchange)', 'languages': ['en'], - 'size': 300 * 1048576, + 'size': 300 * MB, }, { 'name': 'Vikidia', 'languages': ['ca', 'en', 'es', 'eu', 'fr', 'it', 'ru', 'scn'], - 'size': 711 * 1048576, + 'size': 711 * MB, }, { 'name': 'Stackoverflow (Stack Exchange)', 'languages': ['en', 'es', 'ja', 'pt', 'ru'], - 'size': 76 * 1073741824, + 'size': 76 * GB, }, { 'name': 'Superuser (Stack Exchange)', 'languages': ['en'], - 'size': 3 * 1073741824, + 'size': 3 * GB, }, { 'name': 'iFixit', 'languages': ['de', 'en', 'es', 'fr', 'it', 'ja', 'ko', 'nl', 'pt', 'ru', 'tr', 'zh'], - 'size': 3 * 1073741824, + 'size': 3 * GB, }, { 'name': 'Gutenberg', @@ -127,83 +130,88 @@ KIWIX_CATALOG = [ 'grc', 'he', 'hu', 'ia', 'ilo', 'is', 'it', 'iu', 'ja', 'kha', 'kld', 'ko', 'la', 'mi', 'mul', 'myn', 'nah', 'nai', 'nap', 'nav', 'nl', 'no', 'oc', 'oji', 'pl', 'pt', 'rmr', 'ro', 'ru', 'sa', 'sl', 'sr', 'sv', 'te', 'tl', 'zh'], - 'size': 70 * 1073741824, + 'size': 72 * GB, }, { 'name': 'Amateur Radio (Stack Exchange)', 'languages': ['en'], - 'size': 62 * 1048576, + 'size': 62 * MB, }, { 'name': 'DIY (Stack Exchange)', 'languages': ['en'], - 'size': 930 * 1048576, + 'size': 930 * MB, }, { 'name': 'Electronics (Stack Exchange)', 'languages': ['en'], - 'size': 1.4 * 1073741824, + 'size': 1.4 * GB, }, { 'name': 'Unix (Stack Exchange)', 'languages': ['en'], - 'size': 1.1 * 1073741824, + 'size': 1.1 * GB, }, { 'name': 'Ask Ubuntu (Stack Exchange)', 'languages': ['en'], - 'size': 1.9 * 1073741824, + 'size': 1.9 * GB, }, { 'name': 'Bicycles (Stack Exchange)', 'languages': ['en'], - 'size': 350 * 1048576, + 'size': 350 * MB, }, { 'name': 'Biology (Stack Exchange)', 'languages': ['en'], - 'size': 350 * 1048576, + 'size': 350 * MB, }, { 'name': 'Arduino (Stack Exchange)', 'languages': ['en'], - 'size': 240 * 1048576, + 'size': 240 * MB, }, { 'name': '3D Printing (Stack Exchange)', 'languages': ['en'], - 'size': 100 * 1048576, + 'size': 100 * MB, }, { 'name': 'MD Wiki (Wiki Project Med Foundation)', 'languages': ['en'], - 'size': 5.9 * 1073741824, + 'size': 5.9 * GB, }, { 'name': 'WikiMed Medical Encyclopedia', 'languages': ['ar', 'as', 'bn', 'bpy', 'de', 'en', 'es', 'fa', 'fr', 'gu', 'hi', 'it', 'ja', 'kn', 'lo', 'ml', 'mr', 'or', 'pa', 'pt', 'ru', 'ta', 'te', 'uk', 'ur', 'zh'], - 'size': 6.3 * 1073741824, + 'size': 6.3 * GB, }, { 'name': 'Post Disaster Resource Library', 'languages': ['en'], - 'size': 763 * 1048576, + 'size': 763 * MB, }, { 'name': 'Military Medicine', 'languages': ['en'], - 'size': 82 * 1048576, + 'size': 82 * MB, }, { 'name': 'WikEm (Global Emergency Medicine Wiki)', 'languages': ['en'], - 'size': 42 * 1048576, + 'size': 42 * MB, }, { 'name': 'Health (Stack Exchange)', 'languages': ['en'], - 'size': 57 * 1048576, + 'size': 57 * MB, + }, + { + 'name': 'Khan Academy', + 'languages': ['en', 'es', 'fr'], + 'size': 168 * GB, }, ] diff --git a/modules/zim/lib.py b/modules/zim/lib.py index 34031e4a..3219081b 100644 --- a/modules/zim/lib.py +++ b/modules/zim/lib.py @@ -354,6 +354,8 @@ async def subscribe(name: str, language: str, session: Session = None, url = f'https://download.kiwix.org/zim/other/wikem_{language}_all_maxi_' elif name == 'Health (Stack Exchange)': url = f'https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_{language}_all_' + elif name == 'Khan Academy': + url = f'https://download.kiwix.org/zim/other/khanacademy_{language}_all_' else: raise ValueError(f'{name} not a valid Kiwix subscription name!') @@ -452,6 +454,8 @@ def zim_download_url_to_name(url: str) -> Tuple[str, str]: name = 'WikEm (Global Emergency Medicine Wiki)' elif project == 'health.stackexchange.com' and flavor == '_all_': name = 'Health (Stack Exchange)' + elif project == 'khanacademy' and flavor == '_all_': + name = 'Khan Academy' else: raise RuntimeError(f'Could not find name for Zim URL: {url} {project=} {flavor=}') diff --git a/modules/zim/test/test_lib.py b/modules/zim/test/test_lib.py index 94c2aaea..dacfad0b 100644 --- a/modules/zim/test/test_lib.py +++ b/modules/zim/test/test_lib.py @@ -343,6 +343,8 @@ async def test_zim_tag_migration(await_switches, test_session, test_directory, z ('WikEm (Global Emergency Medicine Wiki)', 'en')), ('https://download.kiwix.org/zim/stack_exchange/health.stackexchange.com_en_all_', ('Health (Stack Exchange)', 'en')), + ('https://download.kiwix.org/zim/other/khanacademy_en_all_', + ('Khan Academy', 'en')), ] ) def test_zim_download_url_to_name(url, expected):