From 4810b8bc50f4882b60d15adac670d168168c31fb Mon Sep 17 00:00:00 2001 From: Roland Date: Tue, 6 Oct 2026 16:43:34 -0600 Subject: [PATCH] repos: close the remaining submodule SSRF gaps * .gitmodules is read NUL-delimited, and an unreadable record refuses the submodules. A submodule name with a space (`[submodule "a b"]`) made `--get-regexp`'s "key value" output misfile its URL as a path, so the URL was never checked. * Submodule URLs must have a strict shape every parser reads the same way (no userinfo, backslash, query, fragment, or whitespace), so a URL cannot be checked as one host and fetched from another. * Each checked host is pinned (http.curloptResolve) to the addresses that were checked, so DNS cannot change between the check and git's fetch. Co-Authored-By: Claude Opus 5.5 --- modules/repos/downloader.py | 33 ++++++++---- modules/repos/lib.py | 89 +++++++++++++++++++++---------- modules/repos/test/test_extras.py | 49 +++++++++++++++++ modules/repos/test/test_search.py | 4 +- 4 files changed, 135 insertions(+), 40 deletions(-) diff --git a/modules/repos/downloader.py b/modules/repos/downloader.py index 8bf5357d..d081d706 100644 --- a/modules/repos/downloader.py +++ b/modules/repos/downloader.py @@ -13,7 +13,7 @@ from wrolpi.dates import now from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, make_progress_callback from wrolpi.errors import DownloadError from .lib import git_command, git_env, find_readme, get_repo_size, is_valid_branch, read_readme_text, \ - submodule_url_error + submodule_fetch_config from .models import Repository __all__ = ['GitDownloader', 'git_downloader', 'PreparedRepo', 'ExecutedRepo', 'parse_git_progress'] @@ -189,23 +189,33 @@ class GitDownloader(Downloader): raise DownloadError(f'Submodules are nested more than {MAXIMUM_SUBMODULE_DEPTH} levels deep') git = lambda *args, **kwargs: self._git(download, ctx, directory, *args, **kwargs) # noqa: E731 - output = await git('config', '--file', '.gitmodules', '--get-regexp', r'^submodule\..*\.(url|path)$', + # NUL-delimited: a submodule's name and URL can contain spaces and newlines. Each record is "key\nvalue". + output = await git('config', '--file', '.gitmodules', '-z', '--get-regexp', r'^submodule\..*\.(url|path)$', check=False) urls, paths = [], [] - for line in output.splitlines(): - key, _, value = line.partition(' ') - (urls if key.endswith('.url') else paths).append(value) + for record in output.split('\0'): + if not record: + continue + key, newline, value = record.partition('\n') + if not newline or not key.startswith('submodule.'): + raise DownloadError(f'Refusing to download submodules, cannot read .gitmodules: {record!r}') + if key.endswith('.url'): + urls.append(value) + elif key.endswith('.path'): + paths.append(value) + else: + raise DownloadError(f'Refusing to download submodules, cannot read .gitmodules: {record!r}') repo_host = (urlparse(prepared.url).hostname or '').lower() - for url in urls: - if error := await asyncio.to_thread(submodule_url_error, url, repo_host): - raise DownloadError(f'Refusing to download submodule {url}: {error}') + try: + fetch_config = await asyncio.to_thread(submodule_fetch_config, urls, repo_host) + except ValueError as e: + raise DownloadError(str(e)) await git('submodule', 'sync') args = ['submodule', 'update', '--init', '--force', '--progress'] if prepared.mode == 'snapshot': args.extend(['--depth', '1']) - # A checked host could otherwise redirect git into WROLPi's network. - await git(*args, progress=True, config=('http.followRedirects=false',)) + await git(*args, progress=True, config=fetch_config) for path in paths: submodule = (directory / path).resolve() @@ -278,7 +288,8 @@ class GitDownloader(Downloader): if executed.submodules_error: # The repo is current, but the download is retried so its submodules are too. return DownloadResult(success=False, location=repo.location, - error=f'The repo was updated, but its submodules were not: {executed.submodules_error}') + error='The repo was updated, but its submodules were not: ' + f'{executed.submodules_error}') return DownloadResult(success=True, location=repo.location) diff --git a/modules/repos/lib.py b/modules/repos/lib.py index 561e222f..c1f773e0 100644 --- a/modules/repos/lib.py +++ b/modules/repos/lib.py @@ -33,10 +33,10 @@ logger = logger.getChild(__name__) __all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'create_repository', 'update_repository', 'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', - 'repo_destination_variables', 'is_valid_branch', 'submodule_url_error', 'read_readme_text', - 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', 'get_repo_root', - 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', 'get_repos_config', - 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY'] + 'repo_destination_variables', 'is_valid_branch', 'submodule_url_error', 'submodule_fetch_config', + 'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', + 'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', + 'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY'] DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value @@ -128,33 +128,67 @@ def resolve_host_addresses(host: str) -> List[str]: return list({i[4][0] for i in socket.getaddrinfo(host, 443, type=socket.SOCK_STREAM)}) +# Submodule URLs must have a shape that every URL parser (Python's, git's, curl's) reads the same way: no userinfo, +# backslashes, queries, fragments or whitespace. +ABSOLUTE_SUBMODULE_URL_RE = re.compile( + r'^https://(?P[A-Za-z0-9.-]+|\[[0-9A-Fa-f:.]+\])' + r'(?::(?P[0-9]{1,5}))?' + r'(?:/[A-Za-z0-9._~%!$&\'()*+,;=:/-]*)?$') +RELATIVE_SUBMODULE_URL_RE = re.compile(r'^\.\.?/[A-Za-z0-9._~%/-]*$') + + +def _check_submodule_url(url: str, repo_host: str) -> Tuple[Optional[str], Optional[Tuple[str, int, List[str]]]]: + """Returns why the URL must not be fetched (or None), and the (host, port, addresses) to pin it to (or None).""" + if RELATIVE_SUBMODULE_URL_RE.match(url): + # Relative to the repo's own remote. + return None, None + match = ABSOLUTE_SUBMODULE_URL_RE.match(url) + if not match: + return 'only plain https:// submodule URLs can be downloaded', None + host = match.group('host').lower() + port = int(match.group('port') or 443) + if host == repo_host: + # The user chose this host. + return None, None + try: + addresses = [ipaddress.ip_address(host.strip('[]'))] + pin = None # An address needs no DNS. + except ValueError: + try: + addresses = [ipaddress.ip_address(i.split('%')[0]) for i in resolve_host_addresses(host)] + except (OSError, ValueError): + return f'could not resolve {host}', None + pin = (host, port, [str(i) for i in addresses]) + if not addresses or not all(i.is_global for i in addresses): + return f'{host} is not a public address', None + return None, pin + + def submodule_url_error(url: str, repo_host: str) -> Optional[str]: """Why a submodule URL must not be fetched, or None when it may be. A submodule's URL is chosen by whoever controls the repo, so it may not make WROLPi fetch from its own network (e.g. `https://localhost:8443/...` or `https://10.0.0.1/...`). Allowed: URLs relative to the repo's remote, - the repo's own host (which the user chose), and hosts with only public addresses. (A host which resolves - differently when git fetches it is not prevented.)""" - if url.startswith(('./', '../')): - return None - parsed = urlparse(url) - if parsed.scheme != 'https': - return 'only https:// submodules can be downloaded' - host = (parsed.hostname or '').lower() - if not host: - return 'the URL has no host' - if host == repo_host: - return None - try: - addresses = [ipaddress.ip_address(host)] - except ValueError: - try: - addresses = [ipaddress.ip_address(i.split('%')[0]) for i in resolve_host_addresses(host)] - except (OSError, ValueError): - return f'could not resolve {host}' - if not addresses or not all(i.is_global for i in addresses): - return f'{host} is not a public address' - return None + the repo's own host (which the user chose), and hosts with only public addresses.""" + return _check_submodule_url(url, repo_host)[0] + + +def submodule_fetch_config(urls: List[str], repo_host: str) -> tuple: + """git config for fetching submodules with these URLs. Raises ValueError if any URL must not be fetched. + + Each checked host is pinned to the addresses that were checked, so its DNS cannot change before git fetches it, + and redirects are not followed, so a checked host cannot send git elsewhere.""" + config = ['http.followRedirects=false'] + pinned = set() + for url in urls: + error, pin = _check_submodule_url(url, repo_host) + if error: + raise ValueError(f'Refusing to download submodule {url}: {error}') + if pin and (pin[0], pin[1]) not in pinned: + pinned.add((pin[0], pin[1])) + addresses = ','.join(f'[{i}]' if ':' in i else i for i in pin[2]) + config.append(f'http.curloptResolve={pin[0]}:{pin[1]}:{addresses}') + return tuple(config) def repo_url_key(url: str) -> str: @@ -351,7 +385,8 @@ def create_repository(session: Session, url: str, tag_name: str = None, frequenc directory = directory or collection.format_destination(tag_name) _validate_directory(directory) if session.query(Collection).filter(Collection.directory == directory, Collection.id != collection.id).first(): - raise RepoConflict(f'Another collection already uses the directory {get_relative_to_media_directory(directory)}') + relative = get_relative_to_media_directory(directory) + raise RepoConflict(f'Another collection already uses the directory {relative}') collection.directory = directory download = collection.get_or_create_download(session, parsed.url, frequency, downloader_name='git') diff --git a/modules/repos/test/test_extras.py b/modules/repos/test/test_extras.py index 1fff5d7d..9e181f2f 100644 --- a/modules/repos/test/test_extras.py +++ b/modules/repos/test/test_extras.py @@ -203,6 +203,15 @@ async def test_archive_after_session_closes(test_session, async_client, git_remo ('http://codeberg.org/owner/library', None, False), ('ssh://git@codeberg.org/owner/library', None, False), ('file:///etc', None, False), + # URLs which parsers disagree about are refused. + ('https://codeberg.org\\@10.0.0.5/library', ['217.197.91.145'], False), + ('https://user@codeberg.org/owner/library', ['217.197.91.145'], False), + ('https://10.0.0.5#@codeberg.org/library', ['217.197.91.145'], False), + ('https://codeberg.org/owner/library?x=@10.0.0.5', ['217.197.91.145'], False), + ('https://codeberg.org/owner/lib rary', ['217.197.91.145'], False), + ('https://codeberg.org:443/owner/library', ['217.197.91.145'], True), + ('../../evil@10.0.0.5/x', None, False), + ('..\\evil', None, False), ]) def test_submodule_url_error(url, resolved, allowed): """A submodule URL comes from the (untrusted) repo; it may not make WROLPi fetch from its own network.""" @@ -252,3 +261,43 @@ async def test_nested_submodule_to_private_address(test_session, git_remote_fact # The first level (the repo's own host) was fetched; the nested submodule was not. assert (repo.directory / 'vendor/library/README.md').is_file() assert not (repo.directory / 'vendor/library/nested/README.md').exists() + + +@pytest.mark.parametrize('gitmodules', [ + # A space in the name must not hide the URL. + '[submodule "a b"]\n\tpath = vendor/library\n\turl = https://10.0.0.5/library\n', + # Nor a newline in the URL. + '[submodule "library"]\n\tpath = vendor/library\n\turl = "https://10.0.0.5/library\\nsubmodule.x.path y"\n', +]) +@pytest.mark.asyncio +async def test_submodule_url_cannot_hide(test_session, git_remote_factory, repos_download_manager, gitmodules): + """Every submodule URL is checked, however the .gitmodules is written.""" + library = git_remote_factory('library') + remote = git_remote_factory() + remote.add_submodule('vendor/library', library) + (remote.path / '.gitmodules').write_text(gitmodules) + remote.git('add', '.gitmodules') + remote.git('commit', '--quiet', '-m', 'Tricky .gitmodules') + + repo = lib.create_repository(test_session, remote.url, submodules=True) + await repos_download_manager.wait_for_all_downloads() + test_session.expire_all() + + download = test_session.query(Download).one() + assert download.status == DownloadStatus.deferred + assert 'Refusing' in download.error + assert not (repo.directory / 'vendor/library/README.md').exists() + + + +def test_submodule_fetch_config(): + """Checked hosts are pinned to the addresses that were checked, so DNS cannot change between the check and git's + fetch.""" + with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['217.197.91.145', '2606:4700::1111']): + config = lib.submodule_fetch_config(['https://codeberg.org/a/b', 'https://codeberg.org:8443/c/d', + '../relative', 'https://git.example.com/own/host'], 'git.example.com') + assert config == ( + 'http.followRedirects=false', + 'http.curloptResolve=codeberg.org:443:217.197.91.145,[2606:4700::1111]', + 'http.curloptResolve=codeberg.org:8443:217.197.91.145,[2606:4700::1111]', + ) diff --git a/modules/repos/test/test_search.py b/modules/repos/test/test_search.py index b9a48804..fc42623f 100644 --- a/modules/repos/test/test_search.py +++ b/modules/repos/test/test_search.py @@ -189,5 +189,5 @@ kiwix-serve --port 80 --- #hashtag ''' - assert lib.markdown_to_text(readme) == \ - 'Kiwix Tools\nCI Kiwix is an offline reader. See the docs.\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag' + assert lib.markdown_to_text(readme) == ('Kiwix Tools\nCI Kiwix is an offline reader. See the docs.' + '\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag')