repos: close the remaining submodule SSRF gaps

* .gitmodules is read NUL-delimited, and an unreadable record refuses
  the submodules.  A submodule name with a space (`[submodule "a b"]`)
  made `--get-regexp`'s "key value" output misfile its URL as a path, so
  the URL was never checked.
* Submodule URLs must have a strict shape every parser reads the same
  way (no userinfo, backslash, query, fragment, or whitespace), so a URL
  cannot be checked as one host and fetched from another.
* Each checked host is pinned (http.curloptResolve) to the addresses
  that were checked, so DNS cannot change between the check and git's
  fetch.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-06 16:43:34 -06:00
parent 3b6e962dfc
commit 4810b8bc50
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 135 additions and 40 deletions

View file

@ -13,7 +13,7 @@ from wrolpi.dates import now
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, make_progress_callback
from wrolpi.errors import DownloadError
from .lib import git_command, git_env, find_readme, get_repo_size, is_valid_branch, read_readme_text, \
submodule_url_error
submodule_fetch_config
from .models import Repository
__all__ = ['GitDownloader', 'git_downloader', 'PreparedRepo', 'ExecutedRepo', 'parse_git_progress']
@ -189,23 +189,33 @@ class GitDownloader(Downloader):
raise DownloadError(f'Submodules are nested more than {MAXIMUM_SUBMODULE_DEPTH} levels deep')
git = lambda *args, **kwargs: self._git(download, ctx, directory, *args, **kwargs) # noqa: E731
output = await git('config', '--file', '.gitmodules', '--get-regexp', r'^submodule\..*\.(url|path)$',
# NUL-delimited: a submodule's name and URL can contain spaces and newlines. Each record is "key\nvalue".
output = await git('config', '--file', '.gitmodules', '-z', '--get-regexp', r'^submodule\..*\.(url|path)$',
check=False)
urls, paths = [], []
for line in output.splitlines():
key, _, value = line.partition(' ')
(urls if key.endswith('.url') else paths).append(value)
for record in output.split('\0'):
if not record:
continue
key, newline, value = record.partition('\n')
if not newline or not key.startswith('submodule.'):
raise DownloadError(f'Refusing to download submodules, cannot read .gitmodules: {record!r}')
if key.endswith('.url'):
urls.append(value)
elif key.endswith('.path'):
paths.append(value)
else:
raise DownloadError(f'Refusing to download submodules, cannot read .gitmodules: {record!r}')
repo_host = (urlparse(prepared.url).hostname or '').lower()
for url in urls:
if error := await asyncio.to_thread(submodule_url_error, url, repo_host):
raise DownloadError(f'Refusing to download submodule {url}: {error}')
try:
fetch_config = await asyncio.to_thread(submodule_fetch_config, urls, repo_host)
except ValueError as e:
raise DownloadError(str(e))
await git('submodule', 'sync')
args = ['submodule', 'update', '--init', '--force', '--progress']
if prepared.mode == 'snapshot':
args.extend(['--depth', '1'])
# A checked host could otherwise redirect git into WROLPi's network.
await git(*args, progress=True, config=('http.followRedirects=false',))
await git(*args, progress=True, config=fetch_config)
for path in paths:
submodule = (directory / path).resolve()
@ -278,7 +288,8 @@ class GitDownloader(Downloader):
if executed.submodules_error:
# The repo is current, but the download is retried so its submodules are too.
return DownloadResult(success=False, location=repo.location,
error=f'The repo was updated, but its submodules were not: {executed.submodules_error}')
error='The repo was updated, but its submodules were not: '
f'{executed.submodules_error}')
return DownloadResult(success=True, location=repo.location)

View file

@ -33,10 +33,10 @@ logger = logger.getChild(__name__)
__all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'create_repository',
'update_repository', 'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme',
'repo_destination_variables', 'is_valid_branch', 'submodule_url_error', 'read_readme_text',
'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', 'get_repo_root',
'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', 'get_repos_config',
'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
'repo_destination_variables', 'is_valid_branch', 'submodule_url_error', 'submodule_fetch_config',
'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log',
'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config',
'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value
@ -128,33 +128,67 @@ def resolve_host_addresses(host: str) -> List[str]:
return list({i[4][0] for i in socket.getaddrinfo(host, 443, type=socket.SOCK_STREAM)})
# Submodule URLs must have a shape that every URL parser (Python's, git's, curl's) reads the same way: no userinfo,
# backslashes, queries, fragments or whitespace.
ABSOLUTE_SUBMODULE_URL_RE = re.compile(
r'^https://(?P<host>[A-Za-z0-9.-]+|\[[0-9A-Fa-f:.]+\])'
r'(?::(?P<port>[0-9]{1,5}))?'
r'(?:/[A-Za-z0-9._~%!$&\'()*+,;=:/-]*)?$')
RELATIVE_SUBMODULE_URL_RE = re.compile(r'^\.\.?/[A-Za-z0-9._~%/-]*$')
def _check_submodule_url(url: str, repo_host: str) -> Tuple[Optional[str], Optional[Tuple[str, int, List[str]]]]:
"""Returns why the URL must not be fetched (or None), and the (host, port, addresses) to pin it to (or None)."""
if RELATIVE_SUBMODULE_URL_RE.match(url):
# Relative to the repo's own remote.
return None, None
match = ABSOLUTE_SUBMODULE_URL_RE.match(url)
if not match:
return 'only plain https:// submodule URLs can be downloaded', None
host = match.group('host').lower()
port = int(match.group('port') or 443)
if host == repo_host:
# The user chose this host.
return None, None
try:
addresses = [ipaddress.ip_address(host.strip('[]'))]
pin = None # An address needs no DNS.
except ValueError:
try:
addresses = [ipaddress.ip_address(i.split('%')[0]) for i in resolve_host_addresses(host)]
except (OSError, ValueError):
return f'could not resolve {host}', None
pin = (host, port, [str(i) for i in addresses])
if not addresses or not all(i.is_global for i in addresses):
return f'{host} is not a public address', None
return None, pin
def submodule_url_error(url: str, repo_host: str) -> Optional[str]:
"""Why a submodule URL must not be fetched, or None when it may be.
A submodule's URL is chosen by whoever controls the repo, so it may not make WROLPi fetch from its own network
(e.g. `https://localhost:8443/...` or `https://10.0.0.1/...`). Allowed: URLs relative to the repo's remote,
the repo's own host (which the user chose), and hosts with only public addresses. (A host which resolves
differently when git fetches it is not prevented.)"""
if url.startswith(('./', '../')):
return None
parsed = urlparse(url)
if parsed.scheme != 'https':
return 'only https:// submodules can be downloaded'
host = (parsed.hostname or '').lower()
if not host:
return 'the URL has no host'
if host == repo_host:
return None
try:
addresses = [ipaddress.ip_address(host)]
except ValueError:
try:
addresses = [ipaddress.ip_address(i.split('%')[0]) for i in resolve_host_addresses(host)]
except (OSError, ValueError):
return f'could not resolve {host}'
if not addresses or not all(i.is_global for i in addresses):
return f'{host} is not a public address'
return None
the repo's own host (which the user chose), and hosts with only public addresses."""
return _check_submodule_url(url, repo_host)[0]
def submodule_fetch_config(urls: List[str], repo_host: str) -> tuple:
"""git config for fetching submodules with these URLs. Raises ValueError if any URL must not be fetched.
Each checked host is pinned to the addresses that were checked, so its DNS cannot change before git fetches it,
and redirects are not followed, so a checked host cannot send git elsewhere."""
config = ['http.followRedirects=false']
pinned = set()
for url in urls:
error, pin = _check_submodule_url(url, repo_host)
if error:
raise ValueError(f'Refusing to download submodule {url}: {error}')
if pin and (pin[0], pin[1]) not in pinned:
pinned.add((pin[0], pin[1]))
addresses = ','.join(f'[{i}]' if ':' in i else i for i in pin[2])
config.append(f'http.curloptResolve={pin[0]}:{pin[1]}:{addresses}')
return tuple(config)
def repo_url_key(url: str) -> str:
@ -351,7 +385,8 @@ def create_repository(session: Session, url: str, tag_name: str = None, frequenc
directory = directory or collection.format_destination(tag_name)
_validate_directory(directory)
if session.query(Collection).filter(Collection.directory == directory, Collection.id != collection.id).first():
raise RepoConflict(f'Another collection already uses the directory {get_relative_to_media_directory(directory)}')
relative = get_relative_to_media_directory(directory)
raise RepoConflict(f'Another collection already uses the directory {relative}')
collection.directory = directory
download = collection.get_or_create_download(session, parsed.url, frequency, downloader_name='git')

View file

@ -203,6 +203,15 @@ async def test_archive_after_session_closes(test_session, async_client, git_remo
('http://codeberg.org/owner/library', None, False),
('ssh://git@codeberg.org/owner/library', None, False),
('file:///etc', None, False),
# URLs which parsers disagree about are refused.
('https://codeberg.org\\@10.0.0.5/library', ['217.197.91.145'], False),
('https://user@codeberg.org/owner/library', ['217.197.91.145'], False),
('https://10.0.0.5#@codeberg.org/library', ['217.197.91.145'], False),
('https://codeberg.org/owner/library?x=@10.0.0.5', ['217.197.91.145'], False),
('https://codeberg.org/owner/lib rary', ['217.197.91.145'], False),
('https://codeberg.org:443/owner/library', ['217.197.91.145'], True),
('../../evil@10.0.0.5/x', None, False),
('..\\evil', None, False),
])
def test_submodule_url_error(url, resolved, allowed):
"""A submodule URL comes from the (untrusted) repo; it may not make WROLPi fetch from its own network."""
@ -252,3 +261,43 @@ async def test_nested_submodule_to_private_address(test_session, git_remote_fact
# The first level (the repo's own host) was fetched; the nested submodule was not.
assert (repo.directory / 'vendor/library/README.md').is_file()
assert not (repo.directory / 'vendor/library/nested/README.md').exists()
@pytest.mark.parametrize('gitmodules', [
# A space in the name must not hide the URL.
'[submodule "a b"]\n\tpath = vendor/library\n\turl = https://10.0.0.5/library\n',
# Nor a newline in the URL.
'[submodule "library"]\n\tpath = vendor/library\n\turl = "https://10.0.0.5/library\\nsubmodule.x.path y"\n',
])
@pytest.mark.asyncio
async def test_submodule_url_cannot_hide(test_session, git_remote_factory, repos_download_manager, gitmodules):
"""Every submodule URL is checked, however the .gitmodules is written."""
library = git_remote_factory('library')
remote = git_remote_factory()
remote.add_submodule('vendor/library', library)
(remote.path / '.gitmodules').write_text(gitmodules)
remote.git('add', '.gitmodules')
remote.git('commit', '--quiet', '-m', 'Tricky .gitmodules')
repo = lib.create_repository(test_session, remote.url, submodules=True)
await repos_download_manager.wait_for_all_downloads()
test_session.expire_all()
download = test_session.query(Download).one()
assert download.status == DownloadStatus.deferred
assert 'Refusing' in download.error
assert not (repo.directory / 'vendor/library/README.md').exists()
def test_submodule_fetch_config():
"""Checked hosts are pinned to the addresses that were checked, so DNS cannot change between the check and git's
fetch."""
with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['217.197.91.145', '2606:4700::1111']):
config = lib.submodule_fetch_config(['https://codeberg.org/a/b', 'https://codeberg.org:8443/c/d',
'../relative', 'https://git.example.com/own/host'], 'git.example.com')
assert config == (
'http.followRedirects=false',
'http.curloptResolve=codeberg.org:443:217.197.91.145,[2606:4700::1111]',
'http.curloptResolve=codeberg.org:8443:217.197.91.145,[2606:4700::1111]',
)

View file

@ -189,5 +189,5 @@ kiwix-serve --port 80
---
#hashtag
'''
assert lib.markdown_to_text(readme) == \
'Kiwix Tools\nCI Kiwix is an offline reader. See the docs.\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag'
assert lib.markdown_to_text(readme) == ('Kiwix Tools\nCI Kiwix is an offline reader. See the docs.'
'\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag')