mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-20 15:06:19 -05:00
repos: close the remaining submodule SSRF gaps
* .gitmodules is read NUL-delimited, and an unreadable record refuses the submodules. A submodule name with a space (`[submodule "a b"]`) made `--get-regexp`'s "key value" output misfile its URL as a path, so the URL was never checked. * Submodule URLs must have a strict shape every parser reads the same way (no userinfo, backslash, query, fragment, or whitespace), so a URL cannot be checked as one host and fetched from another. * Each checked host is pinned (http.curloptResolve) to the addresses that were checked, so DNS cannot change between the check and git's fetch. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
3b6e962dfc
commit
4810b8bc50
4 changed files with 135 additions and 40 deletions
|
|
@ -13,7 +13,7 @@ from wrolpi.dates import now
|
|||
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, make_progress_callback
|
||||
from wrolpi.errors import DownloadError
|
||||
from .lib import git_command, git_env, find_readme, get_repo_size, is_valid_branch, read_readme_text, \
|
||||
submodule_url_error
|
||||
submodule_fetch_config
|
||||
from .models import Repository
|
||||
|
||||
__all__ = ['GitDownloader', 'git_downloader', 'PreparedRepo', 'ExecutedRepo', 'parse_git_progress']
|
||||
|
|
@ -189,23 +189,33 @@ class GitDownloader(Downloader):
|
|||
raise DownloadError(f'Submodules are nested more than {MAXIMUM_SUBMODULE_DEPTH} levels deep')
|
||||
git = lambda *args, **kwargs: self._git(download, ctx, directory, *args, **kwargs) # noqa: E731
|
||||
|
||||
output = await git('config', '--file', '.gitmodules', '--get-regexp', r'^submodule\..*\.(url|path)$',
|
||||
# NUL-delimited: a submodule's name and URL can contain spaces and newlines. Each record is "key\nvalue".
|
||||
output = await git('config', '--file', '.gitmodules', '-z', '--get-regexp', r'^submodule\..*\.(url|path)$',
|
||||
check=False)
|
||||
urls, paths = [], []
|
||||
for line in output.splitlines():
|
||||
key, _, value = line.partition(' ')
|
||||
(urls if key.endswith('.url') else paths).append(value)
|
||||
for record in output.split('\0'):
|
||||
if not record:
|
||||
continue
|
||||
key, newline, value = record.partition('\n')
|
||||
if not newline or not key.startswith('submodule.'):
|
||||
raise DownloadError(f'Refusing to download submodules, cannot read .gitmodules: {record!r}')
|
||||
if key.endswith('.url'):
|
||||
urls.append(value)
|
||||
elif key.endswith('.path'):
|
||||
paths.append(value)
|
||||
else:
|
||||
raise DownloadError(f'Refusing to download submodules, cannot read .gitmodules: {record!r}')
|
||||
repo_host = (urlparse(prepared.url).hostname or '').lower()
|
||||
for url in urls:
|
||||
if error := await asyncio.to_thread(submodule_url_error, url, repo_host):
|
||||
raise DownloadError(f'Refusing to download submodule {url}: {error}')
|
||||
try:
|
||||
fetch_config = await asyncio.to_thread(submodule_fetch_config, urls, repo_host)
|
||||
except ValueError as e:
|
||||
raise DownloadError(str(e))
|
||||
|
||||
await git('submodule', 'sync')
|
||||
args = ['submodule', 'update', '--init', '--force', '--progress']
|
||||
if prepared.mode == 'snapshot':
|
||||
args.extend(['--depth', '1'])
|
||||
# A checked host could otherwise redirect git into WROLPi's network.
|
||||
await git(*args, progress=True, config=('http.followRedirects=false',))
|
||||
await git(*args, progress=True, config=fetch_config)
|
||||
|
||||
for path in paths:
|
||||
submodule = (directory / path).resolve()
|
||||
|
|
@ -278,7 +288,8 @@ class GitDownloader(Downloader):
|
|||
if executed.submodules_error:
|
||||
# The repo is current, but the download is retried so its submodules are too.
|
||||
return DownloadResult(success=False, location=repo.location,
|
||||
error=f'The repo was updated, but its submodules were not: {executed.submodules_error}')
|
||||
error='The repo was updated, but its submodules were not: '
|
||||
f'{executed.submodules_error}')
|
||||
return DownloadResult(success=True, location=repo.location)
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -33,10 +33,10 @@ logger = logger.getChild(__name__)
|
|||
|
||||
__all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'create_repository',
|
||||
'update_repository', 'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme',
|
||||
'repo_destination_variables', 'is_valid_branch', 'submodule_url_error', 'read_readme_text',
|
||||
'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', 'get_repo_root',
|
||||
'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', 'get_repos_config',
|
||||
'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
|
||||
'repo_destination_variables', 'is_valid_branch', 'submodule_url_error', 'submodule_fetch_config',
|
||||
'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log',
|
||||
'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config',
|
||||
'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
|
||||
|
||||
DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value
|
||||
|
||||
|
|
@ -128,33 +128,67 @@ def resolve_host_addresses(host: str) -> List[str]:
|
|||
return list({i[4][0] for i in socket.getaddrinfo(host, 443, type=socket.SOCK_STREAM)})
|
||||
|
||||
|
||||
# Submodule URLs must have a shape that every URL parser (Python's, git's, curl's) reads the same way: no userinfo,
|
||||
# backslashes, queries, fragments or whitespace.
|
||||
ABSOLUTE_SUBMODULE_URL_RE = re.compile(
|
||||
r'^https://(?P<host>[A-Za-z0-9.-]+|\[[0-9A-Fa-f:.]+\])'
|
||||
r'(?::(?P<port>[0-9]{1,5}))?'
|
||||
r'(?:/[A-Za-z0-9._~%!$&\'()*+,;=:/-]*)?$')
|
||||
RELATIVE_SUBMODULE_URL_RE = re.compile(r'^\.\.?/[A-Za-z0-9._~%/-]*$')
|
||||
|
||||
|
||||
def _check_submodule_url(url: str, repo_host: str) -> Tuple[Optional[str], Optional[Tuple[str, int, List[str]]]]:
|
||||
"""Returns why the URL must not be fetched (or None), and the (host, port, addresses) to pin it to (or None)."""
|
||||
if RELATIVE_SUBMODULE_URL_RE.match(url):
|
||||
# Relative to the repo's own remote.
|
||||
return None, None
|
||||
match = ABSOLUTE_SUBMODULE_URL_RE.match(url)
|
||||
if not match:
|
||||
return 'only plain https:// submodule URLs can be downloaded', None
|
||||
host = match.group('host').lower()
|
||||
port = int(match.group('port') or 443)
|
||||
if host == repo_host:
|
||||
# The user chose this host.
|
||||
return None, None
|
||||
try:
|
||||
addresses = [ipaddress.ip_address(host.strip('[]'))]
|
||||
pin = None # An address needs no DNS.
|
||||
except ValueError:
|
||||
try:
|
||||
addresses = [ipaddress.ip_address(i.split('%')[0]) for i in resolve_host_addresses(host)]
|
||||
except (OSError, ValueError):
|
||||
return f'could not resolve {host}', None
|
||||
pin = (host, port, [str(i) for i in addresses])
|
||||
if not addresses or not all(i.is_global for i in addresses):
|
||||
return f'{host} is not a public address', None
|
||||
return None, pin
|
||||
|
||||
|
||||
def submodule_url_error(url: str, repo_host: str) -> Optional[str]:
|
||||
"""Why a submodule URL must not be fetched, or None when it may be.
|
||||
|
||||
A submodule's URL is chosen by whoever controls the repo, so it may not make WROLPi fetch from its own network
|
||||
(e.g. `https://localhost:8443/...` or `https://10.0.0.1/...`). Allowed: URLs relative to the repo's remote,
|
||||
the repo's own host (which the user chose), and hosts with only public addresses. (A host which resolves
|
||||
differently when git fetches it is not prevented.)"""
|
||||
if url.startswith(('./', '../')):
|
||||
return None
|
||||
parsed = urlparse(url)
|
||||
if parsed.scheme != 'https':
|
||||
return 'only https:// submodules can be downloaded'
|
||||
host = (parsed.hostname or '').lower()
|
||||
if not host:
|
||||
return 'the URL has no host'
|
||||
if host == repo_host:
|
||||
return None
|
||||
try:
|
||||
addresses = [ipaddress.ip_address(host)]
|
||||
except ValueError:
|
||||
try:
|
||||
addresses = [ipaddress.ip_address(i.split('%')[0]) for i in resolve_host_addresses(host)]
|
||||
except (OSError, ValueError):
|
||||
return f'could not resolve {host}'
|
||||
if not addresses or not all(i.is_global for i in addresses):
|
||||
return f'{host} is not a public address'
|
||||
return None
|
||||
the repo's own host (which the user chose), and hosts with only public addresses."""
|
||||
return _check_submodule_url(url, repo_host)[0]
|
||||
|
||||
|
||||
def submodule_fetch_config(urls: List[str], repo_host: str) -> tuple:
|
||||
"""git config for fetching submodules with these URLs. Raises ValueError if any URL must not be fetched.
|
||||
|
||||
Each checked host is pinned to the addresses that were checked, so its DNS cannot change before git fetches it,
|
||||
and redirects are not followed, so a checked host cannot send git elsewhere."""
|
||||
config = ['http.followRedirects=false']
|
||||
pinned = set()
|
||||
for url in urls:
|
||||
error, pin = _check_submodule_url(url, repo_host)
|
||||
if error:
|
||||
raise ValueError(f'Refusing to download submodule {url}: {error}')
|
||||
if pin and (pin[0], pin[1]) not in pinned:
|
||||
pinned.add((pin[0], pin[1]))
|
||||
addresses = ','.join(f'[{i}]' if ':' in i else i for i in pin[2])
|
||||
config.append(f'http.curloptResolve={pin[0]}:{pin[1]}:{addresses}')
|
||||
return tuple(config)
|
||||
|
||||
|
||||
def repo_url_key(url: str) -> str:
|
||||
|
|
@ -351,7 +385,8 @@ def create_repository(session: Session, url: str, tag_name: str = None, frequenc
|
|||
directory = directory or collection.format_destination(tag_name)
|
||||
_validate_directory(directory)
|
||||
if session.query(Collection).filter(Collection.directory == directory, Collection.id != collection.id).first():
|
||||
raise RepoConflict(f'Another collection already uses the directory {get_relative_to_media_directory(directory)}')
|
||||
relative = get_relative_to_media_directory(directory)
|
||||
raise RepoConflict(f'Another collection already uses the directory {relative}')
|
||||
collection.directory = directory
|
||||
|
||||
download = collection.get_or_create_download(session, parsed.url, frequency, downloader_name='git')
|
||||
|
|
|
|||
|
|
@ -203,6 +203,15 @@ async def test_archive_after_session_closes(test_session, async_client, git_remo
|
|||
('http://codeberg.org/owner/library', None, False),
|
||||
('ssh://git@codeberg.org/owner/library', None, False),
|
||||
('file:///etc', None, False),
|
||||
# URLs which parsers disagree about are refused.
|
||||
('https://codeberg.org\\@10.0.0.5/library', ['217.197.91.145'], False),
|
||||
('https://user@codeberg.org/owner/library', ['217.197.91.145'], False),
|
||||
('https://10.0.0.5#@codeberg.org/library', ['217.197.91.145'], False),
|
||||
('https://codeberg.org/owner/library?x=@10.0.0.5', ['217.197.91.145'], False),
|
||||
('https://codeberg.org/owner/lib rary', ['217.197.91.145'], False),
|
||||
('https://codeberg.org:443/owner/library', ['217.197.91.145'], True),
|
||||
('../../evil@10.0.0.5/x', None, False),
|
||||
('..\\evil', None, False),
|
||||
])
|
||||
def test_submodule_url_error(url, resolved, allowed):
|
||||
"""A submodule URL comes from the (untrusted) repo; it may not make WROLPi fetch from its own network."""
|
||||
|
|
@ -252,3 +261,43 @@ async def test_nested_submodule_to_private_address(test_session, git_remote_fact
|
|||
# The first level (the repo's own host) was fetched; the nested submodule was not.
|
||||
assert (repo.directory / 'vendor/library/README.md').is_file()
|
||||
assert not (repo.directory / 'vendor/library/nested/README.md').exists()
|
||||
|
||||
|
||||
@pytest.mark.parametrize('gitmodules', [
|
||||
# A space in the name must not hide the URL.
|
||||
'[submodule "a b"]\n\tpath = vendor/library\n\turl = https://10.0.0.5/library\n',
|
||||
# Nor a newline in the URL.
|
||||
'[submodule "library"]\n\tpath = vendor/library\n\turl = "https://10.0.0.5/library\\nsubmodule.x.path y"\n',
|
||||
])
|
||||
@pytest.mark.asyncio
|
||||
async def test_submodule_url_cannot_hide(test_session, git_remote_factory, repos_download_manager, gitmodules):
|
||||
"""Every submodule URL is checked, however the .gitmodules is written."""
|
||||
library = git_remote_factory('library')
|
||||
remote = git_remote_factory()
|
||||
remote.add_submodule('vendor/library', library)
|
||||
(remote.path / '.gitmodules').write_text(gitmodules)
|
||||
remote.git('add', '.gitmodules')
|
||||
remote.git('commit', '--quiet', '-m', 'Tricky .gitmodules')
|
||||
|
||||
repo = lib.create_repository(test_session, remote.url, submodules=True)
|
||||
await repos_download_manager.wait_for_all_downloads()
|
||||
test_session.expire_all()
|
||||
|
||||
download = test_session.query(Download).one()
|
||||
assert download.status == DownloadStatus.deferred
|
||||
assert 'Refusing' in download.error
|
||||
assert not (repo.directory / 'vendor/library/README.md').exists()
|
||||
|
||||
|
||||
|
||||
def test_submodule_fetch_config():
|
||||
"""Checked hosts are pinned to the addresses that were checked, so DNS cannot change between the check and git's
|
||||
fetch."""
|
||||
with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['217.197.91.145', '2606:4700::1111']):
|
||||
config = lib.submodule_fetch_config(['https://codeberg.org/a/b', 'https://codeberg.org:8443/c/d',
|
||||
'../relative', 'https://git.example.com/own/host'], 'git.example.com')
|
||||
assert config == (
|
||||
'http.followRedirects=false',
|
||||
'http.curloptResolve=codeberg.org:443:217.197.91.145,[2606:4700::1111]',
|
||||
'http.curloptResolve=codeberg.org:8443:217.197.91.145,[2606:4700::1111]',
|
||||
)
|
||||
|
|
|
|||
|
|
@ -189,5 +189,5 @@ kiwix-serve --port 80
|
|||
---
|
||||
#hashtag
|
||||
'''
|
||||
assert lib.markdown_to_text(readme) == \
|
||||
'Kiwix Tools\nCI Kiwix is an offline reader. See the docs.\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag'
|
||||
assert lib.markdown_to_text(readme) == ('Kiwix Tools\nCI Kiwix is an offline reader. See the docs.'
|
||||
'\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue