mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
2dd8e908c2
A relative submodule URL is resolved by git against the repo's remote, and can climb to another host: with the remote https://git.example.com/owner/repo, `../../../10.0.0.5/x` is https://10.0.0.5/x. Relative URLs were allowed unchecked. git now resolves the URLs first (`submodule init` + `sync` write them to the repo's config), and those resolved URLs, the ones git fetches, are checked. A relative URL is never allowed by the check itself. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
331 lines
15 KiB
Python
331 lines
15 KiB
Python
import io
|
|
import subprocess
|
|
import zipfile
|
|
from http import HTTPStatus
|
|
|
|
import pytest
|
|
from mock import mock
|
|
|
|
from modules.repos import lib
|
|
from modules.repos.models import Repository
|
|
from wrolpi.downloader import Download, DownloadStatus
|
|
|
|
|
|
def git(directory, *args) -> str:
|
|
return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True,
|
|
capture_output=True).stdout.decode().strip()
|
|
|
|
|
|
async def update(test_session, repos_download_manager):
|
|
download = test_session.query(Download).one()
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
return test_session.query(Download).one()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_log(test_session, async_client, git_remote_factory, repos_download_manager):
|
|
"""The history of the checked out branch, newest first, a page at a time."""
|
|
remote = git_remote_factory()
|
|
second = remote.commit({'a.txt': 'a'}, 'Second')
|
|
third = remote.commit({'b.txt': 'b'}, 'Third: with a | and a\ttab')
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/log?limit=2')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.json['total'] == 3
|
|
assert [(i['sha'], i['message']) for i in response.json['commits']] == \
|
|
[(third, 'Third: with a | and a\ttab'), (second, 'Second')]
|
|
assert response.json['commits'][0]['author'] == 'WROLPi Test'
|
|
assert response.json['commits'][0]['date']
|
|
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/log?limit=2&offset=2')
|
|
assert [i['message'] for i in response.json['commits']] == ['Initial commit']
|
|
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/log?limit=nope')
|
|
assert response.status_code == HTTPStatus.BAD_REQUEST
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_log_not_downloaded(test_session, async_client, test_wrolpi_config):
|
|
repo = lib.create_repository(test_session, 'https://github.com/a/b')
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/log')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/archive.zip')
|
|
assert response.status_code == HTTPStatus.NOT_FOUND
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_archive(test_session, async_client, git_remote_factory, repos_download_manager):
|
|
"""A ZIP of the checked out commit; local changes and `.git` are not in it."""
|
|
remote = git_remote_factory(files={'README.md': '# Example', 'src/main.c': 'int main;'})
|
|
repo = lib.create_repository(test_session, remote.url, name='my "repo"\r\nX-Evil: 1')
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
(repo.directory / 'untracked.txt').write_text('local')
|
|
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/archive.zip')
|
|
assert response.status_code == HTTPStatus.OK
|
|
assert response.headers['content-type'] == 'application/zip'
|
|
# The name is safe in a header.
|
|
name = f'my_repo_X-Evil_1-{repo.head_sha[:7]}'
|
|
assert response.headers['content-disposition'] == f'attachment; filename="{name}.zip"'
|
|
assert 'x-evil' not in response.headers
|
|
|
|
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
|
assert sorted(archive.namelist()) == [f'{name}/', f'{name}/README.md', f'{name}/src/', f'{name}/src/main.c']
|
|
assert archive.read(f'{name}/src/main.c') == b'int main;'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submodules(test_session, git_remote_factory, repos_download_manager):
|
|
"""Submodules are only downloaded when the Repo asks for them, and are removed when it no longer does."""
|
|
library = git_remote_factory('library', files={'lib.c': 'v1'})
|
|
remote = git_remote_factory()
|
|
remote.add_submodule('vendor/library', library)
|
|
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
submodule = repo.directory / 'vendor/library'
|
|
assert not (submodule / 'lib.c').exists()
|
|
|
|
lib.update_repository(test_session, repo.id, submodules=True)
|
|
download = await update(test_session, repos_download_manager)
|
|
assert download.status == DownloadStatus.complete
|
|
assert (submodule / 'lib.c').read_text() == 'v1'
|
|
# The submodule has its history, like the Repo.
|
|
assert git(submodule, 'rev-list', '--count', 'HEAD') == '1'
|
|
|
|
# A new submodule commit is checked out when the Repo points at it.
|
|
library.commit({'lib.c': 'v2'}, 'v2')
|
|
remote.git('update-index', '--cacheinfo', f'160000,{library.head},vendor/library')
|
|
remote.git('commit', '--quiet', '-m', 'Bump library')
|
|
await update(test_session, repos_download_manager)
|
|
assert (submodule / 'lib.c').read_text() == 'v2'
|
|
|
|
lib.update_repository(test_session, repo.id, submodules=False)
|
|
await update(test_session, repos_download_manager)
|
|
assert not (submodule / 'lib.c').exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submodule_failure(test_session, git_remote_factory, repos_download_manager):
|
|
"""A submodule which cannot be downloaded (e.g. an ssh URL) does not stop the Repo from updating."""
|
|
library = git_remote_factory('library')
|
|
remote = git_remote_factory()
|
|
remote.add_submodule('vendor/library', library)
|
|
# Rewrite the submodule's URL to one git may not use.
|
|
(remote.path / '.gitmodules').write_text(
|
|
'[submodule "vendor/library"]\n\tpath = vendor/library\n\turl = ssh://git@git.example.com/owner/library\n')
|
|
(remote.path / 'new.txt').write_text('new')
|
|
# Not `remote.commit`, whose `add --all` would delete the submodule (it is not checked out in the remote).
|
|
remote.git('add', '.gitmodules', 'new.txt')
|
|
remote.git('commit', '--quiet', '-m', 'ssh submodule')
|
|
head = remote.head
|
|
|
|
repo = lib.create_repository(test_session, remote.url, submodules=True)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'its submodules were not' in download.error
|
|
# The Repo itself is current.
|
|
assert repo.head_sha == head
|
|
assert (repo.directory / 'new.txt').is_file()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submodules_api_and_config(test_session, async_client, test_wrolpi_config, await_switches):
|
|
body = '{"url": "https://github.com/a/b", "submodules": true}'
|
|
request, response = await async_client.post('/api/repos', content=body)
|
|
assert response.status_code == HTTPStatus.CREATED
|
|
assert response.json['repo']['submodules'] is True
|
|
repo_id = response.json['repo']['id']
|
|
|
|
await await_switches()
|
|
assert lib.get_repos_config().repos[0]['submodules'] is True
|
|
|
|
request, response = await async_client.put(f'/api/repos/{repo_id}', content='{"submodules": false}')
|
|
assert response.json['repo']['submodules'] is False
|
|
|
|
# Restored from the config.
|
|
config = lib.get_repos_config()
|
|
data = config.read_config_file()
|
|
data['repos'][0]['submodules'] = True
|
|
data['version'] += 1
|
|
config.write_config_data(data, config.get_file())
|
|
config.import_config()
|
|
test_session.expire_all()
|
|
assert test_session.query(Repository).one().submodules is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_archive_after_session_closes(test_session, async_client, git_remote_factory, repos_download_manager):
|
|
"""In production the request's session is closed once a streamed response starts; the ZIP must not need it."""
|
|
from sanic import Request
|
|
|
|
remote = git_remote_factory()
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
|
|
original_respond = Request.respond
|
|
|
|
async def respond_and_close_session(self, *args, **kwargs):
|
|
response = await original_respond(self, *args, **kwargs)
|
|
test_session.expunge_all()
|
|
return response
|
|
|
|
with mock.patch.object(Request, 'respond', respond_and_close_session):
|
|
request, response = await async_client.get(f'/api/repos/{repo.id}/archive.zip')
|
|
assert response.status_code == HTTPStatus.OK
|
|
with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
|
|
assert any(i.endswith('/README.md') for i in archive.namelist())
|
|
|
|
|
|
@pytest.mark.parametrize('url,resolved,allowed', [
|
|
# Relative URLs are resolved by git (against the repo's remote) before they are checked.
|
|
('../library.git', None, False),
|
|
('./library', None, False),
|
|
('https://git.example.com/owner/library', None, True), # The repo's own host.
|
|
('https://codeberg.org/owner/library', ['217.197.91.145'], True),
|
|
('https://localhost/library', ['127.0.0.1'], False),
|
|
('https://127.0.0.1/library', None, False),
|
|
('https://10.0.0.5/library', None, False),
|
|
('https://192.168.1.1:8443/library', None, False),
|
|
('https://[::1]/library', None, False),
|
|
('https://169.254.169.254/latest', None, False),
|
|
('https://gitea.lan/library', ['192.168.1.20'], False),
|
|
('https://mixed.example/library', ['93.184.215.14', '10.0.0.1'], False),
|
|
('https://unresolvable.example/library', OSError('no such host'), False),
|
|
('http://codeberg.org/owner/library', None, False),
|
|
('ssh://git@codeberg.org/owner/library', None, False),
|
|
('file:///etc', None, False),
|
|
# URLs which parsers disagree about are refused.
|
|
('https://codeberg.org\\@10.0.0.5/library', ['217.197.91.145'], False),
|
|
('https://user@codeberg.org/owner/library', ['217.197.91.145'], False),
|
|
('https://10.0.0.5#@codeberg.org/library', ['217.197.91.145'], False),
|
|
('https://codeberg.org/owner/library?x=@10.0.0.5', ['217.197.91.145'], False),
|
|
('https://codeberg.org/owner/lib rary', ['217.197.91.145'], False),
|
|
('https://codeberg.org:443/owner/library', ['217.197.91.145'], True),
|
|
('../../evil@10.0.0.5/x', None, False),
|
|
('..\\evil', None, False),
|
|
])
|
|
def test_submodule_url_error(url, resolved, allowed):
|
|
"""A submodule URL comes from the (untrusted) repo; it may not make WROLPi fetch from its own network."""
|
|
side_effect = resolved if isinstance(resolved, Exception) else None
|
|
with mock.patch('modules.repos.lib.resolve_host_addresses', side_effect=side_effect,
|
|
return_value=resolved or []):
|
|
error = lib.submodule_url_error(url, 'git.example.com')
|
|
assert (error is None) is allowed, error
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_submodule_to_private_address(test_session, git_remote_factory, repos_download_manager):
|
|
"""A submodule which points into WROLPi's network is never fetched; the repo itself still updates."""
|
|
library = git_remote_factory('library')
|
|
remote = git_remote_factory()
|
|
remote.add_submodule('vendor/library', library, url='https://10.0.0.5/admin/library')
|
|
|
|
repo = lib.create_repository(test_session, remote.url, submodules=True)
|
|
with mock.patch('modules.repos.lib.resolve_host_addresses') as resolve:
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'not a public address' in download.error
|
|
assert repo.head_sha == remote.head
|
|
assert not (repo.directory / 'vendor/library/README.md').exists()
|
|
resolve.assert_not_called() # An IP address is checked without DNS.
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_nested_submodule_to_private_address(test_session, git_remote_factory, repos_download_manager):
|
|
"""Each level of submodules is checked before it is fetched."""
|
|
evil = git_remote_factory('evil')
|
|
library = git_remote_factory('library')
|
|
library.add_submodule('nested', evil, url='https://127.0.0.1/evil')
|
|
remote = git_remote_factory()
|
|
remote.add_submodule('vendor/library', library)
|
|
|
|
repo = lib.create_repository(test_session, remote.url, submodules=True)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'not a public address' in download.error
|
|
# The first level (the repo's own host) was fetched; the nested submodule was not.
|
|
assert (repo.directory / 'vendor/library/README.md').is_file()
|
|
assert not (repo.directory / 'vendor/library/nested/README.md').exists()
|
|
|
|
|
|
@pytest.mark.parametrize('gitmodules', [
|
|
# A space in the name must not hide the URL.
|
|
'[submodule "a b"]\n\tpath = vendor/library\n\turl = https://10.0.0.5/library\n',
|
|
# Nor a newline in the URL.
|
|
'[submodule "library"]\n\tpath = vendor/library\n\turl = "https://10.0.0.5/library\\nsubmodule.x.path y"\n',
|
|
])
|
|
@pytest.mark.asyncio
|
|
async def test_submodule_url_cannot_hide(test_session, git_remote_factory, repos_download_manager, gitmodules):
|
|
"""Every submodule URL is checked, however the .gitmodules is written."""
|
|
library = git_remote_factory('library')
|
|
remote = git_remote_factory()
|
|
remote.add_submodule('vendor/library', library)
|
|
(remote.path / '.gitmodules').write_text(gitmodules)
|
|
remote.git('add', '.gitmodules')
|
|
remote.git('commit', '--quiet', '-m', 'Tricky .gitmodules')
|
|
|
|
repo = lib.create_repository(test_session, remote.url, submodules=True)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
download = test_session.query(Download).one()
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'Refusing' in download.error
|
|
assert not (repo.directory / 'vendor/library/README.md').exists()
|
|
|
|
|
|
|
|
def test_submodule_fetch_config():
|
|
"""Checked hosts are pinned to the addresses that were checked, so DNS cannot change between the check and git's
|
|
fetch."""
|
|
with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['217.197.91.145', '2606:4700::1111']):
|
|
config = lib.submodule_fetch_config(['https://codeberg.org/a/b', 'https://codeberg.org:8443/c/d',
|
|
'https://git.example.com/own/host'], 'git.example.com')
|
|
assert config == (
|
|
'http.followRedirects=false',
|
|
'http.curloptResolve=codeberg.org:443:217.197.91.145,[2606:4700::1111]',
|
|
'http.curloptResolve=codeberg.org:8443:217.197.91.145,[2606:4700::1111]',
|
|
)
|
|
|
|
|
|
|
|
@pytest.mark.parametrize('relative_url,allowed', [
|
|
('../library', True), # https://git.example.com/owner/library
|
|
('../../../10.0.0.5/library', False), # git resolves this to https://10.0.0.5/library
|
|
])
|
|
@pytest.mark.asyncio
|
|
async def test_relative_submodule_url(test_session, git_remote_factory, repos_download_manager, relative_url,
|
|
allowed):
|
|
"""A relative submodule URL is checked where git resolves it, which may be another host."""
|
|
library = git_remote_factory('library')
|
|
remote = git_remote_factory()
|
|
remote.add_submodule('vendor/library', library, url=relative_url)
|
|
|
|
repo = lib.create_repository(test_session, remote.url, submodules=True)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
download = test_session.query(Download).one()
|
|
if allowed:
|
|
assert download.status == DownloadStatus.complete, download.error
|
|
assert (repo.directory / 'vendor/library/README.md').is_file()
|
|
else:
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'Refusing to download submodule https://10.0.0.5/library' in download.error
|
|
assert not (repo.directory / 'vendor/library/README.md').exists()
|