repos: refuse links anywhere in a git dir; check the directory again at clone time

- Any link in .git (objects, refs, logs, packed-refs, ...) refuses an import or update; git writes through them.
- A Repo's directory, and every directory between it and the media directory, must not be a link when it is cloned
  into or updated, not only when the Repo was added.
- Directories with control characters are refused; inspect's alternates names only the clone's own objects.
- Every git directory in .git/modules is sanitized, however deep (a planted HEAD can no longer hide one), and a
  submodule's .git file must point to one of them.
- sanitize_git_config checks every config before it rewrites any.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-07 09:54:41 -06:00
parent e2432d4ca9
commit b3f6fb144c
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 240 additions and 45 deletions

View file

@ -13,7 +13,7 @@ from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadRes
from wrolpi.errors import DownloadError
from .lib import git_command, git_env, find_readme, get_repo_size, is_valid_branch, read_readme_text, \
submodule_fetch_config, resolve_repo_addresses, pin_config, HostAddresses, clone_ownership_error, mark_clone, \
sanitize_git_config, new_clone_token, working_tree_git_error
sanitize_git_config, new_clone_token, working_tree_git_error, repo_directory_error
from .models import Repository
__all__ = ['GitDownloader', 'git_downloader', 'PreparedRepo', 'ExecutedRepo', 'parse_git_progress']
@ -267,6 +267,8 @@ class GitDownloader(Downloader):
download: Download = None) -> ExecutedRepo | DownloadResult:
download = download if download is not None else Download(url=prepared.url)
directory = prepared.directory
if error := repo_directory_error(directory):
raise DownloadError(error)
directory.parent.mkdir(parents=True, exist_ok=True)
# Resolved once; the repo and its same-host submodules are all fetched from these addresses, so a DNS

View file

@ -346,6 +346,18 @@ def _borrowing_git_file(git_dir: pathlib.Path) -> Optional[str]:
return None
def git_dir_link_error(git_dir: pathlib.Path) -> Optional[str]:
"""Why a git directory holding a link (anywhere, even a dangling one) must not be used, or None. git writes
through links (a fetch writes objects, refs, packed-refs and logs), and reads what they point to."""
if git_dir.is_symlink():
return f'{git_dir} is a link'
for dir_path, dir_names, file_names in os.walk(git_dir):
for name in (*dir_names, *file_names):
if os.path.islink(os.path.join(dir_path, name)):
return f'{os.path.join(dir_path, name)} is a link'
return None
def clone_ownership_error(directory: pathlib.Path, clone_token: Optional[str]) -> Optional[str]:
"""Why the clone in `directory` must not be updated as the Repo with `clone_token`, or None.
@ -355,8 +367,10 @@ def clone_ownership_error(directory: pathlib.Path, clone_token: Optional[str]) -
return f'{directory} is not a git directory'
if borrowed := _borrowing_git_file(git_dir):
return f'{directory} uses another git directory (.git/{borrowed})'
if error := git_dir_link_error(git_dir):
return error
marker = git_dir / REPO_MARKER
if not clone_token or marker.is_symlink() or not marker.is_file() \
if not clone_token or not marker.is_file() \
or not hmac.compare_digest(marker.read_text()[:100].strip(), clone_token):
return f'{directory} was not created by WROLPi. Import it from the Repos page'
return None
@ -406,10 +420,14 @@ def _format_git_config(entries: List[Tuple[str, str]]) -> str:
return '\n'.join(lines) + '\n'
def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None):
def _sanitized_config(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None) -> Optional[str]:
"""The sanitized contents of a git config file, or None if it needs no change. Changes nothing."""
if config_file.is_symlink() or not config_file.is_file():
raise ValueError(f'{config_file} is not a file')
entries = _git_config_entries(config_file)
try:
entries = _git_config_entries(config_file)
except subprocess.CalledProcessError:
raise ValueError(f'{config_file} is not a git config')
keep = []
for key, value in entries:
if ALLOWED_GIT_CONFIG_RE.match(key):
@ -420,57 +438,60 @@ def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pat
if worktree == worktree_root or worktree_root in worktree.parents:
keep.append((key, value))
if len(keep) == len(entries):
return
return None
logger.warning(f'Removing {len(entries) - len(keep)} unexpected settings from {config_file}')
return _format_git_config(keep)
def _write_config(config_file: pathlib.Path, content: str):
# An unpredictable, newly created file: the directory may hold links (e.g. a clone being imported).
fd, new = tempfile.mkstemp(dir=config_file.parent, prefix=f'{config_file.name}.wrolpi-')
try:
with os.fdopen(fd, 'w') as fh:
fh.write(_format_git_config(keep))
fh.write(content)
os.replace(new, config_file)
except BaseException:
pathlib.Path(new).unlink(missing_ok=True)
raise
logger.warning(f'Removed {len(entries) - len(keep)} unexpected settings from {config_file}')
def is_git_dir(path: pathlib.Path) -> bool:
"""Could git use `path` as a git directory (if a `.git` file pointed to it)? Its objects may be its own, or
another directory's (commondir). A HEAD in refs or logs is not enough."""
return (path / 'HEAD').is_file() and ((path / 'objects').is_dir() or (path / 'commondir').exists())
def submodule_git_dirs(directory: pathlib.Path) -> List[pathlib.Path]:
"""Every directory in the repo's .git/modules which git could use as a git directory, however deep (in another
git directory's refs, objects, ...). A submodule's name may contain `/`, so there is no telling them apart."""
modules = directory / '.git' / 'modules'
if modules.is_symlink() or not modules.is_dir():
return []
return [pathlib.Path(dir_path) for dir_path, _, _ in os.walk(modules) if is_git_dir(pathlib.Path(dir_path))]
def sanitize_git_config(directory: pathlib.Path):
"""Keep only the settings WROLPi needs in the configs of the repo in `directory`, and of its submodules.
Every git directory (the repo's, and each submodule's in .git/modules) is held to the same rules: no links, a
regular config file, and nothing which makes git use another directory. Raises ValueError otherwise."""
Every git directory (the repo's, and each one in .git/modules) is held to the same rules: no links, a regular
config file, and nothing which makes git use another directory. Raises ValueError otherwise, before any config
is changed."""
git_dir = directory / '.git'
root = directory.resolve()
_sanitize_config_file(git_dir / 'config')
if error := git_dir_link_error(git_dir):
# git would follow it out of the repo.
raise ValueError(error)
modules = git_dir / 'modules'
if modules.is_symlink():
raise ValueError(f'{modules} is a link')
if not modules.is_dir():
return
for path in modules.rglob('*'):
if path.is_symlink():
# git would follow it out of the repo.
raise ValueError(f'{path} is a link')
_sanitize_submodule_git_dirs(modules, root)
def _sanitize_submodule_git_dirs(modules: pathlib.Path, root: pathlib.Path):
"""Check and sanitize every git directory in a `modules` directory, as git finds them: a directory holding HEAD
is a submodule's git directory (a submodule's name may contain `/`), and its nested submodules are in its own
`modules`. Its refs and logs (which also hold files named HEAD) are not git directories."""
for child in sorted(modules.iterdir()):
if not child.is_dir():
continue
if not (child / 'HEAD').exists():
_sanitize_submodule_git_dirs(child, root)
continue
if borrowed := _borrowing_git_file(child):
raise ValueError(f'{child} uses another git directory ({borrowed})')
if not (child / 'objects').is_dir():
raise ValueError(f'{child} has no objects of its own')
_sanitize_config_file(child / 'config', worktree_root=root)
if (child / 'modules').is_dir():
_sanitize_submodule_git_dirs(child / 'modules', root)
changes = [(git_dir / 'config', _sanitized_config(git_dir / 'config'))]
for submodule_git_dir in submodule_git_dirs(directory):
if borrowed := _borrowing_git_file(submodule_git_dir):
raise ValueError(f'{submodule_git_dir} uses another git directory ({borrowed})')
config = submodule_git_dir / 'config'
if config.exists():
changes.append((config, _sanitized_config(config, worktree_root=root)))
for config, content in changes:
if content is not None:
_write_config(config, content)
def working_tree_git_error(directory: pathlib.Path) -> Optional[str]:
@ -481,6 +502,7 @@ def working_tree_git_error(directory: pathlib.Path) -> Optional[str]:
make git use a config which was never checked."""
root = directory.resolve()
modules = root / '.git' / 'modules'
git_dirs = {i.resolve() for i in submodule_git_dirs(root)}
for dir_path, dir_names, file_names in os.walk(root):
here = pathlib.Path(dir_path)
if '.git' in dir_names:
@ -500,6 +522,8 @@ def working_tree_git_error(directory: pathlib.Path) -> Optional[str]:
target = (here / content[len('gitdir: '):]).resolve()
if modules not in target.parents:
return f'{git_file} points outside the repo\'s submodules'
if target not in git_dirs:
return f'{git_file} does not point to a submodule\'s git directory'
return None
@ -782,6 +806,33 @@ def remote_https_url(url: str) -> Optional[str]:
return None
CONTROL_CHARACTERS_RE = re.compile(r'[\x00-\x1f\x7f]')
def repo_directory_error(directory: pathlib.Path) -> Optional[str]:
"""Why a Repo's directory must not be cloned into (or updated) now, or None. It was checked when the Repo was
added, but a link could have been made since: at the directory, or any directory between it and the media
directory, which would make git write somewhere else."""
media_directory = get_media_directory()
directory = pathlib.Path(directory)
if CONTROL_CHARACTERS_RE.search(str(directory)):
return f'{directory!r} contains control characters'
try:
relative = directory.relative_to(media_directory)
except ValueError:
return f'{directory} is not in the media directory'
if not relative.parts or '..' in relative.parts:
return f'{directory} is not a directory a Repo can use'
path = media_directory
for part in relative.parts:
path = path / part
if path.is_symlink():
return f'{path} is a link'
if directory.resolve() != media_directory.resolve() / relative:
return f'{directory} is not in the media directory'
return None
def _resolve_repo_directory(session: Session, directory: str) -> pathlib.Path:
"""A directory (relative to the media directory) a Repo may use: in the media directory, not WROLPi's own
directories, and not overlapping another Collection's. It does not have to be ignored."""
@ -789,6 +840,9 @@ def _resolve_repo_directory(session: Session, directory: str) -> pathlib.Path:
relative = pathlib.Path((directory or '').strip())
if not str(relative) or relative.is_absolute() or '..' in relative.parts:
raise InvalidRepo('The directory must be relative to the media directory')
if CONTROL_CHARACTERS_RE.search(str(relative)):
# A newline would split a line git reads (e.g. alternates).
raise InvalidRepo('The directory cannot contain control characters')
# Resolved (and confined to the media directory) before anything about it is reported, so a link cannot reveal
# what exists elsewhere.
path = (media_directory / relative).resolve()
@ -844,6 +898,8 @@ def resolve_import_directory(session: Session, directory: str) -> pathlib.Path:
raise InvalidRepo(f'{directory} cannot be imported: its .git/config is not a file')
if borrowed := _borrowing_git_file(git_dir):
raise InvalidRepo(f'{directory} cannot be imported: it uses another git directory (.git/{borrowed})')
if error := git_dir_link_error(git_dir):
raise InvalidRepo(f'{directory} cannot be imported: {error}')
return path
@ -852,10 +908,17 @@ def _read_only_git_dir(directory: pathlib.Path):
"""A throwaway git directory with the clone's HEAD, refs and objects, but none of its config (which has not been
sanitized, and could include anything). The clone's objects are only read, through alternates."""
git_dir = directory / '.git'
objects = git_dir / 'objects'
if objects.is_symlink() or not objects.is_dir():
raise InvalidRepo(f'{objects} is not a directory')
# alternates is read line by line: it must name only the clone's own objects.
objects = str(objects.resolve())
if CONTROL_CHARACTERS_RE.search(objects):
raise InvalidRepo(f'{objects!r} contains control characters')
with tempfile.TemporaryDirectory(prefix='wrolpi-inspect-') as temporary:
view = pathlib.Path(temporary)
(view / 'objects/info').mkdir(parents=True)
(view / 'objects/info/alternates').write_text(f'{(git_dir / "objects").resolve()}\n')
(view / 'objects/info/alternates').write_text(f'{objects}\n')
config = '[core]\n\trepositoryformatversion = 0\n\tbare = true\n'
entries = _git_config_entries(git_dir / 'config')
object_format = [v for k, v in entries if k.lower() == 'extensions.objectformat']
@ -900,8 +963,8 @@ def inspect_import(session: Session, directory: str, name: str = None, tag_name:
url_override: str = None) -> dict:
"""What importing the git clone in `directory` as a Repo would do. Changes nothing.
`url` is the URL the import will use (its host and owner can be in the Repo's directory); without it, the URL
comes from the clone's origin."""
The clone stays in `directory`. `url_override` is the URL the import will use, which must match the clone's
origin; without it, the URL comes from the origin."""
path = resolve_import_directory(session, directory)
entries = _git_config_entries(path / '.git/config')
keys = {key.lower() for key, _ in entries}

View file

@ -234,7 +234,8 @@ async def test_inspect_never_reads_the_clones_config(test_session, test_director
def test_working_tree_git_file_is_read_like_git(tmp_path):
"""git trims only the line ending from a `.git` file, not spaces; so does WROLPi."""
repo = tmp_path / 'repo'
(repo / '.git/modules/library').mkdir(parents=True)
(repo / '.git/modules/library/objects').mkdir(parents=True)
(repo / '.git/modules/library/HEAD').write_text('ref: refs/heads/main\n')
(repo / 'vendor/library').mkdir(parents=True)
# git reads this as "<...>/library " (with the space): not the checked git directory.
(repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library/../../../.. \n')

View file

@ -186,10 +186,14 @@ def test_marker_and_config_do_not_follow_links(tmp_path):
assert not (git_dir / lib.REPO_MARKER).is_symlink()
assert (git_dir / lib.REPO_MARKER).read_text().strip() == 'a1b2c3'
# A git directory holding any link is refused.
(git_dir / 'config').write_text('[http]\n\tproxy = evil\n[core]\n\tbare = false\n')
(git_dir / 'config.wrolpi').symlink_to(victim)
lib.sanitize_git_config(repo)
with pytest.raises(ValueError, match='link'):
lib.sanitize_git_config(repo)
assert victim.read_text() == 'precious'
(git_dir / 'config.wrolpi').unlink()
lib.sanitize_git_config(repo)
assert 'proxy' not in (git_dir / 'config').read_text()
# A config which is a link is refused.
@ -405,7 +409,7 @@ def test_working_tree_git_files(tmp_path, plant):
"""A submodule's `.git` (in the working tree) decides which git directory git uses for it; it must be a file
pointing inside the repo's own .git/modules."""
repo = tmp_path / 'repo'
(repo / '.git/modules/library').mkdir(parents=True)
_real_submodule_git_dir(repo / '.git/modules/library')
(repo / 'vendor/library').mkdir(parents=True)
(repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library\n')
assert lib.working_tree_git_error(repo) is None
@ -446,3 +450,128 @@ async def test_update_refuses_planted_submodule_git_file(test_session, test_dire
assert download.status == DownloadStatus.deferred
assert 'outside' in download.error
assert list(evil.iterdir()) == []
def _real_submodule_git_dir(path):
(path / 'objects').mkdir(parents=True)
(path / 'refs').mkdir()
(path / 'HEAD').write_text('ref: refs/heads/main\n')
(path / 'config').write_text('[core]\n\tbare = false\n')
def test_planted_head_does_not_hide_a_submodule_git_dir(tmp_path):
"""git finds a submodule's git directory at .git/modules/<name>, however deep; a git directory planted on a path
prefix (or anywhere else) cannot hide it from being sanitized."""
repo = tmp_path / 'repo'
(repo / '.git').mkdir(parents=True)
(repo / '.git/config').write_text('[core]\n\tbare = false\n')
_real_submodule_git_dir(repo / '.git/modules/vendor')
real = repo / '.git/modules/vendor/library'
_real_submodule_git_dir(real)
(real / 'config').write_text('[protocol "file"]\n\tallow = always\n[url "file:///tmp/evil/"]\n'
'\tinsteadOf = https://example.com/\n')
hidden = repo / '.git/modules/vendor/refs/library'
_real_submodule_git_dir(hidden)
(hidden / 'config').write_text('[http]\n\tproxy = evil\n')
lib.sanitize_git_config(repo)
assert 'protocol' not in (real / 'config').read_text()
assert 'insteadOf' not in (real / 'config').read_text()
assert 'proxy' not in (hidden / 'config').read_text()
# A submodule's `.git` must point at one of the (sanitized) git directories.
(repo / 'vendor/library').mkdir(parents=True)
(repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/vendor/library\n')
assert lib.working_tree_git_error(repo) is None
(repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/vendor/library/objects\n')
assert lib.working_tree_git_error(repo)
@pytest.mark.parametrize('link', ['objects', 'refs', 'logs', 'packed-refs', 'refs/remotes', 'objects/pack'])
@pytest.mark.asyncio
async def test_links_in_git_dir_are_refused(test_session, test_directory, async_client, test_wrolpi_config,
git_remote_factory, repos_download_manager, link):
"""git writes through a link in a git directory (fetch writes objects, refs and logs), so a git directory
holding one is neither imported nor updated."""
from modules.repos.errors import InvalidRepo
import shutil
def plant(git_dir):
target = git_dir / link
outside = test_directory / f'outside-{link.replace("/", "-")}-{git_dir.parent.name}'
if target.exists():
shutil.move(target, outside)
else:
outside.write_text('')
target.symlink_to(outside)
git(test_directory, 'init', '--quiet', 'code/linked')
git(test_directory / 'code/linked', 'commit', '--quiet', '--allow-empty', '-m', 'x')
git(test_directory / 'code/linked', 'pack-refs', '--all')
git(test_directory / 'code/linked', 'gc', '--quiet')
plant(test_directory / 'code/linked/.git')
with pytest.raises(InvalidRepo, match='link'):
lib.inspect_import(test_session, 'code/linked')
remote = git_remote_factory()
repo = lib.create_repository(test_session, remote.url)
await repos_download_manager.wait_for_all_downloads()
git(repo.directory, 'pack-refs', '--all')
git(repo.directory, 'gc', '--quiet')
plant(repo.directory / '.git')
download = await update(test_session, repos_download_manager)
assert download.status == DownloadStatus.deferred
assert 'link' in download.error
@pytest.mark.asyncio
async def test_control_characters_in_directories_are_refused(test_session, test_directory, async_client,
test_wrolpi_config):
"""A newline would split a line git reads (e.g. alternates)."""
from modules.repos.errors import InvalidRepo
for directory in ('foo\n/tmp/abs', 'foo\rbar', 'tab\there'):
with pytest.raises(InvalidRepo, match='character'):
lib.resolve_new_repo_directory(test_session, directory)
@pytest.mark.parametrize('link_at', ['directory', 'parent'])
@pytest.mark.asyncio
async def test_clone_refuses_links_made_after_adding(test_session, test_directory, async_client, test_wrolpi_config,
git_remote_factory, repos_download_manager, tmp_path_factory,
link_at):
"""A repo's directory (or a parent) which became a link after the repo was added is not cloned into."""
remote = git_remote_factory()
lib.create_repository(test_session, remote.url, directory='projects/example')
outside = tmp_path_factory.mktemp('outside')
if link_at == 'directory':
(test_directory / 'projects').mkdir()
(test_directory / 'projects/example').symlink_to(outside)
else:
(test_directory / 'projects').symlink_to(outside)
await repos_download_manager.wait_for_all_downloads()
download = test_session.query(Download).one()
assert download.status == DownloadStatus.deferred
assert 'link' in download.error
assert list(outside.iterdir()) == []
@pytest.mark.asyncio
async def test_refused_sanitize_changes_nothing(test_session, test_directory, async_client, test_wrolpi_config):
"""Every refusal is found before any config is rewritten."""
from modules.repos.errors import InvalidRepo
git(test_directory, 'init', '--quiet', 'code/clone')
directory = test_directory / 'code/clone'
git(directory, 'commit', '--quiet', '--allow-empty', '-m', 'x')
git(directory, 'remote', 'add', 'origin', 'https://git.example.com/owner/clone')
with (directory / '.git/config').open('a') as fh:
fh.write('[http]\n\tproxy = http://127.0.0.1:9\n')
module = directory / '.git/modules/library'
_real_submodule_git_dir(module)
# Found only after the clone's own config was checked.
(module / 'commondir').write_text('../..\n')
before = (directory / '.git/config').read_text()
with pytest.raises(InvalidRepo):
await lib.import_repository(test_session, 'code/clone', confirm=True)
assert (directory / '.git/config').read_text() == before