From b3f6fb144c2a2e8803e18f5115db33e0545d7435 Mon Sep 17 00:00:00 2001 From: Roland Date: Wed, 7 Oct 2026 09:54:41 -0600 Subject: [PATCH] repos: refuse links anywhere in a git dir; check the directory again at clone time - Any link in .git (objects, refs, logs, packed-refs, ...) refuses an import or update; git writes through them. - A Repo's directory, and every directory between it and the media directory, must not be a link when it is cloned into or updated, not only when the Repo was added. - Directories with control characters are refused; inspect's alternates names only the clone's own objects. - Every git directory in .git/modules is sanitized, however deep (a planted HEAD can no longer hide one), and a submodule's .git file must point to one of them. - sanitize_git_config checks every config before it rewrites any. Co-Authored-By: Claude Opus 5.5 --- modules/repos/downloader.py | 4 +- modules/repos/lib.py | 145 +++++++++++++++++++-------- modules/repos/test/test_import.py | 3 +- modules/repos/test/test_ownership.py | 133 +++++++++++++++++++++++- 4 files changed, 240 insertions(+), 45 deletions(-) diff --git a/modules/repos/downloader.py b/modules/repos/downloader.py index be41c7f0..c6602a8b 100644 --- a/modules/repos/downloader.py +++ b/modules/repos/downloader.py @@ -13,7 +13,7 @@ from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadRes from wrolpi.errors import DownloadError from .lib import git_command, git_env, find_readme, get_repo_size, is_valid_branch, read_readme_text, \ submodule_fetch_config, resolve_repo_addresses, pin_config, HostAddresses, clone_ownership_error, mark_clone, \ - sanitize_git_config, new_clone_token, working_tree_git_error + sanitize_git_config, new_clone_token, working_tree_git_error, repo_directory_error from .models import Repository __all__ = ['GitDownloader', 'git_downloader', 'PreparedRepo', 'ExecutedRepo', 'parse_git_progress'] @@ -267,6 +267,8 @@ class GitDownloader(Downloader): download: Download = None) -> ExecutedRepo | DownloadResult: download = download if download is not None else Download(url=prepared.url) directory = prepared.directory + if error := repo_directory_error(directory): + raise DownloadError(error) directory.parent.mkdir(parents=True, exist_ok=True) # Resolved once; the repo and its same-host submodules are all fetched from these addresses, so a DNS diff --git a/modules/repos/lib.py b/modules/repos/lib.py index dd509732..021b429c 100644 --- a/modules/repos/lib.py +++ b/modules/repos/lib.py @@ -346,6 +346,18 @@ def _borrowing_git_file(git_dir: pathlib.Path) -> Optional[str]: return None +def git_dir_link_error(git_dir: pathlib.Path) -> Optional[str]: + """Why a git directory holding a link (anywhere, even a dangling one) must not be used, or None. git writes + through links (a fetch writes objects, refs, packed-refs and logs), and reads what they point to.""" + if git_dir.is_symlink(): + return f'{git_dir} is a link' + for dir_path, dir_names, file_names in os.walk(git_dir): + for name in (*dir_names, *file_names): + if os.path.islink(os.path.join(dir_path, name)): + return f'{os.path.join(dir_path, name)} is a link' + return None + + def clone_ownership_error(directory: pathlib.Path, clone_token: Optional[str]) -> Optional[str]: """Why the clone in `directory` must not be updated as the Repo with `clone_token`, or None. @@ -355,8 +367,10 @@ def clone_ownership_error(directory: pathlib.Path, clone_token: Optional[str]) - return f'{directory} is not a git directory' if borrowed := _borrowing_git_file(git_dir): return f'{directory} uses another git directory (.git/{borrowed})' + if error := git_dir_link_error(git_dir): + return error marker = git_dir / REPO_MARKER - if not clone_token or marker.is_symlink() or not marker.is_file() \ + if not clone_token or not marker.is_file() \ or not hmac.compare_digest(marker.read_text()[:100].strip(), clone_token): return f'{directory} was not created by WROLPi. Import it from the Repos page' return None @@ -406,10 +420,14 @@ def _format_git_config(entries: List[Tuple[str, str]]) -> str: return '\n'.join(lines) + '\n' -def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None): +def _sanitized_config(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None) -> Optional[str]: + """The sanitized contents of a git config file, or None if it needs no change. Changes nothing.""" if config_file.is_symlink() or not config_file.is_file(): raise ValueError(f'{config_file} is not a file') - entries = _git_config_entries(config_file) + try: + entries = _git_config_entries(config_file) + except subprocess.CalledProcessError: + raise ValueError(f'{config_file} is not a git config') keep = [] for key, value in entries: if ALLOWED_GIT_CONFIG_RE.match(key): @@ -420,57 +438,60 @@ def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pat if worktree == worktree_root or worktree_root in worktree.parents: keep.append((key, value)) if len(keep) == len(entries): - return + return None + logger.warning(f'Removing {len(entries) - len(keep)} unexpected settings from {config_file}') + return _format_git_config(keep) + + +def _write_config(config_file: pathlib.Path, content: str): # An unpredictable, newly created file: the directory may hold links (e.g. a clone being imported). fd, new = tempfile.mkstemp(dir=config_file.parent, prefix=f'{config_file.name}.wrolpi-') try: with os.fdopen(fd, 'w') as fh: - fh.write(_format_git_config(keep)) + fh.write(content) os.replace(new, config_file) except BaseException: pathlib.Path(new).unlink(missing_ok=True) raise - logger.warning(f'Removed {len(entries) - len(keep)} unexpected settings from {config_file}') + + +def is_git_dir(path: pathlib.Path) -> bool: + """Could git use `path` as a git directory (if a `.git` file pointed to it)? Its objects may be its own, or + another directory's (commondir). A HEAD in refs or logs is not enough.""" + return (path / 'HEAD').is_file() and ((path / 'objects').is_dir() or (path / 'commondir').exists()) + + +def submodule_git_dirs(directory: pathlib.Path) -> List[pathlib.Path]: + """Every directory in the repo's .git/modules which git could use as a git directory, however deep (in another + git directory's refs, objects, ...). A submodule's name may contain `/`, so there is no telling them apart.""" + modules = directory / '.git' / 'modules' + if modules.is_symlink() or not modules.is_dir(): + return [] + return [pathlib.Path(dir_path) for dir_path, _, _ in os.walk(modules) if is_git_dir(pathlib.Path(dir_path))] def sanitize_git_config(directory: pathlib.Path): """Keep only the settings WROLPi needs in the configs of the repo in `directory`, and of its submodules. - Every git directory (the repo's, and each submodule's in .git/modules) is held to the same rules: no links, a - regular config file, and nothing which makes git use another directory. Raises ValueError otherwise.""" + Every git directory (the repo's, and each one in .git/modules) is held to the same rules: no links, a regular + config file, and nothing which makes git use another directory. Raises ValueError otherwise, before any config + is changed.""" git_dir = directory / '.git' root = directory.resolve() - _sanitize_config_file(git_dir / 'config') + if error := git_dir_link_error(git_dir): + # git would follow it out of the repo. + raise ValueError(error) - modules = git_dir / 'modules' - if modules.is_symlink(): - raise ValueError(f'{modules} is a link') - if not modules.is_dir(): - return - for path in modules.rglob('*'): - if path.is_symlink(): - # git would follow it out of the repo. - raise ValueError(f'{path} is a link') - _sanitize_submodule_git_dirs(modules, root) - - -def _sanitize_submodule_git_dirs(modules: pathlib.Path, root: pathlib.Path): - """Check and sanitize every git directory in a `modules` directory, as git finds them: a directory holding HEAD - is a submodule's git directory (a submodule's name may contain `/`), and its nested submodules are in its own - `modules`. Its refs and logs (which also hold files named HEAD) are not git directories.""" - for child in sorted(modules.iterdir()): - if not child.is_dir(): - continue - if not (child / 'HEAD').exists(): - _sanitize_submodule_git_dirs(child, root) - continue - if borrowed := _borrowing_git_file(child): - raise ValueError(f'{child} uses another git directory ({borrowed})') - if not (child / 'objects').is_dir(): - raise ValueError(f'{child} has no objects of its own') - _sanitize_config_file(child / 'config', worktree_root=root) - if (child / 'modules').is_dir(): - _sanitize_submodule_git_dirs(child / 'modules', root) + changes = [(git_dir / 'config', _sanitized_config(git_dir / 'config'))] + for submodule_git_dir in submodule_git_dirs(directory): + if borrowed := _borrowing_git_file(submodule_git_dir): + raise ValueError(f'{submodule_git_dir} uses another git directory ({borrowed})') + config = submodule_git_dir / 'config' + if config.exists(): + changes.append((config, _sanitized_config(config, worktree_root=root))) + for config, content in changes: + if content is not None: + _write_config(config, content) def working_tree_git_error(directory: pathlib.Path) -> Optional[str]: @@ -481,6 +502,7 @@ def working_tree_git_error(directory: pathlib.Path) -> Optional[str]: make git use a config which was never checked.""" root = directory.resolve() modules = root / '.git' / 'modules' + git_dirs = {i.resolve() for i in submodule_git_dirs(root)} for dir_path, dir_names, file_names in os.walk(root): here = pathlib.Path(dir_path) if '.git' in dir_names: @@ -500,6 +522,8 @@ def working_tree_git_error(directory: pathlib.Path) -> Optional[str]: target = (here / content[len('gitdir: '):]).resolve() if modules not in target.parents: return f'{git_file} points outside the repo\'s submodules' + if target not in git_dirs: + return f'{git_file} does not point to a submodule\'s git directory' return None @@ -782,6 +806,33 @@ def remote_https_url(url: str) -> Optional[str]: return None +CONTROL_CHARACTERS_RE = re.compile(r'[\x00-\x1f\x7f]') + + +def repo_directory_error(directory: pathlib.Path) -> Optional[str]: + """Why a Repo's directory must not be cloned into (or updated) now, or None. It was checked when the Repo was + added, but a link could have been made since: at the directory, or any directory between it and the media + directory, which would make git write somewhere else.""" + media_directory = get_media_directory() + directory = pathlib.Path(directory) + if CONTROL_CHARACTERS_RE.search(str(directory)): + return f'{directory!r} contains control characters' + try: + relative = directory.relative_to(media_directory) + except ValueError: + return f'{directory} is not in the media directory' + if not relative.parts or '..' in relative.parts: + return f'{directory} is not a directory a Repo can use' + path = media_directory + for part in relative.parts: + path = path / part + if path.is_symlink(): + return f'{path} is a link' + if directory.resolve() != media_directory.resolve() / relative: + return f'{directory} is not in the media directory' + return None + + def _resolve_repo_directory(session: Session, directory: str) -> pathlib.Path: """A directory (relative to the media directory) a Repo may use: in the media directory, not WROLPi's own directories, and not overlapping another Collection's. It does not have to be ignored.""" @@ -789,6 +840,9 @@ def _resolve_repo_directory(session: Session, directory: str) -> pathlib.Path: relative = pathlib.Path((directory or '').strip()) if not str(relative) or relative.is_absolute() or '..' in relative.parts: raise InvalidRepo('The directory must be relative to the media directory') + if CONTROL_CHARACTERS_RE.search(str(relative)): + # A newline would split a line git reads (e.g. alternates). + raise InvalidRepo('The directory cannot contain control characters') # Resolved (and confined to the media directory) before anything about it is reported, so a link cannot reveal # what exists elsewhere. path = (media_directory / relative).resolve() @@ -844,6 +898,8 @@ def resolve_import_directory(session: Session, directory: str) -> pathlib.Path: raise InvalidRepo(f'{directory} cannot be imported: its .git/config is not a file') if borrowed := _borrowing_git_file(git_dir): raise InvalidRepo(f'{directory} cannot be imported: it uses another git directory (.git/{borrowed})') + if error := git_dir_link_error(git_dir): + raise InvalidRepo(f'{directory} cannot be imported: {error}') return path @@ -852,10 +908,17 @@ def _read_only_git_dir(directory: pathlib.Path): """A throwaway git directory with the clone's HEAD, refs and objects, but none of its config (which has not been sanitized, and could include anything). The clone's objects are only read, through alternates.""" git_dir = directory / '.git' + objects = git_dir / 'objects' + if objects.is_symlink() or not objects.is_dir(): + raise InvalidRepo(f'{objects} is not a directory') + # alternates is read line by line: it must name only the clone's own objects. + objects = str(objects.resolve()) + if CONTROL_CHARACTERS_RE.search(objects): + raise InvalidRepo(f'{objects!r} contains control characters') with tempfile.TemporaryDirectory(prefix='wrolpi-inspect-') as temporary: view = pathlib.Path(temporary) (view / 'objects/info').mkdir(parents=True) - (view / 'objects/info/alternates').write_text(f'{(git_dir / "objects").resolve()}\n') + (view / 'objects/info/alternates').write_text(f'{objects}\n') config = '[core]\n\trepositoryformatversion = 0\n\tbare = true\n' entries = _git_config_entries(git_dir / 'config') object_format = [v for k, v in entries if k.lower() == 'extensions.objectformat'] @@ -900,8 +963,8 @@ def inspect_import(session: Session, directory: str, name: str = None, tag_name: url_override: str = None) -> dict: """What importing the git clone in `directory` as a Repo would do. Changes nothing. - `url` is the URL the import will use (its host and owner can be in the Repo's directory); without it, the URL - comes from the clone's origin.""" + The clone stays in `directory`. `url_override` is the URL the import will use, which must match the clone's + origin; without it, the URL comes from the origin.""" path = resolve_import_directory(session, directory) entries = _git_config_entries(path / '.git/config') keys = {key.lower() for key, _ in entries} diff --git a/modules/repos/test/test_import.py b/modules/repos/test/test_import.py index 5027de37..bc107fe0 100644 --- a/modules/repos/test/test_import.py +++ b/modules/repos/test/test_import.py @@ -234,7 +234,8 @@ async def test_inspect_never_reads_the_clones_config(test_session, test_director def test_working_tree_git_file_is_read_like_git(tmp_path): """git trims only the line ending from a `.git` file, not spaces; so does WROLPi.""" repo = tmp_path / 'repo' - (repo / '.git/modules/library').mkdir(parents=True) + (repo / '.git/modules/library/objects').mkdir(parents=True) + (repo / '.git/modules/library/HEAD').write_text('ref: refs/heads/main\n') (repo / 'vendor/library').mkdir(parents=True) # git reads this as "<...>/library " (with the space): not the checked git directory. (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library/../../../.. \n') diff --git a/modules/repos/test/test_ownership.py b/modules/repos/test/test_ownership.py index f59e1465..914debd1 100644 --- a/modules/repos/test/test_ownership.py +++ b/modules/repos/test/test_ownership.py @@ -186,10 +186,14 @@ def test_marker_and_config_do_not_follow_links(tmp_path): assert not (git_dir / lib.REPO_MARKER).is_symlink() assert (git_dir / lib.REPO_MARKER).read_text().strip() == 'a1b2c3' + # A git directory holding any link is refused. (git_dir / 'config').write_text('[http]\n\tproxy = evil\n[core]\n\tbare = false\n') (git_dir / 'config.wrolpi').symlink_to(victim) - lib.sanitize_git_config(repo) + with pytest.raises(ValueError, match='link'): + lib.sanitize_git_config(repo) assert victim.read_text() == 'precious' + (git_dir / 'config.wrolpi').unlink() + lib.sanitize_git_config(repo) assert 'proxy' not in (git_dir / 'config').read_text() # A config which is a link is refused. @@ -405,7 +409,7 @@ def test_working_tree_git_files(tmp_path, plant): """A submodule's `.git` (in the working tree) decides which git directory git uses for it; it must be a file pointing inside the repo's own .git/modules.""" repo = tmp_path / 'repo' - (repo / '.git/modules/library').mkdir(parents=True) + _real_submodule_git_dir(repo / '.git/modules/library') (repo / 'vendor/library').mkdir(parents=True) (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library\n') assert lib.working_tree_git_error(repo) is None @@ -446,3 +450,128 @@ async def test_update_refuses_planted_submodule_git_file(test_session, test_dire assert download.status == DownloadStatus.deferred assert 'outside' in download.error assert list(evil.iterdir()) == [] + + +def _real_submodule_git_dir(path): + (path / 'objects').mkdir(parents=True) + (path / 'refs').mkdir() + (path / 'HEAD').write_text('ref: refs/heads/main\n') + (path / 'config').write_text('[core]\n\tbare = false\n') + + +def test_planted_head_does_not_hide_a_submodule_git_dir(tmp_path): + """git finds a submodule's git directory at .git/modules/, however deep; a git directory planted on a path + prefix (or anywhere else) cannot hide it from being sanitized.""" + repo = tmp_path / 'repo' + (repo / '.git').mkdir(parents=True) + (repo / '.git/config').write_text('[core]\n\tbare = false\n') + _real_submodule_git_dir(repo / '.git/modules/vendor') + real = repo / '.git/modules/vendor/library' + _real_submodule_git_dir(real) + (real / 'config').write_text('[protocol "file"]\n\tallow = always\n[url "file:///tmp/evil/"]\n' + '\tinsteadOf = https://example.com/\n') + hidden = repo / '.git/modules/vendor/refs/library' + _real_submodule_git_dir(hidden) + (hidden / 'config').write_text('[http]\n\tproxy = evil\n') + + lib.sanitize_git_config(repo) + assert 'protocol' not in (real / 'config').read_text() + assert 'insteadOf' not in (real / 'config').read_text() + assert 'proxy' not in (hidden / 'config').read_text() + + # A submodule's `.git` must point at one of the (sanitized) git directories. + (repo / 'vendor/library').mkdir(parents=True) + (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/vendor/library\n') + assert lib.working_tree_git_error(repo) is None + (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/vendor/library/objects\n') + assert lib.working_tree_git_error(repo) + + +@pytest.mark.parametrize('link', ['objects', 'refs', 'logs', 'packed-refs', 'refs/remotes', 'objects/pack']) +@pytest.mark.asyncio +async def test_links_in_git_dir_are_refused(test_session, test_directory, async_client, test_wrolpi_config, + git_remote_factory, repos_download_manager, link): + """git writes through a link in a git directory (fetch writes objects, refs and logs), so a git directory + holding one is neither imported nor updated.""" + from modules.repos.errors import InvalidRepo + import shutil + + def plant(git_dir): + target = git_dir / link + outside = test_directory / f'outside-{link.replace("/", "-")}-{git_dir.parent.name}' + if target.exists(): + shutil.move(target, outside) + else: + outside.write_text('') + target.symlink_to(outside) + + git(test_directory, 'init', '--quiet', 'code/linked') + git(test_directory / 'code/linked', 'commit', '--quiet', '--allow-empty', '-m', 'x') + git(test_directory / 'code/linked', 'pack-refs', '--all') + git(test_directory / 'code/linked', 'gc', '--quiet') + plant(test_directory / 'code/linked/.git') + with pytest.raises(InvalidRepo, match='link'): + lib.inspect_import(test_session, 'code/linked') + + remote = git_remote_factory() + repo = lib.create_repository(test_session, remote.url) + await repos_download_manager.wait_for_all_downloads() + git(repo.directory, 'pack-refs', '--all') + git(repo.directory, 'gc', '--quiet') + plant(repo.directory / '.git') + download = await update(test_session, repos_download_manager) + assert download.status == DownloadStatus.deferred + assert 'link' in download.error + + +@pytest.mark.asyncio +async def test_control_characters_in_directories_are_refused(test_session, test_directory, async_client, + test_wrolpi_config): + """A newline would split a line git reads (e.g. alternates).""" + from modules.repos.errors import InvalidRepo + for directory in ('foo\n/tmp/abs', 'foo\rbar', 'tab\there'): + with pytest.raises(InvalidRepo, match='character'): + lib.resolve_new_repo_directory(test_session, directory) + + +@pytest.mark.parametrize('link_at', ['directory', 'parent']) +@pytest.mark.asyncio +async def test_clone_refuses_links_made_after_adding(test_session, test_directory, async_client, test_wrolpi_config, + git_remote_factory, repos_download_manager, tmp_path_factory, + link_at): + """A repo's directory (or a parent) which became a link after the repo was added is not cloned into.""" + remote = git_remote_factory() + lib.create_repository(test_session, remote.url, directory='projects/example') + outside = tmp_path_factory.mktemp('outside') + if link_at == 'directory': + (test_directory / 'projects').mkdir() + (test_directory / 'projects/example').symlink_to(outside) + else: + (test_directory / 'projects').symlink_to(outside) + await repos_download_manager.wait_for_all_downloads() + + download = test_session.query(Download).one() + assert download.status == DownloadStatus.deferred + assert 'link' in download.error + assert list(outside.iterdir()) == [] + + +@pytest.mark.asyncio +async def test_refused_sanitize_changes_nothing(test_session, test_directory, async_client, test_wrolpi_config): + """Every refusal is found before any config is rewritten.""" + from modules.repos.errors import InvalidRepo + git(test_directory, 'init', '--quiet', 'code/clone') + directory = test_directory / 'code/clone' + git(directory, 'commit', '--quiet', '--allow-empty', '-m', 'x') + git(directory, 'remote', 'add', 'origin', 'https://git.example.com/owner/clone') + with (directory / '.git/config').open('a') as fh: + fh.write('[http]\n\tproxy = http://127.0.0.1:9\n') + module = directory / '.git/modules/library' + _real_submodule_git_dir(module) + # Found only after the clone's own config was checked. + (module / 'commondir').write_text('../..\n') + before = (directory / '.git/config').read_text() + + with pytest.raises(InvalidRepo): + await lib.import_repository(test_session, 'code/clone', confirm=True) + assert (directory / '.git/config').read_text() == before