repos: submodule links must stay in the repo; import checks before it moves

* A submodule's working-tree `.git` names the git directory git uses for
  it.  Before any submodule command (update, or deinit when submodules
  are turned off), and before an import, every `.git` below the top of
  the working tree must be a file pointing inside the repo's own
  .git/modules; a link, a directory, or a file pointing elsewhere could
  make git use a config that was never checked.
* Import validates and sanitizes the clone where it is, then moves it,
  and moves it back if marking it fails, so a refused or failed import
  never leaves a clone moved without a Repo.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-06 19:03:07 -06:00
parent d0ea6ab313
commit 4b034ae3ee
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 141 additions and 14 deletions

View file

@ -37,14 +37,14 @@ from .models import Repository, REPO_MODES
logger = logger.getChild(__name__)
__all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'REPO_MARKER',
'clone_ownership_error', 'mark_clone', 'sanitize_git_config', 'create_repository', 'remote_url_key',
'remote_https_url', 'resolve_import_directory', 'inspect_import', 'import_repository',
'update_repository', 'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme',
'repos_destination_root', 'validate_repos_destination', 'is_valid_branch', 'submodule_url_error',
'HostAddresses', 'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config',
'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log',
'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config',
'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
'clone_ownership_error', 'mark_clone', 'sanitize_git_config', 'working_tree_git_error',
'create_repository', 'remote_url_key', 'remote_https_url', 'resolve_import_directory', 'inspect_import',
'import_repository', 'update_repository', 'delete_repository', 'get_repositories', 'list_repo_tree',
'find_readme', 'repos_destination_root', 'validate_repos_destination', 'is_valid_branch',
'submodule_url_error', 'HostAddresses', 'resolve_repo_addresses', 'pin_config', 'is_public_address',
'submodule_fetch_config', 'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name',
'get_repo_log', 'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig',
'repos_config', 'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value
@ -472,6 +472,35 @@ def _sanitize_submodule_git_dirs(modules: pathlib.Path, root: pathlib.Path):
_sanitize_submodule_git_dirs(child / 'modules', root)
def working_tree_git_error(directory: pathlib.Path) -> Optional[str]:
"""Why a `.git` in the repo's working tree (below its top) must not be used, or None.
A submodule's `.git` file names the git directory git uses for it. It must point inside the repo's own
.git/modules, which is what `sanitize_git_config` checks; a link, a directory, or a file pointing elsewhere could
make git use a config which was never checked."""
root = directory.resolve()
modules = root / '.git' / 'modules'
for dir_path, dir_names, file_names in os.walk(root):
here = pathlib.Path(dir_path)
if '.git' in dir_names:
dir_names.remove('.git') # Never walk into a git directory.
if here != root:
return f'{here / ".git"} is a directory, not a submodule\'s link'
continue
if '.git' not in file_names:
continue
git_file = here / '.git'
if git_file.is_symlink():
return f'{git_file} is a link'
content = git_file.read_text(errors='replace')[:4096].strip()
if not content.startswith('gitdir: ') or '\n' in content:
return f'{git_file} is not a submodule\'s link'
target = (here / content[len('gitdir: '):]).resolve()
if modules not in target.parents:
return f'{git_file} points outside the repo\'s submodules'
return None
def find_readme(directory: pathlib.Path) -> Optional[str]:
"""Return the name of the preferred README file in `directory`, if any."""
try:
@ -869,22 +898,38 @@ async def import_repository(session: Session, directory: str, url: str = None, c
repo = _create_repository_records(session, parsed.url, tag_name, mode, branch, info['name'], description, None,
submodules)
destination = repo.directory
if destination.resolve() != source:
moving = destination.resolve() != source
if moving:
if source in destination.resolve().parents:
raise InvalidRepo('A repo cannot be moved into itself')
if destination.exists():
raise RepoConflict(f'{get_relative_to_media_directory(destination)} already exists')
# Everything which can refuse the import happens before the clone is moved.
if error := await asyncio.to_thread(working_tree_git_error, source):
raise InvalidRepo(f'{directory} cannot be imported: {error}')
try:
await asyncio.to_thread(sanitize_git_config, source)
except ValueError as e:
raise InvalidRepo(f'{directory} cannot be imported: {e}')
await asyncio.to_thread(_git_output_sync, source, 'config', '--replace-all', 'remote.origin.url', parsed.url)
if moving:
destination.parent.mkdir(parents=True, exist_ok=True)
await asyncio.to_thread(shutil.move, source, destination)
logger.info(f'Imported repo moved from {source} to {destination}')
try:
mark_clone(destination, repo.clone_token)
except BaseException:
if moving:
# Leave the clone where it was found.
await asyncio.to_thread(shutil.move, destination, source)
raise
if moving:
# Forget the files that were indexed in the old location.
from wrolpi.files.worker import file_worker
file_worker.queue_refresh([source])
await asyncio.to_thread(sanitize_git_config, destination)
await asyncio.to_thread(_git_output_sync, destination, 'config', '--replace-all', 'remote.origin.url', parsed.url)
mark_clone(destination, repo.clone_token)
for key, value in _read_head_commit(destination).items():
setattr(repo, key, value)
repo.readme_path = find_readme(destination)