repos: write sanitized git configs directly; one place formats a Repo's directory

* The sanitized config is written as INI by WROLPi (values quoted), so
  no kept value is ever passed to `git config` as an argument.
* origin's url and fetch refspec are set with --replace-all (a planted
  duplicate made `git config key value` fail), in both modes.
* format_repo_destination formats a Repo's directory from its name, Tag,
  owner and host, so a directory can be known before the Repo exists
  (Import shows it); Collection.format_destination uses it for repos.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
Roland 2026-10-06 17:59:32 -06:00
parent 1e084958ee
commit 68880463eb
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
4 changed files with 92 additions and 29 deletions

View file

@ -139,7 +139,9 @@ class GitDownloader(Downloader):
if error := clone_ownership_error(directory, prepared.url):
raise DownloadError(error)
await asyncio.to_thread(sanitize_git_config, directory)
await git('config', 'remote.origin.url', prepared.url)
await git('config', '--replace-all', 'remote.origin.url', prepared.url)
refspec = FULL_REFSPEC if prepared.mode == 'full' else f'+refs/heads/{branch}:refs/remotes/origin/{branch}'
await git('config', '--replace-all', 'remote.origin.fetch', refspec)
old_head = await git('rev-parse', '--verify', '--quiet', 'HEAD', check=False)
if prepared.mode == 'snapshot':
@ -147,7 +149,6 @@ class GitDownloader(Downloader):
target = 'FETCH_HEAD'
else:
# Fetch every branch and tag. Branches deleted upstream are kept (no --prune).
await git('config', 'remote.origin.fetch', FULL_REFSPEC)
args = ['fetch', '--progress', '--tags', '--force', 'origin']
if (directory / '.git/shallow').exists():
# This was a snapshot; get the whole history.

View file

@ -34,12 +34,12 @@ logger = logger.getChild(__name__)
__all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'REPO_MARKER',
'clone_ownership_error', 'mark_clone', 'sanitize_git_config', 'create_repository', 'update_repository',
'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', 'repo_destination_variables',
'repos_destination_root', 'validate_repos_destination', 'is_valid_branch', 'submodule_url_error',
'HostAddresses', 'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config',
'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log',
'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config',
'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', 'repos_destination_root',
'validate_repos_destination', 'is_valid_branch', 'submodule_url_error', 'HostAddresses',
'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config', 'read_readme_text',
'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', 'get_repo_root',
'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', 'get_repos_config',
'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value
@ -349,6 +349,28 @@ def _git_config_entries(config_file: pathlib.Path) -> List[Tuple[str, str]]:
return entries
def _quote_git_config(value: str) -> str:
"""A git config value (or subsection), quoted so git reads it back exactly."""
escaped = value.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n').replace('\t', '\\t')
return f'"{escaped}"'
def _format_git_config(entries: List[Tuple[str, str]]) -> str:
"""Write git config entries ("section[.subsection].name", value) as a config file.
Written here (not with `git config`), so no value is ever read as an argument."""
sections = {}
for key, value in entries:
section, _, rest = key.partition('.')
subsection, _, name = rest.rpartition('.')
sections.setdefault((section, subsection), []).append((name, value))
lines = []
for (section, subsection), values in sections.items():
lines.append(f'[{section} {_quote_git_config(subsection)}]' if subsection else f'[{section}]')
lines.extend(f'\t{name} = {_quote_git_config(value)}' for name, value in values)
return '\n'.join(lines) + '\n'
def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None):
entries = _git_config_entries(config_file)
keep = []
@ -363,11 +385,7 @@ def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pat
if len(keep) == len(entries):
return
new = config_file.with_name(f'{config_file.name}.wrolpi')
new.unlink(missing_ok=True)
new.touch()
for key, value in keep:
subprocess.run(git_command('config', '--file', str(new), '--add', key, value), env=git_env(),
capture_output=True, check=True)
new.write_text(_format_git_config(keep))
os.replace(new, config_file)
logger.warning(f'Removed {len(entries) - len(keep)} unexpected settings from {config_file}')
@ -516,14 +534,26 @@ def validate_repos_destination(destination: str, config=None) -> str:
return ''
def repo_destination_variables(collection: Collection) -> dict:
"""Variables for `repos_destination` which are stored on the Repository (not the Collection)."""
def format_repo_destination(name: str, tag_name: str = None, owner: str = None, host: str = None) -> pathlib.Path:
"""The directory a Repo with this name, Tag, owner and host is saved in (see `repos_destination`)."""
name = escape_file_name(name) if name else ''
tag_name = tag_name or ''
variables = dict(
name=name, repo_name=name,
tag=tag_name, repo_tag=tag_name, tag_name=tag_name,
repo_owner=escape_file_name(owner) if owner else '',
repo_host=escape_file_name(host) if host else '',
)
destination = get_wrolpi_config().repos_destination % variables
return get_media_directory() / destination.lstrip('/')
def format_collection_destination(collection: Collection, tag_name: str = None) -> pathlib.Path:
"""The directory of a Repo's Collection (its owner and host are on its Repository)."""
session = Session.object_session(collection)
repo = Repository.get_by_collection_id(session, collection.id) if session and collection.id else None
return dict(
repo_owner=escape_file_name(repo.owner) if repo and repo.owner else '',
repo_host=escape_file_name(repo.host) if repo and repo.host else '',
)
return format_repo_destination(collection.name, tag_name, repo.owner if repo else None,
repo.host if repo else None)
def get_repository_by_url(session: Session, url: str) -> Optional[Repository]:

View file

@ -128,3 +128,40 @@ def test_sanitize_git_config_submodules(tmp_path):
lib.sanitize_git_config(repo)
assert 'worktree = ../../../vendor/library' in (module / 'config').read_text()
assert 'worktree' not in (evil / 'config').read_text()
@pytest.mark.asyncio
async def test_update_with_duplicate_and_option_like_settings(test_session, git_remote_factory,
repos_download_manager):
"""Planted extra or option-like values in the kept settings cannot break or redirect an update."""
remote = git_remote_factory()
repo = lib.create_repository(test_session, remote.url)
await repos_download_manager.wait_for_all_downloads()
config = repo.directory / '.git/config'
config.write_text(config.read_text() + (
'[remote "origin"]\n\turl = --file=/tmp/evil\n\turl = https://127.0.0.1/evil\n'
'\tfetch = +refs/heads/*:refs/heads/*\n'
'[branch "main"]\n\tmerge = "quoted \\"value\\" \\\\ and\\ttab"\n'
))
head = remote.commit({'new.txt': 'new'}, 'New')
download = await update(test_session, repos_download_manager)
assert download.status == DownloadStatus.complete, download.error
assert repo.head_sha == head
assert git(repo.directory, 'config', '--get-all', 'remote.origin.url') == remote.url
assert git(repo.directory, 'config', '--get-all', 'remote.origin.fetch') == '+refs/heads/*:refs/remotes/origin/*'
def test_sanitized_config_round_trips(tmp_path):
"""Kept values are written back exactly, whatever they contain."""
repo = tmp_path / 'repo'
(repo / '.git').mkdir(parents=True)
values = ['--file=/tmp/evil', 'a "quoted" value', 'back\\slash', 'tab\there', ' leading space', 'semi;colon#hash']
subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), 'http.proxy', 'evil'], check=True)
for value in values:
subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), '--add', 'branch.main.merge', value],
check=True)
lib.sanitize_git_config(repo)
result = git(repo, 'config', '--file', '.git/config', '--get-all', 'branch.main.merge')
assert result.split('\n') == values
assert 'proxy' not in (repo / '.git/config').read_text()

View file

@ -1015,14 +1015,17 @@ class Collection(ModelHelper, Base):
"""
from wrolpi.common import get_wrolpi_config, escape_file_name
if self.kind == 'repo':
# Local import to avoid circular import: collections -> repos -> collections
from modules.repos.lib import format_collection_destination
return format_collection_destination(self, tag_name)
config = get_wrolpi_config()
if self.kind == 'channel':
template = config.videos_destination
elif self.kind == 'domain':
template = config.archive_destination
elif self.kind == 'repo':
template = config.repos_destination
else:
template = '%(kind)s/%(tag)s/%(name)s'
@ -1039,15 +1042,7 @@ class Collection(ModelHelper, Base):
# Backward compatibility aliases for archives
domain=escaped_name,
domain_tag=tag_name or '',
# Repos
repo_name=escaped_name,
repo_tag=tag_name or '',
tag_name=tag_name or '',
)
if self.kind == 'repo':
# Local import to avoid circular import: collections -> repos -> collections
from modules.repos.lib import repo_destination_variables
variables.update(repo_destination_variables(self))
destination = template % variables
return get_media_directory() / destination.lstrip('/')