mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
repos: write sanitized git configs directly; one place formats a Repo's directory
* The sanitized config is written as INI by WROLPi (values quoted), so no kept value is ever passed to `git config` as an argument. * origin's url and fetch refspec are set with --replace-all (a planted duplicate made `git config key value` fail), in both modes. * format_repo_destination formats a Repo's directory from its name, Tag, owner and host, so a directory can be known before the Repo exists (Import shows it); Collection.format_destination uses it for repos. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
This commit is contained in:
parent
1e084958ee
commit
68880463eb
4 changed files with 92 additions and 29 deletions
|
|
@ -139,7 +139,9 @@ class GitDownloader(Downloader):
|
|||
if error := clone_ownership_error(directory, prepared.url):
|
||||
raise DownloadError(error)
|
||||
await asyncio.to_thread(sanitize_git_config, directory)
|
||||
await git('config', 'remote.origin.url', prepared.url)
|
||||
await git('config', '--replace-all', 'remote.origin.url', prepared.url)
|
||||
refspec = FULL_REFSPEC if prepared.mode == 'full' else f'+refs/heads/{branch}:refs/remotes/origin/{branch}'
|
||||
await git('config', '--replace-all', 'remote.origin.fetch', refspec)
|
||||
|
||||
old_head = await git('rev-parse', '--verify', '--quiet', 'HEAD', check=False)
|
||||
if prepared.mode == 'snapshot':
|
||||
|
|
@ -147,7 +149,6 @@ class GitDownloader(Downloader):
|
|||
target = 'FETCH_HEAD'
|
||||
else:
|
||||
# Fetch every branch and tag. Branches deleted upstream are kept (no --prune).
|
||||
await git('config', 'remote.origin.fetch', FULL_REFSPEC)
|
||||
args = ['fetch', '--progress', '--tags', '--force', 'origin']
|
||||
if (directory / '.git/shallow').exists():
|
||||
# This was a snapshot; get the whole history.
|
||||
|
|
|
|||
|
|
@ -34,12 +34,12 @@ logger = logger.getChild(__name__)
|
|||
|
||||
__all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'REPO_MARKER',
|
||||
'clone_ownership_error', 'mark_clone', 'sanitize_git_config', 'create_repository', 'update_repository',
|
||||
'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', 'repo_destination_variables',
|
||||
'repos_destination_root', 'validate_repos_destination', 'is_valid_branch', 'submodule_url_error',
|
||||
'HostAddresses', 'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config',
|
||||
'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log',
|
||||
'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config',
|
||||
'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
|
||||
'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', 'repos_destination_root',
|
||||
'validate_repos_destination', 'is_valid_branch', 'submodule_url_error', 'HostAddresses',
|
||||
'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config', 'read_readme_text',
|
||||
'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', 'get_repo_root',
|
||||
'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', 'get_repos_config',
|
||||
'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY']
|
||||
|
||||
DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value
|
||||
|
||||
|
|
@ -349,6 +349,28 @@ def _git_config_entries(config_file: pathlib.Path) -> List[Tuple[str, str]]:
|
|||
return entries
|
||||
|
||||
|
||||
def _quote_git_config(value: str) -> str:
|
||||
"""A git config value (or subsection), quoted so git reads it back exactly."""
|
||||
escaped = value.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n').replace('\t', '\\t')
|
||||
return f'"{escaped}"'
|
||||
|
||||
|
||||
def _format_git_config(entries: List[Tuple[str, str]]) -> str:
|
||||
"""Write git config entries ("section[.subsection].name", value) as a config file.
|
||||
|
||||
Written here (not with `git config`), so no value is ever read as an argument."""
|
||||
sections = {}
|
||||
for key, value in entries:
|
||||
section, _, rest = key.partition('.')
|
||||
subsection, _, name = rest.rpartition('.')
|
||||
sections.setdefault((section, subsection), []).append((name, value))
|
||||
lines = []
|
||||
for (section, subsection), values in sections.items():
|
||||
lines.append(f'[{section} {_quote_git_config(subsection)}]' if subsection else f'[{section}]')
|
||||
lines.extend(f'\t{name} = {_quote_git_config(value)}' for name, value in values)
|
||||
return '\n'.join(lines) + '\n'
|
||||
|
||||
|
||||
def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None):
|
||||
entries = _git_config_entries(config_file)
|
||||
keep = []
|
||||
|
|
@ -363,11 +385,7 @@ def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pat
|
|||
if len(keep) == len(entries):
|
||||
return
|
||||
new = config_file.with_name(f'{config_file.name}.wrolpi')
|
||||
new.unlink(missing_ok=True)
|
||||
new.touch()
|
||||
for key, value in keep:
|
||||
subprocess.run(git_command('config', '--file', str(new), '--add', key, value), env=git_env(),
|
||||
capture_output=True, check=True)
|
||||
new.write_text(_format_git_config(keep))
|
||||
os.replace(new, config_file)
|
||||
logger.warning(f'Removed {len(entries) - len(keep)} unexpected settings from {config_file}')
|
||||
|
||||
|
|
@ -516,14 +534,26 @@ def validate_repos_destination(destination: str, config=None) -> str:
|
|||
return ''
|
||||
|
||||
|
||||
def repo_destination_variables(collection: Collection) -> dict:
|
||||
"""Variables for `repos_destination` which are stored on the Repository (not the Collection)."""
|
||||
def format_repo_destination(name: str, tag_name: str = None, owner: str = None, host: str = None) -> pathlib.Path:
|
||||
"""The directory a Repo with this name, Tag, owner and host is saved in (see `repos_destination`)."""
|
||||
name = escape_file_name(name) if name else ''
|
||||
tag_name = tag_name or ''
|
||||
variables = dict(
|
||||
name=name, repo_name=name,
|
||||
tag=tag_name, repo_tag=tag_name, tag_name=tag_name,
|
||||
repo_owner=escape_file_name(owner) if owner else '',
|
||||
repo_host=escape_file_name(host) if host else '',
|
||||
)
|
||||
destination = get_wrolpi_config().repos_destination % variables
|
||||
return get_media_directory() / destination.lstrip('/')
|
||||
|
||||
|
||||
def format_collection_destination(collection: Collection, tag_name: str = None) -> pathlib.Path:
|
||||
"""The directory of a Repo's Collection (its owner and host are on its Repository)."""
|
||||
session = Session.object_session(collection)
|
||||
repo = Repository.get_by_collection_id(session, collection.id) if session and collection.id else None
|
||||
return dict(
|
||||
repo_owner=escape_file_name(repo.owner) if repo and repo.owner else '',
|
||||
repo_host=escape_file_name(repo.host) if repo and repo.host else '',
|
||||
)
|
||||
return format_repo_destination(collection.name, tag_name, repo.owner if repo else None,
|
||||
repo.host if repo else None)
|
||||
|
||||
|
||||
def get_repository_by_url(session: Session, url: str) -> Optional[Repository]:
|
||||
|
|
|
|||
|
|
@ -128,3 +128,40 @@ def test_sanitize_git_config_submodules(tmp_path):
|
|||
lib.sanitize_git_config(repo)
|
||||
assert 'worktree = ../../../vendor/library' in (module / 'config').read_text()
|
||||
assert 'worktree' not in (evil / 'config').read_text()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_update_with_duplicate_and_option_like_settings(test_session, git_remote_factory,
|
||||
repos_download_manager):
|
||||
"""Planted extra or option-like values in the kept settings cannot break or redirect an update."""
|
||||
remote = git_remote_factory()
|
||||
repo = lib.create_repository(test_session, remote.url)
|
||||
await repos_download_manager.wait_for_all_downloads()
|
||||
|
||||
config = repo.directory / '.git/config'
|
||||
config.write_text(config.read_text() + (
|
||||
'[remote "origin"]\n\turl = --file=/tmp/evil\n\turl = https://127.0.0.1/evil\n'
|
||||
'\tfetch = +refs/heads/*:refs/heads/*\n'
|
||||
'[branch "main"]\n\tmerge = "quoted \\"value\\" \\\\ and\\ttab"\n'
|
||||
))
|
||||
head = remote.commit({'new.txt': 'new'}, 'New')
|
||||
download = await update(test_session, repos_download_manager)
|
||||
assert download.status == DownloadStatus.complete, download.error
|
||||
assert repo.head_sha == head
|
||||
assert git(repo.directory, 'config', '--get-all', 'remote.origin.url') == remote.url
|
||||
assert git(repo.directory, 'config', '--get-all', 'remote.origin.fetch') == '+refs/heads/*:refs/remotes/origin/*'
|
||||
|
||||
|
||||
def test_sanitized_config_round_trips(tmp_path):
|
||||
"""Kept values are written back exactly, whatever they contain."""
|
||||
repo = tmp_path / 'repo'
|
||||
(repo / '.git').mkdir(parents=True)
|
||||
values = ['--file=/tmp/evil', 'a "quoted" value', 'back\\slash', 'tab\there', ' leading space', 'semi;colon#hash']
|
||||
subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), 'http.proxy', 'evil'], check=True)
|
||||
for value in values:
|
||||
subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), '--add', 'branch.main.merge', value],
|
||||
check=True)
|
||||
lib.sanitize_git_config(repo)
|
||||
result = git(repo, 'config', '--file', '.git/config', '--get-all', 'branch.main.merge')
|
||||
assert result.split('\n') == values
|
||||
assert 'proxy' not in (repo / '.git/config').read_text()
|
||||
|
|
|
|||
|
|
@ -1015,14 +1015,17 @@ class Collection(ModelHelper, Base):
|
|||
"""
|
||||
from wrolpi.common import get_wrolpi_config, escape_file_name
|
||||
|
||||
if self.kind == 'repo':
|
||||
# Local import to avoid circular import: collections -> repos -> collections
|
||||
from modules.repos.lib import format_collection_destination
|
||||
return format_collection_destination(self, tag_name)
|
||||
|
||||
config = get_wrolpi_config()
|
||||
|
||||
if self.kind == 'channel':
|
||||
template = config.videos_destination
|
||||
elif self.kind == 'domain':
|
||||
template = config.archive_destination
|
||||
elif self.kind == 'repo':
|
||||
template = config.repos_destination
|
||||
else:
|
||||
template = '%(kind)s/%(tag)s/%(name)s'
|
||||
|
||||
|
|
@ -1039,15 +1042,7 @@ class Collection(ModelHelper, Base):
|
|||
# Backward compatibility aliases for archives
|
||||
domain=escaped_name,
|
||||
domain_tag=tag_name or '',
|
||||
# Repos
|
||||
repo_name=escaped_name,
|
||||
repo_tag=tag_name or '',
|
||||
tag_name=tag_name or '',
|
||||
)
|
||||
if self.kind == 'repo':
|
||||
# Local import to avoid circular import: collections -> repos -> collections
|
||||
from modules.repos.lib import repo_destination_variables
|
||||
variables.update(repo_destination_variables(self))
|
||||
|
||||
destination = template % variables
|
||||
return get_media_directory() / destination.lstrip('/')
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue