diff --git a/modules/repos/downloader.py b/modules/repos/downloader.py index 8f8fa014..be72df0b 100644 --- a/modules/repos/downloader.py +++ b/modules/repos/downloader.py @@ -139,7 +139,9 @@ class GitDownloader(Downloader): if error := clone_ownership_error(directory, prepared.url): raise DownloadError(error) await asyncio.to_thread(sanitize_git_config, directory) - await git('config', 'remote.origin.url', prepared.url) + await git('config', '--replace-all', 'remote.origin.url', prepared.url) + refspec = FULL_REFSPEC if prepared.mode == 'full' else f'+refs/heads/{branch}:refs/remotes/origin/{branch}' + await git('config', '--replace-all', 'remote.origin.fetch', refspec) old_head = await git('rev-parse', '--verify', '--quiet', 'HEAD', check=False) if prepared.mode == 'snapshot': @@ -147,7 +149,6 @@ class GitDownloader(Downloader): target = 'FETCH_HEAD' else: # Fetch every branch and tag. Branches deleted upstream are kept (no --prune). - await git('config', 'remote.origin.fetch', FULL_REFSPEC) args = ['fetch', '--progress', '--tags', '--force', 'origin'] if (directory / '.git/shallow').exists(): # This was a snapshot; get the whole history. diff --git a/modules/repos/lib.py b/modules/repos/lib.py index bf419156..2569d529 100644 --- a/modules/repos/lib.py +++ b/modules/repos/lib.py @@ -34,12 +34,12 @@ logger = logger.getChild(__name__) __all__ = ['ParsedRepoURL', 'parse_repo_url', 'repo_url_key', 'git_command', 'git_env', 'REPO_MARKER', 'clone_ownership_error', 'mark_clone', 'sanitize_git_config', 'create_repository', 'update_repository', - 'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', 'repo_destination_variables', - 'repos_destination_root', 'validate_repos_destination', 'is_valid_branch', 'submodule_url_error', - 'HostAddresses', 'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config', - 'read_readme_text', 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', - 'get_repo_root', 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', - 'get_repos_config', 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY'] + 'delete_repository', 'get_repositories', 'list_repo_tree', 'find_readme', 'repos_destination_root', + 'validate_repos_destination', 'is_valid_branch', 'submodule_url_error', 'HostAddresses', + 'resolve_repo_addresses', 'pin_config', 'is_public_address', 'submodule_fetch_config', 'read_readme_text', + 'search_repos', 'count_repos', 'search_repos_by_name', 'get_repo_log', 'get_repo_root', + 'repo_archive_name', 'stream_repo_archive', 'ReposConfig', 'repos_config', 'get_repos_config', + 'save_repos_config', 'import_repos_config', 'DEFAULT_REPO_FREQUENCY'] DEFAULT_REPO_FREQUENCY = DownloadFrequency.weekly.value @@ -349,6 +349,28 @@ def _git_config_entries(config_file: pathlib.Path) -> List[Tuple[str, str]]: return entries +def _quote_git_config(value: str) -> str: + """A git config value (or subsection), quoted so git reads it back exactly.""" + escaped = value.replace('\\', '\\\\').replace('"', '\\"').replace('\n', '\\n').replace('\t', '\\t') + return f'"{escaped}"' + + +def _format_git_config(entries: List[Tuple[str, str]]) -> str: + """Write git config entries ("section[.subsection].name", value) as a config file. + + Written here (not with `git config`), so no value is ever read as an argument.""" + sections = {} + for key, value in entries: + section, _, rest = key.partition('.') + subsection, _, name = rest.rpartition('.') + sections.setdefault((section, subsection), []).append((name, value)) + lines = [] + for (section, subsection), values in sections.items(): + lines.append(f'[{section} {_quote_git_config(subsection)}]' if subsection else f'[{section}]') + lines.extend(f'\t{name} = {_quote_git_config(value)}' for name, value in values) + return '\n'.join(lines) + '\n' + + def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pathlib.Path] = None): entries = _git_config_entries(config_file) keep = [] @@ -363,11 +385,7 @@ def _sanitize_config_file(config_file: pathlib.Path, worktree_root: Optional[pat if len(keep) == len(entries): return new = config_file.with_name(f'{config_file.name}.wrolpi') - new.unlink(missing_ok=True) - new.touch() - for key, value in keep: - subprocess.run(git_command('config', '--file', str(new), '--add', key, value), env=git_env(), - capture_output=True, check=True) + new.write_text(_format_git_config(keep)) os.replace(new, config_file) logger.warning(f'Removed {len(entries) - len(keep)} unexpected settings from {config_file}') @@ -516,14 +534,26 @@ def validate_repos_destination(destination: str, config=None) -> str: return '' -def repo_destination_variables(collection: Collection) -> dict: - """Variables for `repos_destination` which are stored on the Repository (not the Collection).""" +def format_repo_destination(name: str, tag_name: str = None, owner: str = None, host: str = None) -> pathlib.Path: + """The directory a Repo with this name, Tag, owner and host is saved in (see `repos_destination`).""" + name = escape_file_name(name) if name else '' + tag_name = tag_name or '' + variables = dict( + name=name, repo_name=name, + tag=tag_name, repo_tag=tag_name, tag_name=tag_name, + repo_owner=escape_file_name(owner) if owner else '', + repo_host=escape_file_name(host) if host else '', + ) + destination = get_wrolpi_config().repos_destination % variables + return get_media_directory() / destination.lstrip('/') + + +def format_collection_destination(collection: Collection, tag_name: str = None) -> pathlib.Path: + """The directory of a Repo's Collection (its owner and host are on its Repository).""" session = Session.object_session(collection) repo = Repository.get_by_collection_id(session, collection.id) if session and collection.id else None - return dict( - repo_owner=escape_file_name(repo.owner) if repo and repo.owner else '', - repo_host=escape_file_name(repo.host) if repo and repo.host else '', - ) + return format_repo_destination(collection.name, tag_name, repo.owner if repo else None, + repo.host if repo else None) def get_repository_by_url(session: Session, url: str) -> Optional[Repository]: diff --git a/modules/repos/test/test_ownership.py b/modules/repos/test/test_ownership.py index 82ea3e5b..754cd26d 100644 --- a/modules/repos/test/test_ownership.py +++ b/modules/repos/test/test_ownership.py @@ -128,3 +128,40 @@ def test_sanitize_git_config_submodules(tmp_path): lib.sanitize_git_config(repo) assert 'worktree = ../../../vendor/library' in (module / 'config').read_text() assert 'worktree' not in (evil / 'config').read_text() + + +@pytest.mark.asyncio +async def test_update_with_duplicate_and_option_like_settings(test_session, git_remote_factory, + repos_download_manager): + """Planted extra or option-like values in the kept settings cannot break or redirect an update.""" + remote = git_remote_factory() + repo = lib.create_repository(test_session, remote.url) + await repos_download_manager.wait_for_all_downloads() + + config = repo.directory / '.git/config' + config.write_text(config.read_text() + ( + '[remote "origin"]\n\turl = --file=/tmp/evil\n\turl = https://127.0.0.1/evil\n' + '\tfetch = +refs/heads/*:refs/heads/*\n' + '[branch "main"]\n\tmerge = "quoted \\"value\\" \\\\ and\\ttab"\n' + )) + head = remote.commit({'new.txt': 'new'}, 'New') + download = await update(test_session, repos_download_manager) + assert download.status == DownloadStatus.complete, download.error + assert repo.head_sha == head + assert git(repo.directory, 'config', '--get-all', 'remote.origin.url') == remote.url + assert git(repo.directory, 'config', '--get-all', 'remote.origin.fetch') == '+refs/heads/*:refs/remotes/origin/*' + + +def test_sanitized_config_round_trips(tmp_path): + """Kept values are written back exactly, whatever they contain.""" + repo = tmp_path / 'repo' + (repo / '.git').mkdir(parents=True) + values = ['--file=/tmp/evil', 'a "quoted" value', 'back\\slash', 'tab\there', ' leading space', 'semi;colon#hash'] + subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), 'http.proxy', 'evil'], check=True) + for value in values: + subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), '--add', 'branch.main.merge', value], + check=True) + lib.sanitize_git_config(repo) + result = git(repo, 'config', '--file', '.git/config', '--get-all', 'branch.main.merge') + assert result.split('\n') == values + assert 'proxy' not in (repo / '.git/config').read_text() diff --git a/wrolpi/collections/models.py b/wrolpi/collections/models.py index 42f3eef9..ef882b6a 100644 --- a/wrolpi/collections/models.py +++ b/wrolpi/collections/models.py @@ -1015,14 +1015,17 @@ class Collection(ModelHelper, Base): """ from wrolpi.common import get_wrolpi_config, escape_file_name + if self.kind == 'repo': + # Local import to avoid circular import: collections -> repos -> collections + from modules.repos.lib import format_collection_destination + return format_collection_destination(self, tag_name) + config = get_wrolpi_config() if self.kind == 'channel': template = config.videos_destination elif self.kind == 'domain': template = config.archive_destination - elif self.kind == 'repo': - template = config.repos_destination else: template = '%(kind)s/%(tag)s/%(name)s' @@ -1039,15 +1042,7 @@ class Collection(ModelHelper, Base): # Backward compatibility aliases for archives domain=escaped_name, domain_tag=tag_name or '', - # Repos - repo_name=escaped_name, - repo_tag=tag_name or '', - tag_name=tag_name or '', ) - if self.kind == 'repo': - # Local import to avoid circular import: collections -> repos -> collections - from modules.repos.lib import repo_destination_variables - variables.update(repo_destination_variables(self)) destination = template % variables return get_media_directory() / destination.lstrip('/')