import os import pytest from mock import mock from modules.repos import lib from modules.repos.errors import InvalidRepo, RepoConflict from modules.repos.models import Repository from wrolpi.collections.models import Collection from wrolpi.common import get_wrolpi_config from wrolpi.downloader import Download, DownloadFrequency from wrolpi.errors import UnknownDirectory, ValidationError from wrolpi.files.models import FileGroup @pytest.mark.parametrize('url,expected', [ ('https://github.com/kiwix/kiwix-tools', lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'kiwix-tools')), ('https://GitHub.com/kiwix/kiwix-tools.git/', lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools.git', 'github.com', 'kiwix', 'kiwix-tools')), ('https://github.com/kiwix/kiwix-tools/tree/feature/x', lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'kiwix-tools', 'feature/x')), ('https://gitlab.com/group/sub/project/-/tree/dev', lib.ParsedRepoURL('https://gitlab.com/group/sub/project', 'gitlab.com', 'group/sub', 'project', 'dev')), ('https://git.example.com:8443/project', lib.ParsedRepoURL('https://git.example.com:8443/project', 'git.example.com', '', 'project')), ]) def test_parse_repo_url(url, expected): assert lib.parse_repo_url(url) == expected @pytest.mark.parametrize('url', [ 'http://github.com/kiwix/kiwix-tools', 'git@github.com:kiwix/kiwix-tools.git', 'ssh://git@github.com/kiwix/kiwix-tools', 'file:///etc', 'https://user:token@github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools?tab=readme', 'https://github.com', 'https://github.com/a/..', 'https://github.com/../b', 'https://github.com/a/./b', '', ]) def test_parse_repo_url_invalid(url): with pytest.raises(InvalidRepo): lib.parse_repo_url(url) def test_repo_url_key(): assert lib.repo_url_key('https://GitHub.com/a/b.git/') == lib.repo_url_key('https://github.com/a/b') assert lib.repo_url_key('https://github.com/a/b') != lib.repo_url_key('https://github.com/a/c') def test_git_command_is_restricted(): cmd = lib.git_command('clone', 'https://example.com/a') assert 'protocol.allow=never' in cmd assert 'protocol.https.allow=always' in cmd assert 'core.symlinks=false' in cmd assert 'core.hooksPath=/dev/null' in cmd assert cmd[-2:] == ('clone', 'https://example.com/a') assert lib.git_env()['GIT_TERMINAL_PROMPT'] == '0' @pytest.mark.asyncio async def test_create_repository(test_session, test_directory, test_wrolpi_config, async_client): repo = lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software') collection = test_session.query(Collection).one() assert collection.kind == 'repo' assert collection.name == 'kiwix-tools' assert collection.tag_name == 'software' assert collection.directory == test_directory / 'repos/software/kiwix-tools' assert repo.collection_id == collection.id assert (repo.url, repo.host, repo.owner, repo.mode, repo.branch) == \ ('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'full', None) # Repos are always kept up to date; weekly by default. download = test_session.query(Download).one() assert download.url == repo.url assert download.downloader == 'git' assert download.frequency == DownloadFrequency.weekly assert download.collection_id == collection.id assert repo.download == download # Without a tag. repo = lib.create_repository(test_session, 'https://github.com/kiwix/libkiwix') assert repo.directory == test_directory / 'repos/libkiwix' @pytest.mark.asyncio async def test_create_repository_conflicts(test_session, test_wrolpi_config, async_client): lib.create_repository(test_session, 'https://github.com/a/utils') # The same repo, even with `.git`. with pytest.raises(RepoConflict): lib.create_repository(test_session, 'https://github.com/a/utils.git') # Another owner's repo with the same name; a new name is suggested. with pytest.raises(RepoConflict) as e: lib.create_repository(test_session, 'https://github.com/b/utils') assert "'b-utils'" in str(e.value) test_session.rollback() repo = lib.create_repository(test_session, 'https://github.com/b/utils', name='b-utils') assert repo.name == 'b-utils' assert test_session.query(Repository).count() == 2 @pytest.mark.asyncio async def test_create_repository_invalid(test_session, test_wrolpi_config, async_client): with pytest.raises(InvalidRepo): lib.create_repository(test_session, 'http://github.com/a/b') with pytest.raises(Exception): lib.create_repository(test_session, 'https://github.com/a/b', mode='bare') with pytest.raises(Exception): lib.create_repository(test_session, 'https://github.com/a/b', frequency=0) @pytest.mark.asyncio async def test_repos_destination(test_session, test_directory, test_wrolpi_config, async_client): """The repos destination can use the repo's owner and host.""" get_wrolpi_config().repos_destination = 'repos/%(repo_host)s/%(repo_owner)s/%(repo_name)s' repo = lib.create_repository(test_session, 'https://gitlab.com/group/sub/project') assert repo.directory == test_directory / 'repos/gitlab.com/group⧸sub/project' get_wrolpi_config().repos_destination = 'repos/%(tag_name)s/%(repo_name)s' repo = lib.create_repository(test_session, 'https://github.com/a/b', tag_name='Tag') assert repo.directory == test_directory / 'repos/Tag/b' @pytest.mark.asyncio async def test_update_repository(test_session, test_wrolpi_config, async_client): repo = lib.create_repository(test_session, 'https://github.com/a/b', branch='dev') lib.update_repository(test_session, repo.id, description='My repo', frequency=DownloadFrequency.daily, mode='snapshot', branch='') assert repo.collection.description == 'My repo' assert repo.download.frequency == DownloadFrequency.daily assert repo.mode == 'snapshot' # An empty branch follows the default branch. assert repo.branch is None @pytest.mark.asyncio async def test_delete_repository(test_session, test_directory, test_wrolpi_config, await_background_tasks, async_client): keep = lib.create_repository(test_session, 'https://github.com/a/keep') delete = lib.create_repository(test_session, 'https://github.com/a/delete') for repo in (keep, delete): repo.directory.mkdir(parents=True) (repo.directory / 'README.md').write_text('readme') keep_directory, delete_directory = keep.directory, delete.directory await lib.delete_repository(test_session, keep.id) await lib.delete_repository(test_session, delete.id, delete_files=True) await await_background_tasks() assert test_session.query(Repository).count() == 0 assert test_session.query(Collection).count() == 0 assert test_session.query(Download).count() == 0 # Files are kept by default. assert (keep_directory / 'README.md').is_file() assert not delete_directory.exists() @pytest.mark.asyncio async def test_list_repo_tree(test_session, test_directory, test_wrolpi_config, async_client): repo = lib.create_repository(test_session, 'https://github.com/a/b') with pytest.raises(UnknownDirectory): lib.list_repo_tree(repo) root = repo.directory (root / '.git').mkdir(parents=True) (root / 'src/lib').mkdir(parents=True) (root / 'src/Readme.rst').write_text('src') (root / 'README.md').write_text('# B') (root / 'a.txt').write_text('aaa') (root / 'Docs').mkdir() tree = lib.list_repo_tree(repo) assert tree == dict( path='', readme_path='README.md', entries=[ dict(name='Docs', path='Docs', is_dir=True, size=None), dict(name='src', path='src', is_dir=True, size=None), dict(name='a.txt', path='a.txt', is_dir=False, size=3), dict(name='README.md', path='README.md', is_dir=False, size=3), ], ) tree = lib.list_repo_tree(repo, '/src/') assert tree['path'] == 'src' assert tree['readme_path'] == 'src/Readme.rst' assert [i['path'] for i in tree['entries']] == ['src/lib', 'src/Readme.rst'] with pytest.raises(InvalidRepo): lib.list_repo_tree(repo, '../..') with pytest.raises(UnknownDirectory): lib.list_repo_tree(repo, 'missing') @pytest.mark.asyncio async def test_repos_directory_is_ignored(test_session, test_directory, test_wrolpi_config, refresh_files): """Files in the repos directory are never indexed, even when an old config does not ignore it.""" config = get_wrolpi_config() config.ignored_directories = ['config'] config.import_config() assert 'repos' in config.ignored_directories (test_directory / 'repos/a/b').mkdir(parents=True) (test_directory / 'repos/a/b/README.md').write_text('# B') (test_directory / 'other.txt').write_text('other') await refresh_files() assert [i.primary_path.name for i in test_session.query(FileGroup)] == ['other.txt'] @pytest.mark.parametrize('branch,valid', [ ('main', True), ('feature/x-1', True), ('release-1.2', True), ('--upload-pack=touch /tmp/pwned', False), ('-b', False), ('a..b', False), ('a b', False), ('a~1', False), ('a^', False), ('a:b', False), ('a?', False), ('a*', False), ('a[', False), ('a\\b', False), ('a@{1}', False), ('@', False), ('a/', False), ('/a', False), ('a//b', False), ('a.lock', False), ('a/.b', False), ('a.', False), ('a\x00b', False), ('', False), ]) def test_is_valid_branch(branch, valid): assert lib.is_valid_branch(branch) is valid @pytest.mark.asyncio async def test_invalid_branch_rejected(test_session, async_client, test_wrolpi_config): """A branch is passed to git, it must never be read as an option.""" with pytest.raises(ValidationError): lib.create_repository(test_session, 'https://github.com/a/b', branch='--upload-pack=x') with pytest.raises(ValidationError): lib.create_repository(test_session, 'https://github.com/a/b/tree/a..b') repo = lib.create_repository(test_session, 'https://github.com/a/b') with pytest.raises(ValidationError): lib.update_repository(test_session, repo.id, branch='-x') @pytest.mark.asyncio async def test_repo_directory_traversal(test_session, test_directory, async_client, test_wrolpi_config): """A Repo's directory is always strictly inside the media directory, and never escapes its template.""" for name in ('..', '.', ' '): with pytest.raises(InvalidRepo): lib.create_repository(test_session, 'https://github.com/a/b', tag_name='tag', name=name) test_session.rollback() with pytest.raises(InvalidRepo): lib.create_repository(test_session, 'https://github.com/a/b', directory=test_directory / 'repos/../../etc') test_session.rollback() with pytest.raises(InvalidRepo): lib.create_repository(test_session, 'https://github.com/a/b', directory=test_directory) test_session.rollback() # The config can not move an existing Repo outside the media directory. lib.create_repository(test_session, 'https://github.com/a/b') with pytest.raises(InvalidRepo): lib.ReposConfig._import_repo(test_session, dict(url='https://github.com/a/b', directory='../outside')) test_session.rollback() assert test_session.query(Repository).one().directory == test_directory / 'repos/b' def test_git_env_is_not_inherited(): """The process environment cannot undo git_command's restrictions.""" hostile = dict( GIT_ALLOW_PROTOCOL='file', GIT_CONFIG_COUNT='1', GIT_CONFIG_KEY_0='url.https://127.0.0.1/.insteadOf', GIT_CONFIG_VALUE_0='https://example.com/', GIT_CONFIG_PARAMETERS="'protocol.allow'='always'", GIT_SSL_NO_VERIFY='1', GIT_PROXY_COMMAND='evil', GIT_SSH_COMMAND='evil', HTTPS_PROXY='http://127.0.0.1:9', https_proxy='http://127.0.0.1:9', CURL_CA_BUNDLE='/tmp/evil.pem', GIT_DIR='/tmp/evil', ) with mock.patch.dict(os.environ, dict(hostile, PATH='/usr/bin:/bin', SSL_CERT_FILE='/etc/ssl/cert.pem')): env = lib.git_env() assert not set(hostile) & set(env), set(hostile) & set(env) assert env['PATH'] == '/usr/bin:/bin' assert env['SSL_CERT_FILE'] == '/etc/ssl/cert.pem' assert env['GIT_TERMINAL_PROMPT'] == '0' def test_git_command_ignores_hostile_environment(tmp_path): """With git's environment variables set against it, git still refuses file:// (only https is allowed).""" import subprocess origin = tmp_path / 'origin' subprocess.run(['git', 'init', '--quiet', str(origin)], check=True) with mock.patch.dict(os.environ, dict(GIT_ALLOW_PROTOCOL='file')): result = subprocess.run(lib.git_command('ls-remote', f'file://{origin}'), env=lib.git_env(), capture_output=True) assert result.returncode != 0 assert b"transport 'file' not allowed" in result.stderr