import json import subprocess from http import HTTPStatus import pytest from modules.repos import lib from modules.repos.errors import InvalidRepo, RepoConflict from modules.repos.models import Repository from wrolpi.downloader import Download, DownloadStatus from wrolpi.errors import UnknownDirectory, ValidationError from wrolpi.files.models import FileGroup def git(directory, *args) -> str: env = dict(GIT_AUTHOR_NAME='t', GIT_AUTHOR_EMAIL='t@t', GIT_COMMITTER_NAME='t', GIT_COMMITTER_EMAIL='t@t', PATH='/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin') return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True, env=env, capture_output=True).stdout.decode().strip() @pytest.fixture def user_clone(test_directory, git_remote_factory): """A clone the user made themselves, in their media directory, with an `origin`.""" def factory(relative='code/kiwix-tools', remote=None): remote = remote or git_remote_factory('kiwix-tools', owner='kiwix', files={'README.md': '# Kiwix tools'}) directory = test_directory / relative directory.parent.mkdir(parents=True, exist_ok=True) git(directory.parent, 'clone', '--quiet', str(remote.path), directory.name) git(directory, 'remote', 'set-url', 'origin', remote.url) return directory, remote return factory @pytest.mark.parametrize('origin,expected', [ ('https://github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools'), ('https://github.com/kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools.git'), ('git@github.com:kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools'), ('ssh://git@github.com:22/kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools'), ('git://github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools'), ('/home/user/kiwix-tools', None), ('../kiwix-tools', None), ]) def test_remote_https_url(origin, expected): assert lib.remote_https_url(origin) == expected def test_remote_url_key(): assert lib.remote_url_key('git@GitHub.com:kiwix/kiwix-tools.git') == \ lib.remote_url_key('https://github.com/kiwix/kiwix-tools/') == 'github.com/kiwix/kiwix-tools' assert lib.remote_url_key('https://github.com/kiwix/other') != lib.remote_url_key('https://github.com/kiwix/x') assert lib.remote_url_key('/local/path') is None @pytest.mark.asyncio async def test_inspect_import(test_session, async_client, test_wrolpi_config, user_clone): """Inspecting a clone reports what an import would do, and changes nothing.""" directory, remote = user_clone() git(directory, 'commit', '--quiet', '--allow-empty', '-m', 'Local work') config_before = (directory / '.git/config').read_text() request, response = await async_client.post('/api/repos/import/inspect', content=json.dumps(dict(directory='code/kiwix-tools', tag_name='Software'))) assert response.status_code == HTTPStatus.OK, response.json info = response.json['inspection'] assert info['directory'] == 'code/kiwix-tools' assert info['origin'] == remote.url assert info['url'] == remote.url assert info['name'] == 'kiwix-tools' assert info['branch'] == 'main' assert info['head_message'] == 'Local work' assert info['head_sha'] == git(directory, 'rev-parse', 'HEAD') assert info['head_date'] assert info['local_commits'] == 1 assert info['shallow'] is False assert info['destination'] == 'repos/Software/kiwix-tools' assert info['will_move'] is True assert info['destination_exists'] is False assert (directory / '.git/config').read_text() == config_before assert not (directory / '.git' / lib.REPO_MARKER).exists() @pytest.mark.asyncio async def test_inspect_import_refuses(test_session, test_directory, async_client, test_wrolpi_config, user_clone): (test_directory / 'not-git').mkdir() with pytest.raises(InvalidRepo): lib.inspect_import(test_session, 'not-git') with pytest.raises(UnknownDirectory): lib.inspect_import(test_session, 'missing') for directory in ('', '/etc', '../outside', 'config', 'tags', 'videos'): (test_directory / 'videos').mkdir(exist_ok=True) with pytest.raises((InvalidRepo, UnknownDirectory)): lib.inspect_import(test_session, directory) # `.git` must be the clone's own directory. (test_directory / 'gitfile').mkdir() (test_directory / 'gitfile/.git').write_text('gitdir: /etc\n') with pytest.raises(InvalidRepo): lib.inspect_import(test_session, 'gitfile') # A partial clone's missing objects would come from the network. directory, _ = user_clone('code/partial') git(directory, 'config', 'remote.origin.promisor', 'true') with pytest.raises(InvalidRepo, match='partial clone'): lib.inspect_import(test_session, 'code/partial') # Already a Repo, or inside one. repo = lib.create_repository(test_session, 'https://github.com/a/b') repo.directory.mkdir(parents=True) git(repo.directory, 'init', '--quiet') with pytest.raises(RepoConflict): lib.inspect_import(test_session, 'repos/b') @pytest.mark.asyncio async def test_import(test_session, test_directory, async_client, test_wrolpi_config, user_clone, repos_download_manager, refresh_files): """An imported clone is moved into the Repos directory, owned by WROLPi, and updated like any Repo.""" directory, remote = user_clone() (directory / 'local-note.txt').write_text('untracked') with (directory / '.git/config').open('a') as fh: fh.write('[http]\n\tproxy = http://127.0.0.1:9\n[user]\n\tname = Someone\n') # The clone was indexed where it was. await refresh_files() assert test_session.query(FileGroup).filter(FileGroup.primary_path.contains('code/kiwix-tools')).count() body = dict(directory='code/kiwix-tools', tag_name='Software', confirm=True, frequency=86400) request, response = await async_client.post('/api/repos/import', content=json.dumps(body)) assert response.status_code == HTTPStatus.CREATED, response.json imported = response.json['repo'] assert imported['name'] == 'kiwix-tools' assert imported['url'] == remote.url assert imported['directory'] == 'repos/Software/kiwix-tools' assert imported['tag_name'] == 'Software' assert imported['frequency'] == 86400 assert imported['branch'] is None # Follows the default branch, like a new Repo. assert imported['head_sha'] == remote.head assert imported['readme_path'] == 'README.md' new_directory = test_directory / 'repos/Software/kiwix-tools' assert not directory.exists() assert (new_directory / 'README.md').is_file() assert (new_directory / '.git' / lib.REPO_MARKER).is_file() config = (new_directory / '.git/config').read_text() assert 'proxy' not in config and 'Someone' not in config # The old location is forgotten; the new one is never indexed. await refresh_files() assert test_session.query(FileGroup).filter(FileGroup.primary_path.contains('kiwix-tools')).count() == 0 # From now on it is a mirror: an update fetches the origin, and discards local files. await repos_download_manager.wait_for_all_downloads() head = remote.commit({'new.txt': 'new'}, 'New upstream') download = test_session.query(Download).one() download.renew() test_session.commit() await repos_download_manager.wait_for_all_downloads() test_session.expire_all() download = test_session.query(Download).one() assert download.status == DownloadStatus.complete, download.error assert test_session.query(Repository).one().head_sha == head assert (new_directory / 'new.txt').is_file() assert not (new_directory / 'local-note.txt').exists() @pytest.mark.asyncio async def test_import_in_place(test_session, test_directory, async_client, test_wrolpi_config, user_clone): """A clone already where its Repo would be is not moved.""" directory, remote = user_clone('repos/kiwix-tools') repo = await lib.import_repository(test_session, 'repos/kiwix-tools', confirm=True) assert repo.directory == directory assert (directory / '.git' / lib.REPO_MARKER).is_file() @pytest.mark.asyncio async def test_import_refuses(test_session, test_directory, async_client, test_wrolpi_config, user_clone, git_remote_factory): directory, remote = user_clone() # Without confirmation. with pytest.raises(ValidationError): await lib.import_repository(test_session, 'code/kiwix-tools') # A URL which is not the clone's origin. with pytest.raises(InvalidRepo, match="not this clone's origin"): await lib.import_repository(test_session, 'code/kiwix-tools', url='https://github.com/evil/other', confirm=True) # The destination is taken. (test_directory / 'repos/kiwix-tools').mkdir(parents=True) with pytest.raises(RepoConflict): await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True) test_session.rollback() assert test_session.query(Repository).count() == 0 assert (directory / '.git').is_dir() assert not (directory / '.git' / lib.REPO_MARKER).exists() # Without an https origin, the URL must be given (and must still be the origin). other, _ = user_clone('code/ssh-clone', remote=git_remote_factory('library')) git(other, 'remote', 'set-url', 'origin', 'git@git.example.com:owner/library.git') assert lib.inspect_import(test_session, 'code/ssh-clone')['url'] == 'https://git.example.com/owner/library' git(other, 'remote', 'remove', 'origin') assert lib.inspect_import(test_session, 'code/ssh-clone')['url'] is None with pytest.raises(InvalidRepo, match='https:// URL'): await lib.import_repository(test_session, 'code/ssh-clone', confirm=True) repo = await lib.import_repository(test_session, 'code/ssh-clone', url='https://git.example.com/owner/library', confirm=True) assert repo.url == 'https://git.example.com/owner/library' @pytest.mark.asyncio async def test_import_checks_before_moving(test_session, test_directory, async_client, test_wrolpi_config, user_clone): """Everything which can refuse an import is checked before the clone is moved.""" directory, _ = user_clone() (directory / 'other').mkdir() (directory / 'other/.git').write_text(f'gitdir: {test_directory}\n') with pytest.raises(InvalidRepo, match='outside'): await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True) test_session.rollback() assert (directory / '.git').is_dir() assert not (test_directory / 'repos/kiwix-tools').exists() assert test_session.query(Repository).count() == 0 @pytest.mark.asyncio async def test_import_moves_back_on_failure(test_session, test_directory, async_client, test_wrolpi_config, user_clone): """If the import fails after the move, the clone is moved back.""" from mock import mock directory, _ = user_clone() with mock.patch('modules.repos.lib.mark_clone', side_effect=OSError('disk full')): with pytest.raises(OSError): await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True) test_session.rollback() assert (directory / '.git').is_dir() assert not (test_directory / 'repos/kiwix-tools').exists() assert test_session.query(Repository).count() == 0