mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
4b034ae3ee
* A submodule's working-tree `.git` names the git directory git uses for it. Before any submodule command (update, or deinit when submodules are turned off), and before an import, every `.git` below the top of the working tree must be a file pointing inside the repo's own .git/modules; a link, a directory, or a file pointing elsewhere could make git use a config that was never checked. * Import validates and sanitizes the clone where it is, then moves it, and moves it back if marking it fails, so a refused or failed import never leaves a clone moved without a Repo. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
239 lines
12 KiB
Python
239 lines
12 KiB
Python
import json
|
|
import subprocess
|
|
from http import HTTPStatus
|
|
|
|
import pytest
|
|
|
|
from modules.repos import lib
|
|
from modules.repos.errors import InvalidRepo, RepoConflict
|
|
from modules.repos.models import Repository
|
|
from wrolpi.downloader import Download, DownloadStatus
|
|
from wrolpi.errors import UnknownDirectory, ValidationError
|
|
from wrolpi.files.models import FileGroup
|
|
|
|
|
|
def git(directory, *args) -> str:
|
|
env = dict(GIT_AUTHOR_NAME='t', GIT_AUTHOR_EMAIL='t@t', GIT_COMMITTER_NAME='t', GIT_COMMITTER_EMAIL='t@t',
|
|
PATH='/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin')
|
|
return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True, env=env,
|
|
capture_output=True).stdout.decode().strip()
|
|
|
|
|
|
@pytest.fixture
|
|
def user_clone(test_directory, git_remote_factory):
|
|
"""A clone the user made themselves, in their media directory, with an `origin`."""
|
|
|
|
def factory(relative='code/kiwix-tools', remote=None):
|
|
remote = remote or git_remote_factory('kiwix-tools', owner='kiwix', files={'README.md': '# Kiwix tools'})
|
|
directory = test_directory / relative
|
|
directory.parent.mkdir(parents=True, exist_ok=True)
|
|
git(directory.parent, 'clone', '--quiet', str(remote.path), directory.name)
|
|
git(directory, 'remote', 'set-url', 'origin', remote.url)
|
|
return directory, remote
|
|
|
|
return factory
|
|
|
|
|
|
@pytest.mark.parametrize('origin,expected', [
|
|
('https://github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools'),
|
|
('https://github.com/kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools.git'),
|
|
('git@github.com:kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools'),
|
|
('ssh://git@github.com:22/kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools'),
|
|
('git://github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools'),
|
|
('/home/user/kiwix-tools', None),
|
|
('../kiwix-tools', None),
|
|
])
|
|
def test_remote_https_url(origin, expected):
|
|
assert lib.remote_https_url(origin) == expected
|
|
|
|
|
|
def test_remote_url_key():
|
|
assert lib.remote_url_key('git@GitHub.com:kiwix/kiwix-tools.git') == \
|
|
lib.remote_url_key('https://github.com/kiwix/kiwix-tools/') == 'github.com/kiwix/kiwix-tools'
|
|
assert lib.remote_url_key('https://github.com/kiwix/other') != lib.remote_url_key('https://github.com/kiwix/x')
|
|
assert lib.remote_url_key('/local/path') is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_inspect_import(test_session, async_client, test_wrolpi_config, user_clone):
|
|
"""Inspecting a clone reports what an import would do, and changes nothing."""
|
|
directory, remote = user_clone()
|
|
git(directory, 'commit', '--quiet', '--allow-empty', '-m', 'Local work')
|
|
config_before = (directory / '.git/config').read_text()
|
|
|
|
request, response = await async_client.post('/api/repos/import/inspect',
|
|
content=json.dumps(dict(directory='code/kiwix-tools',
|
|
tag_name='Software')))
|
|
assert response.status_code == HTTPStatus.OK, response.json
|
|
info = response.json['inspection']
|
|
assert info['directory'] == 'code/kiwix-tools'
|
|
assert info['origin'] == remote.url
|
|
assert info['url'] == remote.url
|
|
assert info['name'] == 'kiwix-tools'
|
|
assert info['branch'] == 'main'
|
|
assert info['head_message'] == 'Local work'
|
|
assert info['head_sha'] == git(directory, 'rev-parse', 'HEAD')
|
|
assert info['head_date']
|
|
assert info['local_commits'] == 1
|
|
assert info['shallow'] is False
|
|
assert info['destination'] == 'repos/Software/kiwix-tools'
|
|
assert info['will_move'] is True
|
|
assert info['destination_exists'] is False
|
|
|
|
assert (directory / '.git/config').read_text() == config_before
|
|
assert not (directory / '.git' / lib.REPO_MARKER).exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_inspect_import_refuses(test_session, test_directory, async_client, test_wrolpi_config, user_clone):
|
|
(test_directory / 'not-git').mkdir()
|
|
with pytest.raises(InvalidRepo):
|
|
lib.inspect_import(test_session, 'not-git')
|
|
with pytest.raises(UnknownDirectory):
|
|
lib.inspect_import(test_session, 'missing')
|
|
for directory in ('', '/etc', '../outside', 'config', 'tags', 'videos'):
|
|
(test_directory / 'videos').mkdir(exist_ok=True)
|
|
with pytest.raises((InvalidRepo, UnknownDirectory)):
|
|
lib.inspect_import(test_session, directory)
|
|
|
|
# `.git` must be the clone's own directory.
|
|
(test_directory / 'gitfile').mkdir()
|
|
(test_directory / 'gitfile/.git').write_text('gitdir: /etc\n')
|
|
with pytest.raises(InvalidRepo):
|
|
lib.inspect_import(test_session, 'gitfile')
|
|
|
|
# A partial clone's missing objects would come from the network.
|
|
directory, _ = user_clone('code/partial')
|
|
git(directory, 'config', 'remote.origin.promisor', 'true')
|
|
with pytest.raises(InvalidRepo, match='partial clone'):
|
|
lib.inspect_import(test_session, 'code/partial')
|
|
|
|
# Already a Repo, or inside one.
|
|
repo = lib.create_repository(test_session, 'https://github.com/a/b')
|
|
repo.directory.mkdir(parents=True)
|
|
git(repo.directory, 'init', '--quiet')
|
|
with pytest.raises(RepoConflict):
|
|
lib.inspect_import(test_session, 'repos/b')
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_import(test_session, test_directory, async_client, test_wrolpi_config, user_clone,
|
|
repos_download_manager, refresh_files):
|
|
"""An imported clone is moved into the Repos directory, owned by WROLPi, and updated like any Repo."""
|
|
directory, remote = user_clone()
|
|
(directory / 'local-note.txt').write_text('untracked')
|
|
with (directory / '.git/config').open('a') as fh:
|
|
fh.write('[http]\n\tproxy = http://127.0.0.1:9\n[user]\n\tname = Someone\n')
|
|
# The clone was indexed where it was.
|
|
await refresh_files()
|
|
assert test_session.query(FileGroup).filter(FileGroup.primary_path.contains('code/kiwix-tools')).count()
|
|
|
|
body = dict(directory='code/kiwix-tools', tag_name='Software', confirm=True, frequency=86400)
|
|
request, response = await async_client.post('/api/repos/import', content=json.dumps(body))
|
|
assert response.status_code == HTTPStatus.CREATED, response.json
|
|
imported = response.json['repo']
|
|
assert imported['name'] == 'kiwix-tools'
|
|
assert imported['url'] == remote.url
|
|
assert imported['directory'] == 'repos/Software/kiwix-tools'
|
|
assert imported['tag_name'] == 'Software'
|
|
assert imported['frequency'] == 86400
|
|
assert imported['branch'] is None # Follows the default branch, like a new Repo.
|
|
assert imported['head_sha'] == remote.head
|
|
assert imported['readme_path'] == 'README.md'
|
|
|
|
new_directory = test_directory / 'repos/Software/kiwix-tools'
|
|
assert not directory.exists()
|
|
assert (new_directory / 'README.md').is_file()
|
|
assert (new_directory / '.git' / lib.REPO_MARKER).is_file()
|
|
config = (new_directory / '.git/config').read_text()
|
|
assert 'proxy' not in config and 'Someone' not in config
|
|
# The old location is forgotten; the new one is never indexed.
|
|
await refresh_files()
|
|
assert test_session.query(FileGroup).filter(FileGroup.primary_path.contains('kiwix-tools')).count() == 0
|
|
|
|
# From now on it is a mirror: an update fetches the origin, and discards local files.
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
head = remote.commit({'new.txt': 'new'}, 'New upstream')
|
|
download = test_session.query(Download).one()
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
download = test_session.query(Download).one()
|
|
assert download.status == DownloadStatus.complete, download.error
|
|
assert test_session.query(Repository).one().head_sha == head
|
|
assert (new_directory / 'new.txt').is_file()
|
|
assert not (new_directory / 'local-note.txt').exists()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_import_in_place(test_session, test_directory, async_client, test_wrolpi_config, user_clone):
|
|
"""A clone already where its Repo would be is not moved."""
|
|
directory, remote = user_clone('repos/kiwix-tools')
|
|
repo = await lib.import_repository(test_session, 'repos/kiwix-tools', confirm=True)
|
|
assert repo.directory == directory
|
|
assert (directory / '.git' / lib.REPO_MARKER).is_file()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_import_refuses(test_session, test_directory, async_client, test_wrolpi_config, user_clone,
|
|
git_remote_factory):
|
|
directory, remote = user_clone()
|
|
|
|
# Without confirmation.
|
|
with pytest.raises(ValidationError):
|
|
await lib.import_repository(test_session, 'code/kiwix-tools')
|
|
# A URL which is not the clone's origin.
|
|
with pytest.raises(InvalidRepo, match="not this clone's origin"):
|
|
await lib.import_repository(test_session, 'code/kiwix-tools', url='https://github.com/evil/other',
|
|
confirm=True)
|
|
# The destination is taken.
|
|
(test_directory / 'repos/kiwix-tools').mkdir(parents=True)
|
|
with pytest.raises(RepoConflict):
|
|
await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True)
|
|
test_session.rollback()
|
|
assert test_session.query(Repository).count() == 0
|
|
assert (directory / '.git').is_dir()
|
|
assert not (directory / '.git' / lib.REPO_MARKER).exists()
|
|
|
|
# Without an https origin, the URL must be given (and must still be the origin).
|
|
other, _ = user_clone('code/ssh-clone', remote=git_remote_factory('library'))
|
|
git(other, 'remote', 'set-url', 'origin', 'git@git.example.com:owner/library.git')
|
|
assert lib.inspect_import(test_session, 'code/ssh-clone')['url'] == 'https://git.example.com/owner/library'
|
|
git(other, 'remote', 'remove', 'origin')
|
|
assert lib.inspect_import(test_session, 'code/ssh-clone')['url'] is None
|
|
with pytest.raises(InvalidRepo, match='https:// URL'):
|
|
await lib.import_repository(test_session, 'code/ssh-clone', confirm=True)
|
|
repo = await lib.import_repository(test_session, 'code/ssh-clone', url='https://git.example.com/owner/library',
|
|
confirm=True)
|
|
assert repo.url == 'https://git.example.com/owner/library'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_import_checks_before_moving(test_session, test_directory, async_client, test_wrolpi_config,
|
|
user_clone):
|
|
"""Everything which can refuse an import is checked before the clone is moved."""
|
|
directory, _ = user_clone()
|
|
(directory / 'other').mkdir()
|
|
(directory / 'other/.git').write_text(f'gitdir: {test_directory}\n')
|
|
with pytest.raises(InvalidRepo, match='outside'):
|
|
await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True)
|
|
test_session.rollback()
|
|
assert (directory / '.git').is_dir()
|
|
assert not (test_directory / 'repos/kiwix-tools').exists()
|
|
assert test_session.query(Repository).count() == 0
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_import_moves_back_on_failure(test_session, test_directory, async_client, test_wrolpi_config,
|
|
user_clone):
|
|
"""If the import fails after the move, the clone is moved back."""
|
|
from mock import mock
|
|
directory, _ = user_clone()
|
|
with mock.patch('modules.repos.lib.mark_clone', side_effect=OSError('disk full')):
|
|
with pytest.raises(OSError):
|
|
await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True)
|
|
test_session.rollback()
|
|
assert (directory / '.git').is_dir()
|
|
assert not (test_directory / 'repos/kiwix-tools').exists()
|
|
assert test_session.query(Repository).count() == 0
|