mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
f49288b8ec
Branch names are passed to git as arguments and come from the API, the config, a `/tree/<branch>` URL, or the remote itself (its default branch). A name starting with `-` would be read as an option, so every branch must pass git's ref-name rules, including in the downloader just before use. A repo URL with a `.` or `..` path segment (or a `..` name) could format a directory outside its template (e.g. `repos/<tag>/..` is the whole repos directory). Such URLs and names are rejected, and every repo directory, including one from repos.yaml, must be strictly inside the media directory with no `..`. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
278 lines
11 KiB
Python
278 lines
11 KiB
Python
import pytest
|
||
|
||
from modules.repos import lib
|
||
from modules.repos.errors import InvalidRepo, RepoConflict
|
||
from modules.repos.models import Repository
|
||
from wrolpi.collections.models import Collection
|
||
from wrolpi.common import get_wrolpi_config
|
||
from wrolpi.downloader import Download, DownloadFrequency
|
||
from wrolpi.errors import UnknownDirectory, ValidationError
|
||
from wrolpi.files.models import FileGroup
|
||
|
||
|
||
@pytest.mark.parametrize('url,expected', [
|
||
('https://github.com/kiwix/kiwix-tools',
|
||
lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'kiwix-tools')),
|
||
('https://GitHub.com/kiwix/kiwix-tools.git/',
|
||
lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools.git', 'github.com', 'kiwix', 'kiwix-tools')),
|
||
('https://github.com/kiwix/kiwix-tools/tree/feature/x',
|
||
lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'kiwix-tools', 'feature/x')),
|
||
('https://gitlab.com/group/sub/project/-/tree/dev',
|
||
lib.ParsedRepoURL('https://gitlab.com/group/sub/project', 'gitlab.com', 'group/sub', 'project', 'dev')),
|
||
('https://git.example.com:8443/project',
|
||
lib.ParsedRepoURL('https://git.example.com:8443/project', 'git.example.com', '', 'project')),
|
||
])
|
||
def test_parse_repo_url(url, expected):
|
||
assert lib.parse_repo_url(url) == expected
|
||
|
||
|
||
@pytest.mark.parametrize('url', [
|
||
'http://github.com/kiwix/kiwix-tools',
|
||
'git@github.com:kiwix/kiwix-tools.git',
|
||
'ssh://git@github.com/kiwix/kiwix-tools',
|
||
'file:///etc',
|
||
'https://user:token@github.com/kiwix/kiwix-tools',
|
||
'https://github.com/kiwix/kiwix-tools?tab=readme',
|
||
'https://github.com',
|
||
'https://github.com/a/..',
|
||
'https://github.com/../b',
|
||
'https://github.com/a/./b',
|
||
'',
|
||
])
|
||
def test_parse_repo_url_invalid(url):
|
||
with pytest.raises(InvalidRepo):
|
||
lib.parse_repo_url(url)
|
||
|
||
|
||
def test_repo_url_key():
|
||
assert lib.repo_url_key('https://GitHub.com/a/b.git/') == lib.repo_url_key('https://github.com/a/b')
|
||
assert lib.repo_url_key('https://github.com/a/b') != lib.repo_url_key('https://github.com/a/c')
|
||
|
||
|
||
def test_git_command_is_restricted():
|
||
cmd = lib.git_command('clone', 'https://example.com/a')
|
||
assert 'protocol.allow=never' in cmd
|
||
assert 'protocol.https.allow=always' in cmd
|
||
assert 'core.symlinks=false' in cmd
|
||
assert 'core.hooksPath=/dev/null' in cmd
|
||
assert cmd[-2:] == ('clone', 'https://example.com/a')
|
||
assert lib.git_env()['GIT_TERMINAL_PROMPT'] == '0'
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_create_repository(test_session, test_directory, test_wrolpi_config, async_client):
|
||
repo = lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software')
|
||
|
||
collection = test_session.query(Collection).one()
|
||
assert collection.kind == 'repo'
|
||
assert collection.name == 'kiwix-tools'
|
||
assert collection.tag_name == 'software'
|
||
assert collection.directory == test_directory / 'repos/software/kiwix-tools'
|
||
assert repo.collection_id == collection.id
|
||
assert (repo.url, repo.host, repo.owner, repo.mode, repo.branch) == \
|
||
('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'full', None)
|
||
|
||
# Repos are always kept up to date; weekly by default.
|
||
download = test_session.query(Download).one()
|
||
assert download.url == repo.url
|
||
assert download.downloader == 'git'
|
||
assert download.frequency == DownloadFrequency.weekly
|
||
assert download.collection_id == collection.id
|
||
assert repo.download == download
|
||
|
||
# Without a tag.
|
||
repo = lib.create_repository(test_session, 'https://github.com/kiwix/libkiwix')
|
||
assert repo.directory == test_directory / 'repos/libkiwix'
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_create_repository_conflicts(test_session, test_wrolpi_config, async_client):
|
||
lib.create_repository(test_session, 'https://github.com/a/utils')
|
||
|
||
# The same repo, even with `.git`.
|
||
with pytest.raises(RepoConflict):
|
||
lib.create_repository(test_session, 'https://github.com/a/utils.git')
|
||
|
||
# Another owner's repo with the same name; a new name is suggested.
|
||
with pytest.raises(RepoConflict) as e:
|
||
lib.create_repository(test_session, 'https://github.com/b/utils')
|
||
assert "'b-utils'" in str(e.value)
|
||
test_session.rollback()
|
||
|
||
repo = lib.create_repository(test_session, 'https://github.com/b/utils', name='b-utils')
|
||
assert repo.name == 'b-utils'
|
||
assert test_session.query(Repository).count() == 2
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_create_repository_invalid(test_session, test_wrolpi_config, async_client):
|
||
with pytest.raises(InvalidRepo):
|
||
lib.create_repository(test_session, 'http://github.com/a/b')
|
||
with pytest.raises(Exception):
|
||
lib.create_repository(test_session, 'https://github.com/a/b', mode='bare')
|
||
with pytest.raises(Exception):
|
||
lib.create_repository(test_session, 'https://github.com/a/b', frequency=0)
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_repos_destination(test_session, test_directory, test_wrolpi_config, async_client):
|
||
"""The repos destination can use the repo's owner and host."""
|
||
get_wrolpi_config().repos_destination = 'repos/%(repo_host)s/%(repo_owner)s/%(repo_name)s'
|
||
repo = lib.create_repository(test_session, 'https://gitlab.com/group/sub/project')
|
||
assert repo.directory == test_directory / 'repos/gitlab.com/group⧸sub/project'
|
||
|
||
get_wrolpi_config().repos_destination = 'repos/%(tag_name)s/%(repo_name)s'
|
||
repo = lib.create_repository(test_session, 'https://github.com/a/b', tag_name='Tag')
|
||
assert repo.directory == test_directory / 'repos/Tag/b'
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_update_repository(test_session, test_wrolpi_config, async_client):
|
||
repo = lib.create_repository(test_session, 'https://github.com/a/b', branch='dev')
|
||
lib.update_repository(test_session, repo.id, description='My repo', frequency=DownloadFrequency.daily,
|
||
mode='snapshot', branch='')
|
||
assert repo.collection.description == 'My repo'
|
||
assert repo.download.frequency == DownloadFrequency.daily
|
||
assert repo.mode == 'snapshot'
|
||
# An empty branch follows the default branch.
|
||
assert repo.branch is None
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_delete_repository(test_session, test_directory, test_wrolpi_config, await_background_tasks,
|
||
async_client):
|
||
keep = lib.create_repository(test_session, 'https://github.com/a/keep')
|
||
delete = lib.create_repository(test_session, 'https://github.com/a/delete')
|
||
for repo in (keep, delete):
|
||
repo.directory.mkdir(parents=True)
|
||
(repo.directory / 'README.md').write_text('readme')
|
||
keep_directory, delete_directory = keep.directory, delete.directory
|
||
|
||
await lib.delete_repository(test_session, keep.id)
|
||
await lib.delete_repository(test_session, delete.id, delete_files=True)
|
||
await await_background_tasks()
|
||
|
||
assert test_session.query(Repository).count() == 0
|
||
assert test_session.query(Collection).count() == 0
|
||
assert test_session.query(Download).count() == 0
|
||
# Files are kept by default.
|
||
assert (keep_directory / 'README.md').is_file()
|
||
assert not delete_directory.exists()
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_list_repo_tree(test_session, test_directory, test_wrolpi_config, async_client):
|
||
repo = lib.create_repository(test_session, 'https://github.com/a/b')
|
||
with pytest.raises(UnknownDirectory):
|
||
lib.list_repo_tree(repo)
|
||
|
||
root = repo.directory
|
||
(root / '.git').mkdir(parents=True)
|
||
(root / 'src/lib').mkdir(parents=True)
|
||
(root / 'src/Readme.rst').write_text('src')
|
||
(root / 'README.md').write_text('# B')
|
||
(root / 'a.txt').write_text('aaa')
|
||
(root / 'Docs').mkdir()
|
||
|
||
tree = lib.list_repo_tree(repo)
|
||
assert tree == dict(
|
||
path='',
|
||
readme_path='README.md',
|
||
entries=[
|
||
dict(name='Docs', path='Docs', is_dir=True, size=None),
|
||
dict(name='src', path='src', is_dir=True, size=None),
|
||
dict(name='a.txt', path='a.txt', is_dir=False, size=3),
|
||
dict(name='README.md', path='README.md', is_dir=False, size=3),
|
||
],
|
||
)
|
||
|
||
tree = lib.list_repo_tree(repo, '/src/')
|
||
assert tree['path'] == 'src'
|
||
assert tree['readme_path'] == 'src/Readme.rst'
|
||
assert [i['path'] for i in tree['entries']] == ['src/lib', 'src/Readme.rst']
|
||
|
||
with pytest.raises(InvalidRepo):
|
||
lib.list_repo_tree(repo, '../..')
|
||
with pytest.raises(UnknownDirectory):
|
||
lib.list_repo_tree(repo, 'missing')
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_repos_directory_is_ignored(test_session, test_directory, test_wrolpi_config, refresh_files):
|
||
"""Files in the repos directory are never indexed, even when an old config does not ignore it."""
|
||
config = get_wrolpi_config()
|
||
config.ignored_directories = ['config']
|
||
config.import_config()
|
||
assert 'repos' in config.ignored_directories
|
||
|
||
(test_directory / 'repos/a/b').mkdir(parents=True)
|
||
(test_directory / 'repos/a/b/README.md').write_text('# B')
|
||
(test_directory / 'other.txt').write_text('other')
|
||
await refresh_files()
|
||
|
||
assert [i.primary_path.name for i in test_session.query(FileGroup)] == ['other.txt']
|
||
|
||
|
||
@pytest.mark.parametrize('branch,valid', [
|
||
('main', True),
|
||
('feature/x-1', True),
|
||
('release-1.2', True),
|
||
('--upload-pack=touch /tmp/pwned', False),
|
||
('-b', False),
|
||
('a..b', False),
|
||
('a b', False),
|
||
('a~1', False),
|
||
('a^', False),
|
||
('a:b', False),
|
||
('a?', False),
|
||
('a*', False),
|
||
('a[', False),
|
||
('a\\b', False),
|
||
('a@{1}', False),
|
||
('@', False),
|
||
('a/', False),
|
||
('/a', False),
|
||
('a//b', False),
|
||
('a.lock', False),
|
||
('a/.b', False),
|
||
('a.', False),
|
||
('a\x00b', False),
|
||
('', False),
|
||
])
|
||
def test_is_valid_branch(branch, valid):
|
||
assert lib.is_valid_branch(branch) is valid
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_invalid_branch_rejected(test_session, async_client, test_wrolpi_config):
|
||
"""A branch is passed to git, it must never be read as an option."""
|
||
with pytest.raises(ValidationError):
|
||
lib.create_repository(test_session, 'https://github.com/a/b', branch='--upload-pack=x')
|
||
with pytest.raises(ValidationError):
|
||
lib.create_repository(test_session, 'https://github.com/a/b/tree/a..b')
|
||
repo = lib.create_repository(test_session, 'https://github.com/a/b')
|
||
with pytest.raises(ValidationError):
|
||
lib.update_repository(test_session, repo.id, branch='-x')
|
||
|
||
|
||
@pytest.mark.asyncio
|
||
async def test_repo_directory_traversal(test_session, test_directory, async_client, test_wrolpi_config):
|
||
"""A Repo's directory is always strictly inside the media directory, and never escapes its template."""
|
||
for name in ('..', '.', ' '):
|
||
with pytest.raises(InvalidRepo):
|
||
lib.create_repository(test_session, 'https://github.com/a/b', tag_name='tag', name=name)
|
||
test_session.rollback()
|
||
|
||
with pytest.raises(InvalidRepo):
|
||
lib.create_repository(test_session, 'https://github.com/a/b', directory=test_directory / 'repos/../../etc')
|
||
test_session.rollback()
|
||
with pytest.raises(InvalidRepo):
|
||
lib.create_repository(test_session, 'https://github.com/a/b', directory=test_directory)
|
||
test_session.rollback()
|
||
|
||
# The config can not move an existing Repo outside the media directory.
|
||
lib.create_repository(test_session, 'https://github.com/a/b')
|
||
with pytest.raises(InvalidRepo):
|
||
lib.ReposConfig._import_repo(test_session, dict(url='https://github.com/a/b', directory='../outside'))
|
||
test_session.rollback()
|
||
assert test_session.query(Repository).one().directory == test_directory / 'repos/b'
|