mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 02:48:38 -04:00
93d878952c
Repos keeps local mirrors of git repositories, cloned over https and kept up to date on a schedule. - A Repo is a Collection (kind='repo') plus a `repository` row. The new `git` downloader clones it, then updates it (weekly by default, or Never: the clone is kept as it is and only updated on request). - Repos are saved in `repos/<tag>/<name>` (the Repos Directory, ignored by default) or a directory the user chooses. An existing clone can be imported where it is. repos.yaml is the source of truth. - A GitHub-like page shows each repo's README, files, and history, with a ZIP download. READMEs are searchable under the global search's Other tab. - git is hardened against hostile repos and configs: https only, pinned addresses with no private addresses (for the repo and its submodules), sanitized configs, no hooks or links, and only clones WROLPi made or imported are updated. - Tagging a Repo moves its clone as one directory. Repo changes wait for repos.yaml to be imported at startup. - The file mover groups unindexed files with no primary file as one FileGroup, and never replaces a file at the destination. A failed tag move restores the Collection's tag. A Collection's directory cannot leave the media directory. Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
226 lines
9.2 KiB
Python
226 lines
9.2 KiB
Python
import shutil
|
|
import subprocess
|
|
|
|
import pytest
|
|
from mock import mock
|
|
|
|
from modules.repos import lib
|
|
from modules.repos.downloader import parse_git_progress
|
|
from modules.repos.models import Repository
|
|
from wrolpi.downloader import Download, DownloadStatus
|
|
|
|
|
|
def git(directory, *args) -> str:
|
|
return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True,
|
|
capture_output=True).stdout.decode().strip()
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_clone_full(test_session, test_directory, git_remote_factory, repos_download_manager):
|
|
"""A new Repo is cloned with its whole history, and the Repo describes the checked out commit."""
|
|
remote = git_remote_factory('kiwix-tools', owner='kiwix', files={'README.md': '# Kiwix\n', 'src/main.c': 'int'})
|
|
remote.commit({'CHANGELOG': 'v2'}, 'Second commit')
|
|
remote.git('tag', 'v2')
|
|
|
|
repo = lib.create_repository(test_session, remote.url, tag_name='software')
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
directory = test_directory / 'repos/software/kiwix-tools'
|
|
assert repo.directory == directory
|
|
assert (directory / 'README.md').read_text() == '# Kiwix\n'
|
|
assert (directory / 'src/main.c').is_file()
|
|
# Full mode has all the history and tags.
|
|
assert git(directory, 'rev-list', '--count', 'HEAD') == '2'
|
|
assert git(directory, 'tag') == 'v2'
|
|
assert not (directory / '.git/shallow').exists()
|
|
|
|
assert repo.head_sha == remote.head
|
|
assert repo.head_message == 'Second commit'
|
|
assert repo.head_date is not None
|
|
assert repo.default_branch == 'main'
|
|
assert repo.readme_path == 'README.md'
|
|
assert repo.size > 0
|
|
assert repo.last_fetch is not None
|
|
|
|
download, = test_session.query(Download).all()
|
|
assert download.status == DownloadStatus.complete
|
|
assert download.location == f'/repos/{repo.id}'
|
|
assert download.error is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update(test_session, test_directory, git_remote_factory, repos_download_manager):
|
|
"""An update fetches new commits, and discards local changes (a Repo is a mirror)."""
|
|
remote = git_remote_factory()
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
directory = repo.directory
|
|
|
|
# Local changes.
|
|
(directory / 'README.md').write_text('local change')
|
|
(directory / 'untracked.txt').write_text('untracked')
|
|
|
|
new_head = remote.commit({'new.txt': 'new'}, 'New file')
|
|
download, = test_session.query(Download).all()
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
assert (directory / 'new.txt').read_text() == 'new'
|
|
assert (directory / 'README.md').read_text() == '# example\n\nThe example repo.\n'
|
|
assert not (directory / 'untracked.txt').exists()
|
|
assert repo.head_sha == new_head
|
|
assert repo.head_message == 'New file'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_update_keeps_history(test_session, git_remote_factory, repos_download_manager):
|
|
"""Branches deleted upstream are kept, and history rewritten by a force push is saved in a backup ref."""
|
|
remote = git_remote_factory()
|
|
remote.git('branch', 'feature')
|
|
old_head = remote.commit({'a.txt': 'a'}, 'Will be rewritten')
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
directory = repo.directory
|
|
assert 'origin/feature' in git(directory, 'branch', '--remotes')
|
|
|
|
# Upstream deletes a branch, and rewrites main.
|
|
remote.git('branch', '--delete', 'feature')
|
|
remote.git('reset', '--hard', 'HEAD~1')
|
|
new_head = remote.commit({'b.txt': 'b'}, 'Rewritten')
|
|
|
|
download, = test_session.query(Download).all()
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
assert repo.head_sha == new_head
|
|
assert 'origin/feature' in git(directory, 'branch', '--remotes')
|
|
backups = git(directory, 'for-each-ref', '--format=%(objectname)', 'refs/wrolpi/backup/')
|
|
assert backups == old_head
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_snapshot(test_session, git_remote_factory, repos_download_manager):
|
|
"""A snapshot only has the latest commit; switching to full fetches the whole history."""
|
|
remote = git_remote_factory()
|
|
remote.commit({'a.txt': 'a'}, 'Second')
|
|
repo = lib.create_repository(test_session, remote.url, mode='snapshot')
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
directory = repo.directory
|
|
assert (directory / '.git/shallow').is_file()
|
|
assert git(directory, 'rev-list', '--count', 'HEAD') == '1'
|
|
|
|
# Snapshot updates stay shallow.
|
|
remote.commit({'b.txt': 'b'}, 'Third')
|
|
download, = test_session.query(Download).all()
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
assert (directory / 'b.txt').is_file()
|
|
assert git(directory, 'rev-list', '--count', 'HEAD') == '1'
|
|
|
|
lib.update_repository(test_session, repo.id, mode='full')
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
assert not (directory / '.git/shallow').exists()
|
|
assert git(directory, 'rev-list', '--count', 'HEAD') == '3'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_branch(test_session, git_remote_factory, repos_download_manager):
|
|
"""A Repo can follow a branch other than the default branch."""
|
|
remote = git_remote_factory()
|
|
remote.git('checkout', '--quiet', '-b', 'dev')
|
|
dev_head = remote.commit({'dev.txt': 'dev'}, 'Dev')
|
|
remote.git('checkout', '--quiet', 'main')
|
|
|
|
repo = lib.create_repository(test_session, f'{remote.url}/tree/dev')
|
|
assert repo.branch == 'dev'
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
assert repo.head_sha == dev_head
|
|
assert repo.default_branch == 'main'
|
|
assert git(repo.directory, 'branch', '--show-current') == 'dev'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_upstream_gone(test_session, git_remote_factory, repos_download_manager):
|
|
"""When the upstream disappears the download is deferred with git's error; the local copy is kept."""
|
|
remote = git_remote_factory()
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
directory = repo.directory
|
|
head = repo.head_sha
|
|
|
|
shutil.rmtree(remote.path)
|
|
download, = test_session.query(Download).all()
|
|
download.renew()
|
|
test_session.commit()
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
test_session.expire_all()
|
|
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'git ls-remote failed' in download.error
|
|
assert (directory / 'README.md').is_file()
|
|
assert repo.head_sha == head
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_symlinks_are_not_followed(test_session, git_remote_factory, repos_download_manager):
|
|
"""A symlink in a repo is checked out as a plain file, so it cannot expose files outside the repo."""
|
|
remote = git_remote_factory()
|
|
(remote.path / 'shadow').symlink_to('/etc/passwd')
|
|
remote.git('add', 'shadow')
|
|
remote.git('commit', '--quiet', '-m', 'Symlink')
|
|
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
|
|
shadow = repo.directory / 'shadow'
|
|
assert not shadow.is_symlink()
|
|
assert shadow.read_text() == '/etc/passwd'
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_refuses_non_empty_directory(test_session, test_directory, git_remote_factory, repos_download_manager):
|
|
"""A Repo is never cloned over existing files."""
|
|
remote = git_remote_factory()
|
|
directory = test_directory / 'repos/example'
|
|
directory.mkdir(parents=True)
|
|
(directory / 'important.txt').write_text('important')
|
|
|
|
lib.create_repository(test_session, remote.url)
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
|
|
download, = test_session.query(Download).all()
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'not empty' in download.error
|
|
assert (directory / 'important.txt').read_text() == 'important'
|
|
assert test_session.query(Repository).one().head_sha is None
|
|
|
|
|
|
def test_parse_git_progress():
|
|
assert parse_git_progress('Receiving objects: 45% (123/456), 1.20 MiB | 1.00 MiB/s') == \
|
|
dict(stage='Receiving objects', percent=45)
|
|
assert parse_git_progress('remote: Counting objects: 100% (5/5), done.') == \
|
|
dict(stage='Counting objects', percent=100)
|
|
assert parse_git_progress("Cloning into 'example'...") is None
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_invalid_remote_default_branch(test_session, git_remote_factory, repos_download_manager):
|
|
"""The remote chooses its default branch name; an invalid one is never passed to git."""
|
|
remote = git_remote_factory()
|
|
repo = lib.create_repository(test_session, remote.url)
|
|
with mock.patch('modules.repos.downloader.GitDownloader._default_branch', return_value='--upload-pack=x'):
|
|
await repos_download_manager.wait_for_all_downloads()
|
|
|
|
download, = test_session.query(Download).all()
|
|
assert download.status == DownloadStatus.deferred
|
|
assert 'Invalid branch' in download.error
|
|
assert not repo.directory.exists()
|