Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
import pytest
|
|
|
|
|
|
|
|
|
|
from modules.repos import lib
|
|
|
|
|
from modules.repos.models import Repository
|
|
|
|
|
from wrolpi.collections.models import Collection
|
|
|
|
|
from wrolpi.downloader import Download, DownloadFrequency
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_dump_and_import(test_session, test_directory, async_client, test_wrolpi_config, await_switches):
|
|
|
|
|
"""repos.yaml is the source of truth; a lost database is rebuilt from it."""
|
|
|
|
|
lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software',
|
|
|
|
|
description='Tools', frequency=DownloadFrequency.daily)
|
|
|
|
|
lib.create_repository(test_session, 'https://github.com/a/b', mode='snapshot', branch='dev')
|
|
|
|
|
await await_switches()
|
|
|
|
|
|
|
|
|
|
config = lib.get_repos_config()
|
|
|
|
|
assert config.get_file().is_file()
|
|
|
|
|
assert config.repos == [
|
|
|
|
|
dict(name='b', url='https://github.com/a/b', directory='repos/b', tag_name=None, description=None,
|
2026-10-06 16:27:38 -06:00
|
|
|
mode='snapshot', branch='dev', submodules=False, frequency=DownloadFrequency.weekly),
|
Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
dict(name='kiwix-tools', url='https://github.com/kiwix/kiwix-tools', directory='repos/software/kiwix-tools',
|
2026-10-06 16:27:38 -06:00
|
|
|
tag_name='software', description='Tools', mode='full', branch=None, submodules=False,
|
|
|
|
|
frequency=DownloadFrequency.daily),
|
Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
]
|
|
|
|
|
|
|
|
|
|
# Lose the database.
|
|
|
|
|
for repo in test_session.query(Repository).all():
|
|
|
|
|
lib._delete_repository(test_session, repo)
|
|
|
|
|
test_session.commit()
|
|
|
|
|
assert test_session.query(Collection).count() == 0
|
|
|
|
|
|
|
|
|
|
config.import_config()
|
|
|
|
|
test_session.expire_all()
|
|
|
|
|
repos = {i.name: i for i in test_session.query(Repository)}
|
|
|
|
|
assert repos['kiwix-tools'].directory == test_directory / 'repos/software/kiwix-tools'
|
|
|
|
|
assert repos['kiwix-tools'].tag_name == 'software'
|
|
|
|
|
assert repos['kiwix-tools'].collection.description == 'Tools'
|
|
|
|
|
assert repos['kiwix-tools'].download.frequency == DownloadFrequency.daily
|
|
|
|
|
assert (repos['b'].mode, repos['b'].branch) == ('snapshot', 'dev')
|
|
|
|
|
assert {i.downloader for i in test_session.query(Download)} == {'git'}
|
|
|
|
|
|
|
|
|
|
# A repo removed from the config is deleted.
|
|
|
|
|
data = config.read_config_file()
|
|
|
|
|
data['repos'] = data['repos'][1:]
|
|
|
|
|
data['version'] += 1
|
|
|
|
|
config.write_config_data(data, config.get_file())
|
|
|
|
|
config.import_config()
|
|
|
|
|
test_session.expire_all()
|
|
|
|
|
assert [i.name for i in test_session.query(Repository)] == ['kiwix-tools']
|
|
|
|
|
assert test_session.query(Download).count() == 1
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_import_links_existing_download(test_session, async_client, test_wrolpi_config):
|
|
|
|
|
"""A Repo imported after download_manager.yaml re-uses its Download."""
|
|
|
|
|
lib.create_repository(test_session, 'https://github.com/a/b')
|
|
|
|
|
test_session.query(Repository).delete()
|
|
|
|
|
test_session.query(Collection).delete()
|
|
|
|
|
test_session.commit()
|
|
|
|
|
download = test_session.query(Download).one()
|
|
|
|
|
assert download.collection_id is None
|
|
|
|
|
|
|
|
|
|
config = lib.get_repos_config()
|
|
|
|
|
config.write_config_data(dict(version=1, repos=[dict(name='b', url='https://github.com/a/b', frequency=3600)]),
|
|
|
|
|
config.get_file())
|
|
|
|
|
config.import_config()
|
|
|
|
|
test_session.expire_all()
|
|
|
|
|
|
|
|
|
|
repo = test_session.query(Repository).one()
|
|
|
|
|
assert test_session.query(Download).one().id == download.id
|
|
|
|
|
assert download.collection_id == repo.collection_id
|
|
|
|
|
assert download.frequency == 3600
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
|
|
|
async def test_delete_last_repo(test_session, async_client, test_wrolpi_config, await_switches):
|
|
|
|
|
"""Deleting the last Repo empties the config, so it does not return on the next import."""
|
|
|
|
|
repo = lib.create_repository(test_session, 'https://github.com/a/b')
|
|
|
|
|
await await_switches()
|
|
|
|
|
config = lib.get_repos_config()
|
|
|
|
|
assert len(config.read_config_file()['repos']) == 1
|
|
|
|
|
|
|
|
|
|
request, response = await async_client.delete(f'/api/repos/{repo.id}')
|
|
|
|
|
assert config.read_config_file()['repos'] == []
|
|
|
|
|
|
|
|
|
|
config.import_config()
|
|
|
|
|
assert test_session.query(Repository).count() == 0
|