Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
from http import HTTPStatus
|
|
|
|
|
|
|
|
|
|
from sanic import Request, Blueprint, response
|
|
|
|
|
from sanic_ext.extensions.openapi import openapi
|
|
|
|
|
|
|
|
|
|
from wrolpi.api_utils import json_response
|
|
|
|
|
from wrolpi.common import wrol_mode_check, logger
|
|
|
|
|
from wrolpi.downloader import download_manager
|
|
|
|
|
from wrolpi.errors import ValidationError
|
|
|
|
|
from . import lib, schema
|
|
|
|
|
from .models import Repository
|
|
|
|
|
|
|
|
|
|
repos_bp = Blueprint('Repos', '/api/repos')
|
|
|
|
|
|
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.get('/')
|
|
|
|
|
@openapi.definition(summary='List all git Repos')
|
|
|
|
|
async def get_repos(request: Request):
|
|
|
|
|
repos = lib.get_repositories(request.ctx.session)
|
|
|
|
|
return json_response(dict(repos=repos))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.post('/')
|
|
|
|
|
@openapi.definition(
|
|
|
|
|
summary='Add a git Repo; it will be cloned, then kept up to date',
|
|
|
|
|
body=schema.RepoCreateRequest,
|
|
|
|
|
validate=True,
|
|
|
|
|
)
|
|
|
|
|
@wrol_mode_check
|
|
|
|
|
async def post_repo(request: Request, body: schema.RepoCreateRequest):
|
|
|
|
|
session = request.ctx.session
|
|
|
|
|
repo = lib.create_repository(
|
|
|
|
|
session,
|
|
|
|
|
body.url,
|
|
|
|
|
tag_name=body.tag_name or None,
|
|
|
|
|
frequency=body.frequency or lib.DEFAULT_REPO_FREQUENCY,
|
|
|
|
|
mode=body.mode or 'full',
|
|
|
|
|
branch=body.branch or None,
|
|
|
|
|
name=body.name or None,
|
|
|
|
|
description=body.description or None,
|
2026-10-06 16:27:38 -06:00
|
|
|
submodules=body.submodules,
|
Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
)
|
|
|
|
|
repo = Repository.find_by_id(session, repo.id, options=Repository.json_options())
|
|
|
|
|
return json_response(dict(repo=repo.__json__()), status=HTTPStatus.CREATED)
|
|
|
|
|
|
|
|
|
|
|
repos: import an existing git clone as a Repo
POST /api/repos/import/inspect reports what importing a clone in the
media directory would do, without changing it: its origin (an SSH or
git:// origin is converted to https://), branch, latest commit, commits
not on the remote, whether it is shallow, and where it will be moved.
It refuses special directories, Collections' directories, `.git`
files/links, bare repos, and partial clones. It runs no git command
which could run a program from the clone's config (the latest commit is
read with cat-file, not log).
POST /api/repos/import (confirmed, with a URL that is the clone's
origin) moves the clone into the Repos directory (the old location is
refreshed away), sanitizes its config, points origin at the URL, marks
it as WROLPi's, and creates the Repo and its Download. The Repo shows
the clone's commit and README at once; its first update makes it a
mirror (local work is discarded, and the page warns of local commits).
The Repos page has an Import button; the Import page inspects the
chosen directory, then imports.
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 18:09:52 -06:00
|
|
|
@repos_bp.post('/import/inspect')
|
|
|
|
|
@openapi.definition(
|
|
|
|
|
summary='What importing an existing git clone (in the media directory) as a Repo would do. Changes nothing.',
|
|
|
|
|
body=schema.RepoImportInspectRequest,
|
|
|
|
|
validate=True,
|
|
|
|
|
)
|
|
|
|
|
async def post_inspect_import(request: Request, body: schema.RepoImportInspectRequest):
|
|
|
|
|
info = lib.inspect_import(request.ctx.session, body.directory, body.name or None, body.tag_name or None)
|
|
|
|
|
return json_response(dict(inspection=info))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.post('/import')
|
|
|
|
|
@openapi.definition(
|
|
|
|
|
summary='Import an existing git clone as a Repo. It is moved into the Repos directory, and becomes a mirror of '
|
|
|
|
|
'its origin (local changes are discarded by its updates).',
|
|
|
|
|
body=schema.RepoImportRequest,
|
|
|
|
|
validate=True,
|
|
|
|
|
)
|
|
|
|
|
@wrol_mode_check
|
|
|
|
|
async def post_import(request: Request, body: schema.RepoImportRequest):
|
|
|
|
|
session = request.ctx.session
|
|
|
|
|
repo = await lib.import_repository(
|
|
|
|
|
session,
|
|
|
|
|
body.directory,
|
|
|
|
|
url=body.url or None,
|
|
|
|
|
confirm=body.confirm,
|
|
|
|
|
tag_name=body.tag_name or None,
|
|
|
|
|
frequency=body.frequency or lib.DEFAULT_REPO_FREQUENCY,
|
|
|
|
|
mode=body.mode or 'full',
|
|
|
|
|
branch=body.branch or None,
|
|
|
|
|
submodules=body.submodules,
|
|
|
|
|
name=body.name or None,
|
|
|
|
|
description=body.description or None,
|
|
|
|
|
)
|
|
|
|
|
repo = Repository.find_by_id(session, repo.id, options=Repository.json_options())
|
|
|
|
|
return json_response(dict(repo=repo.__json__()), status=HTTPStatus.CREATED)
|
|
|
|
|
|
|
|
|
|
|
2026-10-06 11:02:13 -06:00
|
|
|
@repos_bp.post('/search')
|
|
|
|
|
@openapi.definition(
|
|
|
|
|
summary='Search git Repos by their name and README, and/or by Tag',
|
|
|
|
|
body=schema.RepoSearchRequest,
|
|
|
|
|
validate=True,
|
|
|
|
|
)
|
|
|
|
|
async def post_search_repos(request: Request, body: schema.RepoSearchRequest):
|
|
|
|
|
limit = min(max(body.limit, 1), 100)
|
|
|
|
|
repos, total = lib.search_repos(request.ctx.session, body.search_str, body.tag_names, limit=limit,
|
|
|
|
|
offset=max(body.offset, 0))
|
|
|
|
|
return json_response(dict(repos=repos, totals=dict(repos=total)))
|
|
|
|
|
|
|
|
|
|
|
Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
@repos_bp.get('/<repo_id:int>')
|
|
|
|
|
@openapi.definition(summary='Get a git Repo')
|
|
|
|
|
async def get_repo(request: Request, repo_id: int):
|
|
|
|
|
repo = Repository.find_by_id(request.ctx.session, repo_id, options=Repository.json_options())
|
|
|
|
|
return json_response(dict(repo=repo.__json__()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.put('/<repo_id:int>')
|
|
|
|
|
@openapi.definition(
|
|
|
|
|
summary="Change a git Repo's settings",
|
|
|
|
|
body=schema.RepoUpdateRequest,
|
|
|
|
|
validate=True,
|
|
|
|
|
)
|
|
|
|
|
@wrol_mode_check
|
|
|
|
|
async def put_repo(request: Request, repo_id: int, body: schema.RepoUpdateRequest):
|
|
|
|
|
session = request.ctx.session
|
|
|
|
|
lib.update_repository(session, repo_id, description=body.description, frequency=body.frequency,
|
2026-10-06 16:27:38 -06:00
|
|
|
mode=body.mode, branch=body.branch, submodules=body.submodules)
|
Add a Repos module that mirrors git repositories (backend)
A Repo is a Collection (kind='repo') plus a `repository` row, kept up to
date by a recurring Download handled by the new `git` downloader.
* Full history by default (all branches and tags, no --prune; a force
push saves the old HEAD under refs/wrolpi/backup/); snapshot mode
(--depth 1) and a pinned branch are optional. `/tree/<branch>` URLs
select a branch.
* git runs HTTPS-only, never prompts, never runs hooks, and checks out
symlinks as plain files. Upstream disappearing defers the download
with git's error; the local copy is kept.
* Repos are cloned to `repos_destination` (repos/%(repo_tag)s/%(repo_name)s)
and the `repos` directory is always ignored, so repo files are never
indexed.
* repos.yaml is the source of truth (import, dump, backup restore).
* /api/repos: list, create, get, edit, delete (files kept unless
delete_files), update now, and a per-directory file tree. Tagging uses
the collections tag endpoint.
Also:
* run_command/process_runner can merge stderr into stdout (git writes
progress to stderr).
* move_collection deletes an old directory that only holds empty
directories (the move left an emptied .git behind).
Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-06 10:52:10 -06:00
|
|
|
repo = Repository.find_by_id(session, repo_id, options=Repository.json_options())
|
|
|
|
|
return json_response(dict(repo=repo.__json__()))
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.delete('/<repo_id:int>')
|
|
|
|
|
@openapi.definition(summary='Delete a git Repo. Its files are kept unless `delete_files=true`.')
|
|
|
|
|
@wrol_mode_check
|
|
|
|
|
async def delete_repo(request: Request, repo_id: int):
|
|
|
|
|
delete_files = request.args.get('delete_files', '').lower() in ('1', 'true', 'yes')
|
|
|
|
|
await lib.delete_repository(request.ctx.session, repo_id, delete_files=delete_files)
|
|
|
|
|
return response.empty()
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.post('/<repo_id:int>/update')
|
|
|
|
|
@openapi.definition(summary='Update a git Repo now')
|
|
|
|
|
@wrol_mode_check
|
|
|
|
|
async def post_update_repo(request: Request, repo_id: int):
|
|
|
|
|
session = request.ctx.session
|
|
|
|
|
repo = Repository.find_by_id(session, repo_id, options=Repository.json_options())
|
|
|
|
|
if not (download := repo.download):
|
|
|
|
|
raise ValidationError('Repo has no download')
|
|
|
|
|
download_manager.restart_download(session, download.id)
|
|
|
|
|
return response.empty(HTTPStatus.NO_CONTENT)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.get('/<repo_id:int>/tree')
|
|
|
|
|
@openapi.definition(summary="List a directory of a git Repo's files")
|
|
|
|
|
@openapi.parameter('path', str, 'query', description='Directory in the repo, relative to its root', required=False)
|
|
|
|
|
async def get_repo_tree(request: Request, repo_id: int):
|
|
|
|
|
repo = Repository.find_by_id(request.ctx.session, repo_id)
|
|
|
|
|
tree = lib.list_repo_tree(repo, request.args.get('path', ''))
|
|
|
|
|
return json_response(tree)
|
2026-10-06 16:27:38 -06:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.get('/<repo_id:int>/log')
|
|
|
|
|
@openapi.definition(summary="The commits of a git Repo's checked out branch, newest first")
|
|
|
|
|
@openapi.parameter('limit', int, 'query', description='At most 100', required=False)
|
|
|
|
|
@openapi.parameter('offset', int, 'query', required=False)
|
|
|
|
|
async def get_repo_log(request: Request, repo_id: int):
|
|
|
|
|
repo = Repository.find_by_id(request.ctx.session, repo_id)
|
|
|
|
|
try:
|
|
|
|
|
limit = int(request.args.get('limit', 50))
|
|
|
|
|
offset = int(request.args.get('offset', 0))
|
|
|
|
|
except ValueError:
|
|
|
|
|
raise ValidationError('limit and offset must be numbers')
|
|
|
|
|
log = await lib.get_repo_log(repo, limit=limit, offset=offset)
|
|
|
|
|
return json_response(log)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
@repos_bp.get('/<repo_id:int>/archive.zip')
|
|
|
|
|
@openapi.definition(summary="Download a ZIP of a git Repo's checked out files")
|
|
|
|
|
async def get_repo_archive(request: Request, repo_id: int):
|
|
|
|
|
repo = Repository.find_by_id(request.ctx.session, repo_id)
|
|
|
|
|
# Read everything from the database before the response starts; the request's session closes then.
|
|
|
|
|
directory = lib.get_repo_root(repo)
|
|
|
|
|
name = lib.repo_archive_name(repo)
|
|
|
|
|
response = await request.respond(content_type='application/zip', headers={
|
|
|
|
|
'Content-Disposition': f'attachment; filename="{name}.zip"',
|
|
|
|
|
})
|
|
|
|
|
await lib.stream_repo_archive(directory, name, response.send)
|
|
|
|
|
await response.eof()
|