diff --git a/alembic/env.py b/alembic/env.py index a9575b1e..eb679467 100644 --- a/alembic/env.py +++ b/alembic/env.py @@ -24,6 +24,7 @@ from wrolpi.collections.models import Collection, CollectionItem # noqa: F401 from modules.videos.models import Video, Channel # noqa: F401 from modules.archive.models import Archive # noqa: F401 from modules.zim.models import Zim, TagZimEntry, ZimSubscription # noqa: F401 +from modules.repos.models import Repository # noqa: F401 target_metadata = Base.metadata diff --git a/alembic/versions/2026_10_06_1200_repository.py b/alembic/versions/2026_10_06_1200_repository.py new file mode 100644 index 00000000..e4537267 --- /dev/null +++ b/alembic/versions/2026_10_06_1200_repository.py @@ -0,0 +1,43 @@ +"""Add the repository table for git Repos (Collections with kind='repo'). + +Revision ID: 2026_10_06_1200 +Revises: 2026_10_05_1200 +""" +import sqlalchemy as sa +from alembic import op + +import wrolpi.dates + +# revision identifiers, used by Alembic. +revision = '2026_10_06_1200' +down_revision = '2026_10_05_1200' +branch_labels = None +depends_on = None + + +def upgrade(): + op.create_table( + 'repository', + sa.Column('id', sa.Integer(), nullable=False), + sa.Column('url', sa.String(), nullable=False), + sa.Column('host', sa.String(), nullable=True), + sa.Column('owner', sa.String(), nullable=True), + sa.Column('mode', sa.String(), nullable=False), + sa.Column('branch', sa.String(), nullable=True), + sa.Column('default_branch', sa.String(), nullable=True), + sa.Column('head_sha', sa.String(), nullable=True), + sa.Column('head_date', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('head_message', sa.Text(), nullable=True), + sa.Column('last_fetch', wrolpi.dates.TZDateTime(), nullable=True), + sa.Column('size', sa.BigInteger(), nullable=True), + sa.Column('readme_path', sa.String(), nullable=True), + sa.Column('collection_id', sa.Integer(), nullable=False), + sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'), + sa.PrimaryKeyConstraint('id'), + sa.UniqueConstraint('collection_id'), + sa.UniqueConstraint('url'), + ) + + +def downgrade(): + op.drop_table('repository') diff --git a/alembic/versions/2026_10_06_1300_repository_search.py b/alembic/versions/2026_10_06_1300_repository_search.py new file mode 100644 index 00000000..b4fd7abc --- /dev/null +++ b/alembic/versions/2026_10_06_1300_repository_search.py @@ -0,0 +1,39 @@ +"""Search git Repos by name and README: repository.search_name/readme_text and repository_fts. + +Revision ID: 2026_10_06_1300 +Revises: 2026_10_06_1200 +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_10_06_1300' +down_revision = '2026_10_06_1200' +branch_labels = None +depends_on = None + + +def upgrade(): + from wrolpi.schema_ddl import install_raw_ddl + + op.add_column('repository', sa.Column('search_name', sa.String(), nullable=True)) + op.add_column('repository', sa.Column('readme_text', sa.Text(), nullable=True)) + op.execute(''' + UPDATE repository SET search_name = TRIM( + COALESCE((SELECT name FROM collection WHERE collection.id = repository.collection_id), '') + || ' ' || COALESCE(owner, '')) + ''') + + # Creates repository_fts and its triggers now that the columns exist. The README text is filled by the next + # update of each Repo. + install_raw_ddl(op.get_bind()) + op.execute("INSERT INTO repository_fts(repository_fts) VALUES('rebuild')") + + +def downgrade(): + for name in ('repository_fts_ai', 'repository_fts_ad', 'repository_fts_au'): + op.execute(f'DROP TRIGGER IF EXISTS {name}') + op.execute('DROP TABLE IF EXISTS repository_fts') + with op.batch_alter_table('repository') as batch_op: + batch_op.drop_column('readme_text') + batch_op.drop_column('search_name') diff --git a/alembic/versions/2026_10_06_1400_repository_submodules.py b/alembic/versions/2026_10_06_1400_repository_submodules.py new file mode 100644 index 00000000..123ba97d --- /dev/null +++ b/alembic/versions/2026_10_06_1400_repository_submodules.py @@ -0,0 +1,22 @@ +"""A git Repo can also download its submodules: repository.submodules. + +Revision ID: 2026_10_06_1400 +Revises: 2026_10_06_1300 +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_10_06_1400' +down_revision = '2026_10_06_1300' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('repository', sa.Column('submodules', sa.Boolean(), server_default=sa.false(), nullable=False)) + + +def downgrade(): + with op.batch_alter_table('repository') as batch_op: + batch_op.drop_column('submodules') diff --git a/alembic/versions/2026_10_06_1500_repository_clone_token.py b/alembic/versions/2026_10_06_1500_repository_clone_token.py new file mode 100644 index 00000000..61d936af --- /dev/null +++ b/alembic/versions/2026_10_06_1500_repository_clone_token.py @@ -0,0 +1,22 @@ +"""A git Repo's clone holds a random token: repository.clone_token. + +Revision ID: 2026_10_06_1500 +Revises: 2026_10_06_1400 +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = '2026_10_06_1500' +down_revision = '2026_10_06_1400' +branch_labels = None +depends_on = None + + +def upgrade(): + op.add_column('repository', sa.Column('clone_token', sa.String(), nullable=True)) + + +def downgrade(): + with op.batch_alter_table('repository') as batch_op: + batch_op.drop_column('clone_token') diff --git a/app/cypress/e2e/settings/special-directories.cy.js b/app/cypress/e2e/settings/special-directories.cy.js index 0a9e248d..021df88d 100644 --- a/app/cypress/e2e/settings/special-directories.cy.js +++ b/app/cypress/e2e/settings/special-directories.cy.js @@ -1,5 +1,5 @@ /* - * The five Special Directories fields on /admin/settings. + * The six Special Directories fields on /admin/settings. * * These are the fields that shipped printing their own prefix over their own value, and the * ones whose visible labels were never associated with their inputs. Both faults were @@ -19,6 +19,7 @@ describe('Special Directories settings', () => { ['Map Directory', 'map'], ['Zims Directory', 'zims'], ['Playlists Directory', 'playlists'], + ['Repos Directory', 'repos/%(repo_tag)s/%(repo_name)s'], ]; beforeEach(() => { @@ -38,8 +39,8 @@ describe('Special Directories settings', () => { it('names every directory field, so they can be told apart', () => { /* - * All five fields sit in one grid, hold similar-looking relative paths, and share the - * same prefix. Without a name attached to the input, a screen reader user hears five + * All six fields sit in one grid, hold similar-looking relative paths, and share the + * same prefix. Without a name attached to the input, a screen reader user hears six * fields described only as "/media/wrolpi/" and cannot tell which one saves videos. */ directories.forEach(([label, value]) => { diff --git a/app/src/App.css b/app/src/App.css index 4c961e69..6b423219 100644 --- a/app/src/App.css +++ b/app/src/App.css @@ -326,6 +326,13 @@ pre.wrap-text { padding: 0 0.25em; } +/* A repo's README is part of its page, not a modal; it grows with its content. */ +.repo-readme .markdown-preview { + height: auto; + min-height: 0; + overflow: visible; +} + .markdown-raw { margin: 0; white-space: pre-wrap; diff --git a/app/src/App.js b/app/src/App.js index 25881b13..c3299d98 100644 --- a/app/src/App.js +++ b/app/src/App.js @@ -30,6 +30,7 @@ import {FilePreviewProvider} from "./components/FilePreview"; import {TagsProvider} from "./Tags"; import {ZimRoute} from "./components/Zim"; import {PlaylistsRoute} from "./components/Playlists"; +import {ReposRoute} from "./components/Repos"; import {DocsRoute} from "./components/Docs"; import {FlasherRoute} from "./components/Flasher"; import ErrorBoundary from "./components/ErrorBoundary"; @@ -117,6 +118,7 @@ const router = createBrowserRouter(createRoutesFromElements(}/> }/> }/> + }/> }/> }/> )); diff --git a/app/src/api.js b/app/src/api.js index 907d6012..142f7ec3 100644 --- a/app/src/api.js +++ b/app/src/api.js @@ -7,6 +7,7 @@ import { COLLECTIONS_API, DEFAULT_LIMIT, Downloaders, + REPOS_API, VIDEOS_API, ZIM_API } from "./components/Vars"; @@ -406,6 +407,7 @@ export async function getSettings() { nav_color: content.nav_color, media_directory: content.media_directory, playlists_destination: content.playlists_destination, + repos_destination: content.repos_destination, require_cookies_unlocked: content.require_cookies_unlocked, require_media_mounted: content.require_media_mounted, save_ffprobe_json: content.save_ffprobe_json, @@ -927,6 +929,21 @@ export async function getCollectionTagInfo(collectionId, tagName) { } } +// Tag any Collection (and optionally move its directory). +export async function tagCollection(collectionId, tagName, directory) { + const body = {tag_name: tagName}; + if (directory) { + body['directory'] = directory; + } + const response = await apiPost(`${COLLECTIONS_API}/${collectionId}/tag`, body); + if (response.ok) { + return await response.json(); + } + const message = await getErrorMessage(response, 'Unable to tag. See server logs.'); + toast({type: 'error', title: 'Tag Error', description: message, time: 5000}); + throw new Error(message); +} + export async function tagDomain(domainId, tagName, directory) { const body = { tag_name: tagName, @@ -2051,6 +2068,7 @@ export async function searchSuggestions(search_str) { domains: content.domains, authors: content.authors || [], subjects: content.subjects || [], + repos: content.repos || [], } } else { console.error('Failed to get file search suggestions!'); @@ -2089,8 +2107,8 @@ export async function searchEstimateZims(search_str, tagNames) { } } -export async function searchEstimateOthers(tagNames) { - const body = {tag_names: tagNames}; +export async function searchEstimateOthers(tagNames, searchStr) { + const body = {tag_names: tagNames, search_str: searchStr || null}; const response = await apiPost(`${API_URI}/search_other_estimates`, body); if (response.ok) { const content = await response.json(); @@ -2488,3 +2506,112 @@ export async function moveBookmark(nodeId, parent_id, position = null) { export async function deleteBookmark(nodeId) { return await bookmarksModify(apiDelete(`${API_URI}/bookmarks/${nodeId}`), 'Could not delete bookmark'); } + +async function reposError(response, fallbackMessage) { + const message = await getErrorMessage(response, fallbackMessage); + toast({type: 'error', title: 'Repos Error', description: message, time: 5000}); + throw Error(message); +} + +export async function fetchRepos() { + const response = await apiGet(REPOS_API); + if (response.ok) { + return (await response.json())['repos']; + } + await reposError(response, 'Failed to fetch repos.'); +} + +export async function getRepo(repoId) { + const response = await apiGet(`${REPOS_API}/${repoId}`); + if (response.ok) { + return (await response.json())['repo']; + } + await reposError(response, 'Failed to fetch repo.'); +} + +// `repo`: {url, directory?, name?, tag_name?, frequency?, mode?, branch?, description?} +export async function createRepo(repo) { + const response = await apiPost(REPOS_API, repo); + if (response.ok) { + return (await response.json())['repo']; + } + await reposError(response, 'Failed to add repo.'); +} + +// `changes`: {description?, frequency?, mode?, branch?}. An empty branch follows the remote's default branch. +export async function updateRepo(repoId, changes) { + const response = await apiPut(`${REPOS_API}/${repoId}`, changes); + if (response.ok) { + return (await response.json())['repo']; + } + await reposError(response, 'Failed to update repo.'); +} + +export async function deleteRepo(repoId, deleteFiles = false) { + const response = await apiDelete(`${REPOS_API}/${repoId}?delete_files=${deleteFiles ? 'true' : 'false'}`); + if (!response.ok) { + await reposError(response, 'Failed to delete repo.'); + } +} + +// Download the repo's updates now. +export async function updateRepoNow(repoId) { + const response = await apiPost(`${REPOS_API}/${repoId}/update`); + if (!response.ok) { + await reposError(response, 'Failed to start the repo update.'); + } +} + +// One directory of a repo's files: {path, entries: [{name, path, is_dir, size}], readme_path}. +export async function getRepoTree(repoId, path) { + const query = path ? `?path=${encodeURIComponent(path)}` : ''; + const response = await apiGet(`${REPOS_API}/${repoId}/tree${query}`); + if (response.ok) { + return await response.json(); + } + await reposError(response, 'Failed to list the repo files.'); +} + +// Repos matching the search (name and README) and/or the tags. Each has an escaped `readme_headline`. +export async function searchRepos(searchStr, tagNames) { + const body = {search_str: searchStr || null, tag_names: tagNames || []}; + const response = await apiPost(`${REPOS_API}/search`, body); + if (response.ok) { + const content = await response.json(); + return {repos: content.repos, total: content.totals.repos}; + } + await reposError(response, 'Failed to search repos.'); +} + +// The commits of a repo's checked out branch, newest first: {commits: [{sha, author, date, message}], total}. +export async function getRepoLog(repoId, offset = 0, limit = 50) { + const response = await apiGet(`${REPOS_API}/${repoId}/log?limit=${limit}&offset=${offset}`); + if (response.ok) { + return await response.json(); + } + await reposError(response, 'Failed to fetch the repo history.'); +} + +// A ZIP of a repo's checked out files; the browser downloads it. +export function repoArchiveURL(repoId) { + return `${REPOS_API}/${repoId}/archive.zip`; +} + +// What importing the git clone in `directory` (relative to the media directory) would do. Changes nothing. +export async function inspectRepoImport(directory, name, tagName, url) { + const body = {directory, name: name || null, tag_name: tagName || null, url: url || null}; + const response = await apiPost(`${REPOS_API}/import/inspect`, body); + if (response.ok) { + return {inspection: (await response.json())['inspection']}; + } + return {error: await getErrorMessage(response, 'Cannot import this directory.')}; +} + +// Import an existing git clone as a Repo; it stays where it is. `repo`: {directory, confirm, url?, name?, ...} +export async function importRepo(repo) { + const response = await apiPost(`${REPOS_API}/import`, repo); + if (response.ok) { + return (await response.json())['repo']; + } + await reposError(response, 'Failed to import the repo.'); +} diff --git a/app/src/components/Common.test.js b/app/src/components/Common.test.js index 408b7804..14559845 100644 --- a/app/src/components/Common.test.js +++ b/app/src/components/Common.test.js @@ -1121,7 +1121,7 @@ describe('a row of buttons', () => { * the two had been inline before. * * A row fixes it without touching the stack: the row is the stretched child, and it lays - * its buttons out at their own width. Nine sites, so this scan rather than nine + * its buttons out at their own width. Thirteen sites, so this scan rather than thirteen * assertions -- and a scan because a stretched button is still a working button. */ const offenders = []; @@ -1140,8 +1140,8 @@ describe('a row of buttons', () => { } } - // Nine renders. Common.js declares it, and a declaration is not a ` ({ ...jest.requireActual('../api'), searchChannels: jest.fn(), + searchRepos: jest.fn(), })); const NO_CHANNELS = /No Channels/; -beforeEach(() => searchChannels.mockReset()); +beforeEach(() => { + searchChannels.mockReset(); + searchRepos.mockReset(); + searchRepos.mockResolvedValue({repos: [], total: 0}); +}); test('pending: shows a loader, not "No Channels"', () => { searchChannels.mockReturnValue(new Promise(() => { @@ -54,3 +59,61 @@ test('loaded: lists the channels', async () => { expect(screen.getByText('Cooking')).toBeInTheDocument(); expect(screen.queryByText(NO_CHANNELS)).not.toBeInTheDocument(); }); + +describe('Repos', () => { + test('pending: the Repos section loads while Channels are shown', async () => { + searchChannels.mockResolvedValue({channels: []}); + searchRepos.mockReturnValue(new Promise(() => { + })); + const {container} = render(); + await act(async () => { + }); + expect(screen.getByText(NO_CHANNELS)).toBeInTheDocument(); + expect(container.querySelector('.wrolpi-loading')).toBeInTheDocument(); + expect(screen.queryByText('No Repos')).not.toBeInTheDocument(); + }); + + test('empty: shows "No Repos"', async () => { + searchChannels.mockResolvedValue({channels: []}); + render(); + await act(async () => { + }); + expect(screen.getByText('No Repos')).toBeInTheDocument(); + }); + + test('failed: shows an error only in the Repos section', async () => { + searchChannels.mockResolvedValue({channels: [{id: 1, name: 'Cooking'}]}); + searchRepos.mockRejectedValue(new Error('boom')); + const spy = jest.spyOn(console, 'error').mockImplementation(() => { + }); + try { + render(); + await act(async () => { + }); + expect(screen.getByText('Cooking')).toBeInTheDocument(); + expect(screen.getByText(/Could not fetch the repos/)).toBeInTheDocument(); + } finally { + spy.mockRestore(); + } + }); + + test('loaded: lists repos by the search, with their README headline', async () => { + searchChannels.mockResolvedValue({channels: []}); + searchRepos.mockResolvedValue({ + total: 1, repos: [{ + id: 7, name: 'kiwix-tools', tag_name: null, + // The server escapes the README; only the highlight is markup. + readme_headline: 'Serve Zim files <script>', + }], + }); + const searchParams = new URLSearchParams('q=zim&tag=Software'); + const {container} = render(, {query: {searchParams}}); + await act(async () => { + }); + expect(searchRepos).toHaveBeenCalledWith('zim', ['Software']); + expect(screen.getByText('kiwix-tools').closest('a')).toHaveAttribute('href', '/repos/7'); + expect(container.querySelector('u').textContent).toBe('Zim'); + expect(container.querySelector('script')).toBeNull(); + expect(screen.getByText(/files