Add a Repos module that mirrors git repositories (#783)

Repos keeps local mirrors of git repositories, cloned over https and kept up to date on a schedule.

- A Repo is a Collection (kind='repo') plus a `repository` row.  The new `git` downloader clones it, then updates it
  (weekly by default, or Never: the clone is kept as it is and only updated on request).
- Repos are saved in `repos/<tag>/<name>` (the Repos Directory, ignored by default) or a directory the user chooses.
  An existing clone can be imported where it is.  repos.yaml is the source of truth.
- A GitHub-like page shows each repo's README, files, and history, with a ZIP download.  READMEs are searchable
  under the global search's Other tab.
- git is hardened against hostile repos and configs: https only, pinned addresses with no private addresses (for the
  repo and its submodules), sanitized configs, no hooks or links, and only clones WROLPi made or imported are
  updated.
- Tagging a Repo moves its clone as one directory.  Repo changes wait for repos.yaml to be imported at startup.
- The file mover groups unindexed files with no primary file as one FileGroup, and never replaces a file at the
  destination.  A failed tag move restores the Collection's tag.  A Collection's directory cannot leave the media
  directory.

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>

alembic/env.py

@@ -24,6 +24,7 @@ from wrolpi.collections.models import Collection, CollectionItem  # noqa: F401
 from modules.videos.models import Video, Channel  # noqa: F401
 from modules.archive.models import Archive  # noqa: F401
 from modules.zim.models import Zim, TagZimEntry, ZimSubscription  # noqa: F401
+from modules.repos.models import Repository  # noqa: F401
 
 target_metadata = Base.metadata
 

alembic/versions/2026_10_06_1200_repository.py

@@ -0,0 +1,43 @@
+"""Add the repository table for git Repos (Collections with kind='repo').
+
+Revision ID: 2026_10_06_1200
+Revises: 2026_10_05_1200
+"""
+import sqlalchemy as sa
+from alembic import op
+
+import wrolpi.dates
+
+# revision identifiers, used by Alembic.
+revision = '2026_10_06_1200'
+down_revision = '2026_10_05_1200'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table(
+        'repository',
+        sa.Column('id', sa.Integer(), nullable=False),
+        sa.Column('url', sa.String(), nullable=False),
+        sa.Column('host', sa.String(), nullable=True),
+        sa.Column('owner', sa.String(), nullable=True),
+        sa.Column('mode', sa.String(), nullable=False),
+        sa.Column('branch', sa.String(), nullable=True),
+        sa.Column('default_branch', sa.String(), nullable=True),
+        sa.Column('head_sha', sa.String(), nullable=True),
+        sa.Column('head_date', wrolpi.dates.TZDateTime(), nullable=True),
+        sa.Column('head_message', sa.Text(), nullable=True),
+        sa.Column('last_fetch', wrolpi.dates.TZDateTime(), nullable=True),
+        sa.Column('size', sa.BigInteger(), nullable=True),
+        sa.Column('readme_path', sa.String(), nullable=True),
+        sa.Column('collection_id', sa.Integer(), nullable=False),
+        sa.ForeignKeyConstraint(['collection_id'], ['collection.id'], ondelete='CASCADE'),
+        sa.PrimaryKeyConstraint('id'),
+        sa.UniqueConstraint('collection_id'),
+        sa.UniqueConstraint('url'),
+    )
+
+
+def downgrade():
+    op.drop_table('repository')

alembic/versions/2026_10_06_1300_repository_search.py

@@ -0,0 +1,39 @@
+"""Search git Repos by name and README: repository.search_name/readme_text and repository_fts.
+
+Revision ID: 2026_10_06_1300
+Revises: 2026_10_06_1200
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '2026_10_06_1300'
+down_revision = '2026_10_06_1200'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    from wrolpi.schema_ddl import install_raw_ddl
+
+    op.add_column('repository', sa.Column('search_name', sa.String(), nullable=True))
+    op.add_column('repository', sa.Column('readme_text', sa.Text(), nullable=True))
+    op.execute('''
+        UPDATE repository SET search_name = TRIM(
+            COALESCE((SELECT name FROM collection WHERE collection.id = repository.collection_id), '')
+            || ' ' || COALESCE(owner, ''))
+    ''')
+
+    # Creates repository_fts and its triggers now that the columns exist.  The README text is filled by the next
+    # update of each Repo.
+    install_raw_ddl(op.get_bind())
+    op.execute("INSERT INTO repository_fts(repository_fts) VALUES('rebuild')")
+
+
+def downgrade():
+    for name in ('repository_fts_ai', 'repository_fts_ad', 'repository_fts_au'):
+        op.execute(f'DROP TRIGGER IF EXISTS {name}')
+    op.execute('DROP TABLE IF EXISTS repository_fts')
+    with op.batch_alter_table('repository') as batch_op:
+        batch_op.drop_column('readme_text')
+        batch_op.drop_column('search_name')

alembic/versions/2026_10_06_1400_repository_submodules.py

@@ -0,0 +1,22 @@
+"""A git Repo can also download its submodules: repository.submodules.
+
+Revision ID: 2026_10_06_1400
+Revises: 2026_10_06_1300
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '2026_10_06_1400'
+down_revision = '2026_10_06_1300'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('repository', sa.Column('submodules', sa.Boolean(), server_default=sa.false(), nullable=False))
+
+
+def downgrade():
+    with op.batch_alter_table('repository') as batch_op:
+        batch_op.drop_column('submodules')

alembic/versions/2026_10_06_1500_repository_clone_token.py

@@ -0,0 +1,22 @@
+"""A git Repo's clone holds a random token: repository.clone_token.
+
+Revision ID: 2026_10_06_1500
+Revises: 2026_10_06_1400
+"""
+import sqlalchemy as sa
+from alembic import op
+
+# revision identifiers, used by Alembic.
+revision = '2026_10_06_1500'
+down_revision = '2026_10_06_1400'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column('repository', sa.Column('clone_token', sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table('repository') as batch_op:
+        batch_op.drop_column('clone_token')

app/cypress/e2e/settings/special-directories.cy.js

@@ -1,5 +1,5 @@
 /*
- * The five Special Directories fields on /admin/settings.
+ * The six Special Directories fields on /admin/settings.
  *
  * These are the fields that shipped printing their own prefix over their own value, and the
  * ones whose visible labels were never associated with their inputs.  Both faults were
@@ -19,6 +19,7 @@ describe('Special Directories settings', () => {
         ['Map Directory', 'map'],
         ['Zims Directory', 'zims'],
         ['Playlists Directory', 'playlists'],
+        ['Repos Directory', 'repos/%(repo_tag)s/%(repo_name)s'],
     ];
 
     beforeEach(() => {
@@ -38,8 +39,8 @@ describe('Special Directories settings', () => {
 
     it('names every directory field, so they can be told apart', () => {
         /*
-         * All five fields sit in one grid, hold similar-looking relative paths, and share the
-         * same prefix.  Without a name attached to the input, a screen reader user hears five
+         * All six fields sit in one grid, hold similar-looking relative paths, and share the
+         * same prefix.  Without a name attached to the input, a screen reader user hears six
          * fields described only as "/media/wrolpi/" and cannot tell which one saves videos.
          */
         directories.forEach(([label, value]) => {

app/src/App.css

@@ -326,6 +326,13 @@ pre.wrap-text {
     padding: 0 0.25em;
 }
 
+/* A repo's README is part of its page, not a modal; it grows with its content. */
+.repo-readme .markdown-preview {
+    height: auto;
+    min-height: 0;
+    overflow: visible;
+}
+
 .markdown-raw {
     margin: 0;
     white-space: pre-wrap;

app/src/App.js

@@ -30,6 +30,7 @@ import {FilePreviewProvider} from "./components/FilePreview";
 import {TagsProvider} from "./Tags";
 import {ZimRoute} from "./components/Zim";
 import {PlaylistsRoute} from "./components/Playlists";
+import {ReposRoute} from "./components/Repos";
 import {DocsRoute} from "./components/Docs";
 import {FlasherRoute} from "./components/Flasher";
 import ErrorBoundary from "./components/ErrorBoundary";
@@ -117,6 +118,7 @@ const router = createBrowserRouter(createRoutesFromElements(<Route
     <Route path='map/*' element={<ErrorBoundary><MapRoute/></ErrorBoundary>}/>
     <Route path='zim/*' element={<ErrorBoundary><ZimRoute/></ErrorBoundary>}/>
     <Route path='playlists/*' element={<ErrorBoundary><PlaylistsRoute/></ErrorBoundary>}/>
+    <Route path='repos/*' element={<ErrorBoundary><ReposRoute/></ErrorBoundary>}/>
     <Route path='files/*' element={<ErrorBoundary><FilesRoute/></ErrorBoundary>}/>
     <Route path='flasher/*' element={<ErrorBoundary><FlasherRoute/></ErrorBoundary>}/>
 </Route>));

app/src/api.js

@@ -7,6 +7,7 @@ import {
     COLLECTIONS_API,
     DEFAULT_LIMIT,
     Downloaders,
+    REPOS_API,
     VIDEOS_API,
     ZIM_API
 } from "./components/Vars";
@@ -406,6 +407,7 @@ export async function getSettings() {
             nav_color: content.nav_color,
             media_directory: content.media_directory,
             playlists_destination: content.playlists_destination,
+            repos_destination: content.repos_destination,
             require_cookies_unlocked: content.require_cookies_unlocked,
             require_media_mounted: content.require_media_mounted,
             save_ffprobe_json: content.save_ffprobe_json,
@@ -927,6 +929,21 @@ export async function getCollectionTagInfo(collectionId, tagName) {
     }
 }
 
+// Tag any Collection (and optionally move its directory).
+export async function tagCollection(collectionId, tagName, directory) {
+    const body = {tag_name: tagName};
+    if (directory) {
+        body['directory'] = directory;
+    }
+    const response = await apiPost(`${COLLECTIONS_API}/${collectionId}/tag`, body);
+    if (response.ok) {
+        return await response.json();
+    }
+    const message = await getErrorMessage(response, 'Unable to tag.  See server logs.');
+    toast({type: 'error', title: 'Tag Error', description: message, time: 5000});
+    throw new Error(message);
+}
+
 export async function tagDomain(domainId, tagName, directory) {
     const body = {
         tag_name: tagName,
@@ -2051,6 +2068,7 @@ export async function searchSuggestions(search_str) {
             domains: content.domains,
             authors: content.authors || [],
             subjects: content.subjects || [],
+            repos: content.repos || [],
         }
     } else {
         console.error('Failed to get file search suggestions!');
@@ -2089,8 +2107,8 @@ export async function searchEstimateZims(search_str, tagNames) {
     }
 }
 
-export async function searchEstimateOthers(tagNames) {
-    const body = {tag_names: tagNames};
+export async function searchEstimateOthers(tagNames, searchStr) {
+    const body = {tag_names: tagNames, search_str: searchStr || null};
     const response = await apiPost(`${API_URI}/search_other_estimates`, body);
     if (response.ok) {
         const content = await response.json();
@@ -2488,3 +2506,112 @@ export async function moveBookmark(nodeId, parent_id, position = null) {
 export async function deleteBookmark(nodeId) {
     return await bookmarksModify(apiDelete(`${API_URI}/bookmarks/${nodeId}`), 'Could not delete bookmark');
 }
+
+async function reposError(response, fallbackMessage) {
+    const message = await getErrorMessage(response, fallbackMessage);
+    toast({type: 'error', title: 'Repos Error', description: message, time: 5000});
+    throw Error(message);
+}
+
+export async function fetchRepos() {
+    const response = await apiGet(REPOS_API);
+    if (response.ok) {
+        return (await response.json())['repos'];
+    }
+    await reposError(response, 'Failed to fetch repos.');
+}
+
+export async function getRepo(repoId) {
+    const response = await apiGet(`${REPOS_API}/${repoId}`);
+    if (response.ok) {
+        return (await response.json())['repo'];
+    }
+    await reposError(response, 'Failed to fetch repo.');
+}
+
+// `repo`: {url, directory?, name?, tag_name?, frequency?, mode?, branch?, description?}
+export async function createRepo(repo) {
+    const response = await apiPost(REPOS_API, repo);
+    if (response.ok) {
+        return (await response.json())['repo'];
+    }
+    await reposError(response, 'Failed to add repo.');
+}
+
+// `changes`: {description?, frequency?, mode?, branch?}.  An empty branch follows the remote's default branch.
+export async function updateRepo(repoId, changes) {
+    const response = await apiPut(`${REPOS_API}/${repoId}`, changes);
+    if (response.ok) {
+        return (await response.json())['repo'];
+    }
+    await reposError(response, 'Failed to update repo.');
+}
+
+export async function deleteRepo(repoId, deleteFiles = false) {
+    const response = await apiDelete(`${REPOS_API}/${repoId}?delete_files=${deleteFiles ? 'true' : 'false'}`);
+    if (!response.ok) {
+        await reposError(response, 'Failed to delete repo.');
+    }
+}
+
+// Download the repo's updates now.
+export async function updateRepoNow(repoId) {
+    const response = await apiPost(`${REPOS_API}/${repoId}/update`);
+    if (!response.ok) {
+        await reposError(response, 'Failed to start the repo update.');
+    }
+}
+
+// One directory of a repo's files: {path, entries: [{name, path, is_dir, size}], readme_path}.
+export async function getRepoTree(repoId, path) {
+    const query = path ? `?path=${encodeURIComponent(path)}` : '';
+    const response = await apiGet(`${REPOS_API}/${repoId}/tree${query}`);
+    if (response.ok) {
+        return await response.json();
+    }
+    await reposError(response, 'Failed to list the repo files.');
+}
+
+// Repos matching the search (name and README) and/or the tags.  Each has an escaped `readme_headline`.
+export async function searchRepos(searchStr, tagNames) {
+    const body = {search_str: searchStr || null, tag_names: tagNames || []};
+    const response = await apiPost(`${REPOS_API}/search`, body);
+    if (response.ok) {
+        const content = await response.json();
+        return {repos: content.repos, total: content.totals.repos};
+    }
+    await reposError(response, 'Failed to search repos.');
+}
+
+// The commits of a repo's checked out branch, newest first: {commits: [{sha, author, date, message}], total}.
+export async function getRepoLog(repoId, offset = 0, limit = 50) {
+    const response = await apiGet(`${REPOS_API}/${repoId}/log?limit=${limit}&offset=${offset}`);
+    if (response.ok) {
+        return await response.json();
+    }
+    await reposError(response, 'Failed to fetch the repo history.');
+}
+
+// A ZIP of a repo's checked out files; the browser downloads it.
+export function repoArchiveURL(repoId) {
+    return `${REPOS_API}/${repoId}/archive.zip`;
+}
+
+// What importing the git clone in `directory` (relative to the media directory) would do.  Changes nothing.
+export async function inspectRepoImport(directory, name, tagName, url) {
+    const body = {directory, name: name || null, tag_name: tagName || null, url: url || null};
+    const response = await apiPost(`${REPOS_API}/import/inspect`, body);
+    if (response.ok) {
+        return {inspection: (await response.json())['inspection']};
+    }
+    return {error: await getErrorMessage(response, 'Cannot import this directory.')};
+}
+
+// Import an existing git clone as a Repo; it stays where it is.  `repo`: {directory, confirm, url?, name?, ...}
+export async function importRepo(repo) {
+    const response = await apiPost(`${REPOS_API}/import`, repo);
+    if (response.ok) {
+        return (await response.json())['repo'];
+    }
+    await reposError(response, 'Failed to import the repo.');
+}

app/src/components/Common.test.js

@@ -1121,7 +1121,7 @@ describe('a row of buttons', () => {
          * the two had been inline before.
          *
          * A row fixes it without touching the stack: the row is the stretched child, and it lays
-         * its buttons out at their own width.  Nine sites, so this scan rather than nine
+         * its buttons out at their own width.  Thirteen sites, so this scan rather than thirteen
          * assertions -- and a scan because a stretched button is still a working button.
          */
         const offenders = [];
@@ -1140,8 +1140,8 @@ describe('a row of buttons', () => {
             }
         }
 
-        // Nine renders.  Common.js declares it, and a declaration is not a `<BackButton`.
-        expect(found).toBe(9);
+        // Thirteen renders.  Common.js declares it, and a declaration is not a `<BackButton`.
+        expect(found).toBe(13);
 
         expect(offenders).toEqual([]);
     });

app/src/components/Nav.js

@@ -61,6 +61,7 @@ const allLinks = [
     {text: 'Files', to: '/files', key: 'files'},
     {text: 'Playlists', to: '/playlists', key: 'playlists'},
     {text: 'Zim', to: '/zim', key: 'zim'},
+    {text: 'Repos', to: '/repos', key: 'repos'},
     {text: 'Inventory', to: '/inventory', key: 'inventory'},
     // No `end` on these two: a tab stays current for the pages beneath it, and `end` would
     // drop the mark the moment either grows a sub-page.

app/src/components/OtherSearchView.test.js

@@ -1,7 +1,7 @@
 import React from 'react';
 import {act, render, screen} from '../test-utils';
 import {OtherSearchView} from './Search';
-import {searchChannels} from '../api';
+import {searchChannels, searchRepos} from '../api';
 
 // What the search page's Other tab shows for each state of useSearchChannels.  "No Channels"
 // must only appear once the server has confirmed there are none.
@@ -9,11 +9,16 @@ import {searchChannels} from '../api';
 jest.mock('../api', () => ({
     ...jest.requireActual('../api'),
     searchChannels: jest.fn(),
+    searchRepos: jest.fn(),
 }));
 
 const NO_CHANNELS = /No Channels/;
 
-beforeEach(() => searchChannels.mockReset());
+beforeEach(() => {
+    searchChannels.mockReset();
+    searchRepos.mockReset();
+    searchRepos.mockResolvedValue({repos: [], total: 0});
+});
 
 test('pending: shows a loader, not "No Channels"', () => {
     searchChannels.mockReturnValue(new Promise(() => {
@@ -54,3 +59,61 @@ test('loaded: lists the channels', async () => {
     expect(screen.getByText('Cooking')).toBeInTheDocument();
     expect(screen.queryByText(NO_CHANNELS)).not.toBeInTheDocument();
 });
+
+describe('Repos', () => {
+    test('pending: the Repos section loads while Channels are shown', async () => {
+        searchChannels.mockResolvedValue({channels: []});
+        searchRepos.mockReturnValue(new Promise(() => {
+        }));
+        const {container} = render(<OtherSearchView loading={false}/>);
+        await act(async () => {
+        });
+        expect(screen.getByText(NO_CHANNELS)).toBeInTheDocument();
+        expect(container.querySelector('.wrolpi-loading')).toBeInTheDocument();
+        expect(screen.queryByText('No Repos')).not.toBeInTheDocument();
+    });
+
+    test('empty: shows "No Repos"', async () => {
+        searchChannels.mockResolvedValue({channels: []});
+        render(<OtherSearchView loading={false}/>);
+        await act(async () => {
+        });
+        expect(screen.getByText('No Repos')).toBeInTheDocument();
+    });
+
+    test('failed: shows an error only in the Repos section', async () => {
+        searchChannels.mockResolvedValue({channels: [{id: 1, name: 'Cooking'}]});
+        searchRepos.mockRejectedValue(new Error('boom'));
+        const spy = jest.spyOn(console, 'error').mockImplementation(() => {
+        });
+        try {
+            render(<OtherSearchView loading={false}/>);
+            await act(async () => {
+            });
+            expect(screen.getByText('Cooking')).toBeInTheDocument();
+            expect(screen.getByText(/Could not fetch the repos/)).toBeInTheDocument();
+        } finally {
+            spy.mockRestore();
+        }
+    });
+
+    test('loaded: lists repos by the search, with their README headline', async () => {
+        searchChannels.mockResolvedValue({channels: []});
+        searchRepos.mockResolvedValue({
+            total: 1, repos: [{
+                id: 7, name: 'kiwix-tools', tag_name: null,
+                // The server escapes the README; only the highlight is markup.
+                readme_headline: 'Serve <b>Zim</b> files &lt;script&gt;',
+            }],
+        });
+        const searchParams = new URLSearchParams('q=zim&tag=Software');
+        const {container} = render(<OtherSearchView loading={false}/>, {query: {searchParams}});
+        await act(async () => {
+        });
+        expect(searchRepos).toHaveBeenCalledWith('zim', ['Software']);
+        expect(screen.getByText('kiwix-tools').closest('a')).toHaveAttribute('href', '/repos/7');
+        expect(container.querySelector('u').textContent).toBe('Zim');
+        expect(container.querySelector('script')).toBeNull();
+        expect(screen.getByText(/files <script>/)).toBeInTheDocument();
+    });
+});

app/src/components/Repos.test.js

@@ -0,0 +1,394 @@
+import React from 'react';
+import {Route, Routes} from 'react-router';
+import {act, fireEvent, render, screen, waitFor, within} from '../test-utils';
+import {
+    RepoEditPage,
+    RepoImportPage,
+    repoNameFromURL,
+    RepoNewPage,
+    ReposPage,
+    RepoViewPage,
+    safeRepoHref,
+} from './Repos';
+import {
+    createRepo, deleteRepo, fetchRepos, getFile, getRepo, getRepoLog, getRepoTree, importRepo, inspectRepoImport,
+    updateRepo,
+} from '../api';
+import {FilePreviewContext} from './FilePreview';
+
+jest.mock('../api', () => ({
+    ...jest.requireActual('../api'),
+    createRepo: jest.fn(),
+    deleteRepo: jest.fn(),
+    fetchRepos: jest.fn(),
+    getFile: jest.fn(),
+    getRepo: jest.fn(),
+    getRepoLog: jest.fn(),
+    getRepoTree: jest.fn(),
+    importRepo: jest.fn(),
+    inspectRepoImport: jest.fn(),
+    updateRepo: jest.fn(),
+}));
+
+// The directory picker searches the server; picking a directory is all these specs need.
+jest.mock('./Common', () => ({
+    ...jest.requireActual('./Common'),
+    DirectorySearch: ({onSelect}) => <button type='button' onClick={() => onSelect('code/kiwix-tools')}>
+        Pick directory
+    </button>,
+}));
+
+// The README is fetched from the media server; its rendering is MarkdownPreview's own spec.
+jest.mock('./MarkdownPreview', () => ({
+    ...jest.requireActual('./MarkdownPreview'),
+    MarkdownPreview: ({url}) => <div data-testid='markdown-preview'>{url}</div>,
+}));
+
+const repoFixture = (overrides = {}) => ({
+    id: 1, collection_id: 10, name: 'kiwix-tools', tag_name: null, description: null,
+    url: 'https://github.com/kiwix/kiwix-tools', mode: 'full', branch: null, default_branch: 'main',
+    head_sha: '3f9c2a1e7b', head_message: 'Fix the mimetype', head_date: '2026-09-30T14:02:00Z',
+    size: 1000, directory: 'repos/kiwix-tools', frequency: 604800, download_error: null,
+    ...overrides,
+});
+
+const treeFixture = {
+    path: '',
+    readme_path: 'README.md',
+    entries: [
+        {name: 'src', path: 'src', is_dir: true, size: null},
+        {name: 'README.md', path: 'README.md', is_dir: false, size: 120},
+    ],
+};
+
+const flush = () => act(async () => {
+});
+
+beforeEach(() => jest.clearAllMocks());
+
+describe('safeRepoHref', () => {
+    it('only links https URLs', () => {
+        expect(safeRepoHref('https://github.com/a/b')).toBe('https://github.com/a/b');
+        expect(safeRepoHref('javascript:alert(1)')).toBeNull();
+        expect(safeRepoHref('http://github.com/a/b')).toBeNull();
+        expect(safeRepoHref('file:///etc/passwd')).toBeNull();
+        expect(safeRepoHref('not a url')).toBeNull();
+    });
+});
+
+describe('repoNameFromURL', () => {
+    it('is the last path segment, without .git or a branch', () => {
+        expect(repoNameFromURL('https://github.com/kiwix/kiwix-tools.git')).toBe('kiwix-tools');
+        expect(repoNameFromURL('https://github.com/kiwix/kiwix-tools/tree/dev')).toBe('kiwix-tools');
+        expect(repoNameFromURL('https://gitlab.com/group/sub/project/')).toBe('project');
+        expect(repoNameFromURL('nope')).toBe('');
+    });
+});
+
+describe('ReposPage', () => {
+    it('lists repos, linking each to its page', async () => {
+        fetchRepos.mockResolvedValue([
+            repoFixture(),
+            repoFixture({id: 2, name: 'gone', download_error: 'git ls-remote failed', head_sha: null}),
+        ]);
+        render(<ReposPage/>, {route: '/repos'});
+        await flush();
+        expect(screen.getAllByText('kiwix-tools')[0].closest('a')).toHaveAttribute('href', '/repos/1');
+        expect(screen.getAllByText('gone')[0].closest('a')).toHaveAttribute('href', '/repos/2');
+        expect(screen.getByText('New Repo').closest('a')).toHaveAttribute('href', '/repos/new');
+    });
+
+    it('shows a repo which is never updated', async () => {
+        fetchRepos.mockResolvedValue([repoFixture({frequency: 0})]);
+        render(<ReposPage/>, {route: '/repos'});
+        await flush();
+        expect(screen.getAllByText('Never').length).toBeGreaterThan(0);
+    });
+
+    it('explains an empty list', async () => {
+        fetchRepos.mockResolvedValue([]);
+        render(<ReposPage/>, {route: '/repos'});
+        await flush();
+        expect(screen.getByText(/No repos yet/)).toBeInTheDocument();
+    });
+});
+
+function renderViewPage({repo, tree = treeFixture, query = {}, setPreviewFile = jest.fn()} = {}) {
+    getRepo.mockResolvedValue(repo);
+    getRepoTree.mockResolvedValue(tree);
+    return render(<Routes><Route path='/repos/:repoId' element={<RepoViewPage/>}/></Routes>, {
+        route: '/repos/1',
+        query,
+        contexts: [[FilePreviewContext, {setPreviewFile}]],
+    });
+}
+
+describe('RepoViewPage', () => {
+    it('shows the latest commit, the files, and the README', async () => {
+        renderViewPage({repo: repoFixture()});
+        await flush();
+        expect(getRepoTree).toHaveBeenCalledWith('1', '');
+        expect(screen.getByText('3f9c2a1')).toBeInTheDocument();
+        expect(screen.getByText('Fix the mimetype')).toBeInTheDocument();
+        expect(screen.getByText('main')).toBeInTheDocument();
+        expect(screen.getByText('(default)')).toBeInTheDocument();
+        expect(screen.getByText('https://github.com/kiwix/kiwix-tools').closest('a'))
+            .toHaveAttribute('href', 'https://github.com/kiwix/kiwix-tools');
+        expect(screen.getByText('src')).toBeInTheDocument();
+        expect(screen.getByTestId('markdown-preview')).toHaveTextContent('/media/repos/kiwix-tools/README.md');
+    });
+
+    it('opens a directory through the URL, and a file in the preview', async () => {
+        const updateQuery = jest.fn();
+        const setPreviewFile = jest.fn();
+        const file = {path: 'repos/kiwix-tools/README.md', mimetype: 'text/markdown', size: 120};
+        getFile.mockResolvedValue(file);
+        renderViewPage({repo: repoFixture(), query: {updateQuery}, setPreviewFile});
+        await flush();
+
+        fireEvent.click(screen.getByText('src'));
+        expect(updateQuery).toHaveBeenCalledWith({path: 'src'}, false);
+
+        fireEvent.click(screen.getAllByText('README.md')[0]);
+        await flush();
+        expect(getFile).toHaveBeenCalledWith('repos/kiwix-tools/README.md', true);
+        expect(setPreviewFile).toHaveBeenCalledWith(file);
+    });
+
+    it('lists the directory in the URL, with a way back up', async () => {
+        const tree = {path: 'src/lib', readme_path: null, entries: [{name: 'a.c', path: 'src/lib/a.c', is_dir: false}]};
+        const updateQuery = jest.fn();
+        renderViewPage({repo: repoFixture(), tree, query: {searchParams: new URLSearchParams('path=src/lib'), updateQuery}});
+        await flush();
+        expect(getRepoTree).toHaveBeenCalledWith('1', 'src/lib');
+        expect(screen.queryByTestId('markdown-preview')).not.toBeInTheDocument();
+
+        fireEvent.click(screen.getByText('..'));
+        expect(updateQuery).toHaveBeenCalledWith({path: 'src'}, false);
+        // The breadcrumb's root.
+        fireEvent.click(screen.getAllByText('kiwix-tools').find(i => i.tagName === 'BUTTON'));
+        expect(updateQuery).toHaveBeenCalledWith({path: ''}, false);
+    });
+
+    it('offers a ZIP of the files', async () => {
+        renderViewPage({repo: repoFixture()});
+        await flush();
+        const link = screen.getByText('Download ZIP').closest('a');
+        expect(link).toHaveAttribute('href', expect.stringMatching(/\/api\/repos\/1\/archive\.zip$/));
+        expect(link).toHaveAttribute('download');
+    });
+
+    it('shows the history instead of the files', async () => {
+        getRepoLog.mockResolvedValue({
+            total: 51,
+            commits: [{sha: 'abcdef1234', author: 'Ada', date: '2026-09-30T14:02:00Z', message: 'Fix the mimetype'}],
+        });
+        const updateQuery = jest.fn();
+        renderViewPage({repo: repoFixture(), query: {searchParams: new URLSearchParams('view=history'), updateQuery}});
+        await flush();
+
+        expect(getRepoLog).toHaveBeenCalledWith(1, 0, 50);
+        expect(getRepoTree).not.toHaveBeenCalled();
+        expect(screen.getByText('51 commits')).toBeInTheDocument();
+        expect(screen.getByText('abcdef1')).toBeInTheDocument();
+        expect(screen.getByText('Ada')).toBeInTheDocument();
+
+        // The next page is appended.
+        getRepoLog.mockResolvedValue({
+            total: 51,
+            commits: [{sha: '0123456789', author: 'Ben', date: '2026-09-01T00:00:00Z', message: 'Older'}],
+        });
+        fireEvent.click(screen.getByText('Load more'));
+        await flush();
+        expect(getRepoLog).toHaveBeenLastCalledWith(1, 1, 50);
+        expect(screen.getByText('abcdef1')).toBeInTheDocument();
+        expect(screen.getByText('0123456')).toBeInTheDocument();
+
+        fireEvent.click(screen.getByText('Files'));
+        expect(updateQuery).toHaveBeenCalledWith({view: null}, false);
+    });
+
+    it('explains a repo that is not downloaded yet', async () => {
+        renderViewPage({repo: repoFixture({head_sha: null, head_message: null, head_date: null})});
+        await flush();
+        expect(screen.getByText('Not downloaded yet')).toBeInTheDocument();
+        expect(getRepoTree).not.toHaveBeenCalled();
+        expect(screen.queryByText('Download ZIP')).not.toBeInTheDocument();
+        expect(screen.queryByText('History')).not.toBeInTheDocument();
+    });
+
+    it('keeps showing the files when the last update failed', async () => {
+        renderViewPage({repo: repoFixture({download_error: 'Traceback...\nDownloadError: git ls-remote failed (128)'})});
+        await flush();
+        expect(screen.getByText('The last update failed')).toBeInTheDocument();
+        expect(screen.getByText(/git ls-remote failed/)).toBeInTheDocument();
+        expect(screen.getByText('src')).toBeInTheDocument();
+    });
+
+    it('says a repo which is never updated is only updated on request', async () => {
+        renderViewPage({repo: repoFixture({frequency: 0, download_error: 'git ls-remote failed (128)'})});
+        await flush();
+        expect(screen.getByText('The last update failed')).toBeInTheDocument();
+        expect(screen.getByText(/never updated automatically/)).toBeInTheDocument();
+        expect(screen.queryByText(/will be tried again/)).not.toBeInTheDocument();
+    });
+
+    it('never links an upstream that is not https', async () => {
+        renderViewPage({repo: repoFixture({url: 'javascript:alert(1)'})});
+        await flush();
+        expect(screen.queryByText('javascript:alert(1)')).not.toBeInTheDocument();
+    });
+});
+
+describe('RepoNewPage', () => {
+    it('adds a repo, then opens it', async () => {
+        createRepo.mockResolvedValue(repoFixture({id: 5}));
+        render(<Routes>
+            <Route path='/repos/new' element={<RepoNewPage/>}/>
+            <Route path='/repos/:repoId' element={<div>repo page</div>}/>
+        </Routes>, {route: '/repos/new'});
+
+        const add = screen.getByText('Add Repo').closest('button');
+        expect(add).toBeDisabled();
+        fireEvent.change(screen.getByLabelText(/URL/), {target: {value: 'git@github.com:a/b'}});
+        expect(add).toBeDisabled();
+        expect(screen.getByText('Must be an https:// URL')).toBeInTheDocument();
+
+        fireEvent.change(screen.getByLabelText(/URL/), {target: {value: 'https://github.com/kiwix/kiwix-tools'}});
+        expect(screen.getByPlaceholderText('kiwix-tools')).toBeInTheDocument();
+        fireEvent.change(screen.getByLabelText('Branch'), {target: {value: ' dev '}});
+        fireEvent.click(screen.getByText('Pick directory'));
+        fireEvent.click(add);
+        await flush();
+
+        expect(createRepo).toHaveBeenCalledWith({
+            url: 'https://github.com/kiwix/kiwix-tools', name: null, tag_name: null, frequency: 604800,
+            mode: 'full', branch: 'dev', submodules: false, description: null, directory: 'code/kiwix-tools',
+        });
+        expect(screen.getByText('repo page')).toBeInTheDocument();
+    });
+});
+
+describe('RepoEditPage', () => {
+    function renderEditPage(repo) {
+        getRepo.mockResolvedValue(repo);
+        return render(<Routes>
+            <Route path='/repos/:repoId/edit' element={<RepoEditPage/>}/>
+            <Route path='/repos' element={<div>repos page</div>}/>
+        </Routes>, {route: '/repos/1/edit'});
+    }
+
+    it('saves the settings', async () => {
+        updateRepo.mockResolvedValue(repoFixture());
+        renderEditPage(repoFixture({branch: 'dev', description: 'Tools'}));
+        await flush();
+        expect(screen.getByLabelText('Branch')).toHaveValue('dev');
+
+        expect(screen.getByLabelText('Include submodules')).not.toBeChecked();
+
+        fireEvent.change(screen.getByLabelText('Branch'), {target: {value: ''}});
+        fireEvent.click(screen.getByLabelText('Include submodules'));
+        fireEvent.click(screen.getByText('Save'));
+        await flush();
+        expect(updateRepo).toHaveBeenCalledWith('1', {
+            description: 'Tools', frequency: 604800, mode: 'full', branch: '', submodules: true,
+        });
+    });
+
+    it('keeps a repo which is never updated', async () => {
+        updateRepo.mockResolvedValue(repoFixture({frequency: 0}));
+        renderEditPage(repoFixture({frequency: 0}));
+        await flush();
+        expect(screen.getByDisplayValue('Never')).toBeInTheDocument();
+        fireEvent.click(screen.getByText('Save'));
+        await flush();
+        expect(updateRepo).toHaveBeenCalledWith('1', expect.objectContaining({frequency: 0}));
+    });
+
+    it('deletes the repo, and its files only when asked', async () => {
+        deleteRepo.mockResolvedValue(undefined);
+        renderEditPage(repoFixture());
+        await flush();
+
+        fireEvent.click(screen.getByText('Delete'));
+        const dialog = await screen.findByRole('dialog');
+        fireEvent.click(within(dialog).getByLabelText(/Also delete its files/));
+        fireEvent.click(within(dialog).getByText('Delete'));
+        await flush();
+        expect(deleteRepo).toHaveBeenCalledWith('1', true);
+        expect(screen.getByText('repos page')).toBeInTheDocument();
+    });
+});
+
+describe('RepoImportPage', () => {
+    const inspection = (overrides = {}) => ({
+        directory: 'code/kiwix-tools', origin: 'git@github.com:kiwix/kiwix-tools.git',
+        url: 'https://github.com/kiwix/kiwix-tools', name: 'kiwix-tools', branch: 'main',
+        head_sha: 'abcdef1234', head_message: 'Local work', head_date: '2026-09-30T14:02:00Z', local_commits: 2,
+        shallow: false, ignored: false,
+        ...overrides,
+    });
+
+    function renderImportPage() {
+        return render(<Routes>
+            <Route path='/repos/import' element={<RepoImportPage/>}/>
+            <Route path='/repos/:repoId' element={<div>repo page</div>}/>
+        </Routes>, {route: '/repos/import'});
+    }
+
+    it('inspects the directory, warns about local commits, and imports once confirmed', async () => {
+        inspectRepoImport.mockResolvedValue({inspection: inspection()});
+        importRepo.mockResolvedValue(repoFixture({id: 9}));
+        renderImportPage();
+
+        fireEvent.click(screen.getByText('Pick directory'));
+        expect(await screen.findByText('2 commits are not on the remote')).toBeInTheDocument();
+        expect(inspectRepoImport).toHaveBeenCalledWith('code/kiwix-tools', '', null, '');
+        // Inspected again with the URL the import will use.
+        await waitFor(() => expect(inspectRepoImport).toHaveBeenCalledWith('code/kiwix-tools', '', null,
+            'https://github.com/kiwix/kiwix-tools'));
+        expect(screen.getByText('git@github.com:kiwix/kiwix-tools.git')).toBeInTheDocument();
+        expect(screen.getByText(/Its files are indexed/)).toBeInTheDocument();
+        // The URL comes from the origin.
+        expect(screen.getByLabelText(/^URL/)).toHaveValue('https://github.com/kiwix/kiwix-tools');
+
+        const button = screen.getByText('Import').closest('button');
+        expect(button).toBeDisabled();
+        fireEvent.click(screen.getByLabelText(/I understand/));
+        expect(button).toBeEnabled();
+        fireEvent.click(button);
+        await flush();
+
+        expect(importRepo).toHaveBeenCalledWith({
+            directory: 'code/kiwix-tools', confirm: true, url: 'https://github.com/kiwix/kiwix-tools', name: null,
+            tag_name: null, frequency: 604800, mode: 'full', branch: null, submodules: false,
+        });
+        expect(screen.getByText('repo page')).toBeInTheDocument();
+    });
+
+    it('asks for the URL when the clone has no https origin', async () => {
+        inspectRepoImport.mockResolvedValue({inspection: inspection({origin: null, url: null, local_commits: 0})});
+        renderImportPage();
+        fireEvent.click(screen.getByText('Pick directory'));
+        expect(await screen.findByText(/no https:\/\/ origin/)).toBeInTheDocument();
+        fireEvent.click(screen.getByLabelText(/I understand/));
+        expect(screen.getByText('Import').closest('button')).toBeDisabled();
+        fireEvent.change(screen.getByLabelText(/^URL/), {target: {value: 'https://github.com/kiwix/kiwix-tools'}});
+        expect(screen.getByText('Import').closest('button')).toBeEnabled();
+    });
+
+    it('explains a directory which cannot be imported', async () => {
+        inspectRepoImport.mockResolvedValue({error: 'code/kiwix-tools is not a git clone'});
+        renderImportPage();
+        fireEvent.click(screen.getByText('Pick directory'));
+        expect(await screen.findByText('code/kiwix-tools is not a git clone')).toBeInTheDocument();
+        expect(screen.queryByText('Import')).not.toBeInTheDocument();
+    });
+
+    it('says when the files are not indexed', async () => {
+        inspectRepoImport.mockResolvedValue({inspection: inspection({ignored: true})});
+        renderImportPage();
+        fireEvent.click(screen.getByText('Pick directory'));
+        expect(await screen.findByText(/Its files are not indexed/)).toBeInTheDocument();
+    });
+});

app/src/components/Search.js

@@ -1,7 +1,15 @@
 import React, {useState} from "react";
 import {Link, Route, Routes, useLocation, useNavigate} from "react-router";
 import {FilesSearchView, SearchViewButton} from "./Files";
-import {useLatestRequest, usePages, useSearchChannels, useSearchDate, useSearchDeep, useSearchFilter} from "../hooks/customHooks";
+import {
+    useLatestRequest,
+    usePages,
+    useSearchChannels,
+    useSearchDate,
+    useSearchDeep,
+    useSearchFilter,
+    useSearchRepos,
+} from "../hooks/customHooks";
 import {ShortcutHint} from "./ShortcutHint";
 import {ZimSearchView} from "./Zim";
 import {MapSearchView} from "./MapSearchView";
@@ -12,6 +20,7 @@ import {TagsContext} from "../Tags";
 import {Accordion, Header, Icon, Label, Loading, Panel, Stack} from "./ui";
 import {QueryContext} from "../contexts/contexts";
 import {KeyboardShortcutsContext} from "../contexts/KeyboardShortcutsContext";
+import {HeadlineText} from "./Headline";
 
 const SUGGESTED_APPS = [
     {location: '/admin', title: 'Downloads', description: 'View and control your downloads'},
@@ -139,6 +148,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
         domains: null,
         authors: null,
         subjects: null,
+        repos: null,
         zimsEstimates: [],
     }
     const [suggestions, setSuggestions] = React.useState(defaultSuggestions);
@@ -163,7 +173,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
             // The files estimate always uses deep search server-side, so users know results are possible.
             sendFilesRequest(async () => await searchEstimateFiles(searchStr, tagNames, mimetypes, months, dateRange, anyTag));
             sendZimRequest(async () => await searchEstimateZims(searchStr, tagNames));
-            sendOtherRequest(async () => await searchEstimateOthers(tagNames));
+            sendOtherRequest(async () => await searchEstimateOthers(tagNames, searchStr));
         }
     }, [
         searchStr,
@@ -185,6 +195,7 @@ export function useSuggestions(searchStr, tagNames, filter, anyTag) {
                     domains: generalData.domains,
                     authors: generalData.authors,
                     subjects: generalData.subjects,
+                    repos: generalData.repos,
                 }
             });
         }
@@ -312,6 +323,13 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
                 })
             }
         }
+        if (newSuggestions.repos && newSuggestions.repos.length > 0) {
+            results.repos = {
+                name: 'Repos', results: newSuggestions.repos.map(i => {
+                    return {type: 'repo', title: i.name, id: i.id, location: `/repos/${i.id}`}
+                })
+            }
+        }
         if (newSuggestions.authors && newSuggestions.authors.length > 0) {
             results.authors = {
                 name: 'Authors', results: newSuggestions.authors.map(i => {
@@ -407,6 +425,7 @@ export function useSearchSuggestions(defaultSearchStr, defaultTagNames, anyTag)
             domains: newSuggestions.domains?.length,
             authors: newSuggestions.authors?.length,
             subjects: newSuggestions.subjects?.length,
+            repos: newSuggestions.repos?.length,
             tags: matchingTags?.length,
             apps: matchingApps?.length,
         });
@@ -543,12 +562,47 @@ function SearchChannelPreview({channel}) {
     </div>
 }
 
-// `channels` follows useSearchChannels: null = pending, undefined = fetch failed, [] = none.
+function SearchRepoPreview({repo}) {
+    const {SingleTag} = React.useContext(TagsContext);
+    return <div>
+        <span style={{display: 'inline-flex', alignItems: 'center', gap: 8, flexWrap: 'wrap'}}>
+            <Link to={`/repos/${repo.id}`}>{repo.name}</Link>
+            {repo.tag_name && <SingleTag name={repo.tag_name}/>}
+        </span>
+        {/* The server escapes the README; only its <b> highlights are markup. */}
+        {repo.readme_headline && <div><small><HeadlineText headline={repo.readme_headline}/></small></div>}
+    </div>
+}
+
+function OtherSearchSection({value, title, items, preview, emptyText, errorText}) {
+    let body;
+    if (items === null) {
+        body = <Loading/>;
+    } else if (items === undefined) {
+        body = <ErrorMessage>{errorText}</ErrorMessage>;
+    } else {
+        body = <Stack gap='xs'>{!_.isEmpty(items) ? items.map(preview) : emptyText}</Stack>;
+    }
+    return <Accordion.Item value={value}>
+        <Accordion.Control>
+            <span style={{display: 'inline-flex', alignItems: 'center', gap: 8}}>
+                <Header as='h3' style={{margin: 0}}>{title}</Header>
+                {Array.isArray(items) && <Label>{normalizeEstimate(items.length)}</Label>}
+            </span>
+        </Accordion.Control>
+        <Accordion.Panel>{body}</Accordion.Panel>
+    </Accordion.Item>
+}
+
+// `channels` follows useSearchChannels and `repos` follows useSearchRepos: null = pending,
+// undefined = fetch failed, [] = none.
 export function OtherSearchView({loading}) {
     const {searchParams} = React.useContext(QueryContext);
-    const [activeValue, setActiveValue] = React.useState('channels');
+    const [activeValues, setActiveValues] = React.useState(['channels', 'repos']);
     const activeTags = searchParams.getAll('tag');
+    const searchStr = searchParams.get('q') || '';
     const {channels, loading: channelsLoading} = useSearchChannels(activeTags);
+    const {repos} = useSearchRepos(searchStr, activeTags);
 
     if (loading || channelsLoading || channels === null) {
         return <Panel><Loading/></Panel>
@@ -556,21 +610,22 @@ export function OtherSearchView({loading}) {
         return <Panel><ErrorMessage>Could not fetch the channels.</ErrorMessage></Panel>
     }
 
-    return <Accordion value={activeValue} onChange={setActiveValue}>
-        <Accordion.Item value='channels'>
-            <Accordion.Control>
-                <span style={{display: 'inline-flex', alignItems: 'center', gap: 8}}>
-                    <Header as='h3' style={{margin: 0}}>Channels</Header>
-                    <Label>{normalizeEstimate(channels.length)}</Label>
-                </span>
-            </Accordion.Control>
-            <Accordion.Panel>
-                <Stack gap='xs'>
-                    {!_.isEmpty(channels) ?
-                        channels.map(i => <SearchChannelPreview key={i.id} channel={i}/>)
-                        : 'No Channels'}
-                </Stack>
-            </Accordion.Panel>
-        </Accordion.Item>
+    return <Accordion multiple value={activeValues} onChange={setActiveValues}>
+        <OtherSearchSection
+            value='channels'
+            title='Channels'
+            items={channels}
+            preview={i => <SearchChannelPreview key={i.id} channel={i}/>}
+            emptyText='No Channels'
+            errorText='Could not fetch the channels.'
+        />
+        <OtherSearchSection
+            value='repos'
+            title='Repos'
+            items={repos}
+            preview={i => <SearchRepoPreview key={i.id} repo={i}/>}
+            emptyText='No Repos'
+            errorText='Could not fetch the repos.'
+        />
     </Accordion>
 }

app/src/components/SearchSuggestions.test.js

@@ -103,3 +103,37 @@ test('Channels rows once the server found some', async () => {
     });
     expect(results().channels.results.map(i => i.title)).toEqual(['Food']);
 });
+
+test('Repos rows link to each repo, and the Other estimate gets the search', async () => {
+    let resolve;
+    searchSuggestions.mockReturnValue(new Promise(res => resolve = res));
+    renderProbe();
+    act(() => {
+        hook.setSearchStr('kiwix');
+    });
+    act(() => {
+        jest.advanceTimersByTime(DEBOUNCE);
+    });
+    await act(async () => {
+        resolve({channels: [], domains: [], authors: [], subjects: [], repos: [{id: 4, name: 'kiwix-tools'}]});
+    });
+    expect(results().repos.results).toEqual([{type: 'repo', title: 'kiwix-tools', id: 4, location: '/repos/4'}]);
+    // Repos count toward the Other tab by their name and README, not only by tag.
+    expect(searchEstimateOthers).toHaveBeenCalledWith([], 'kiwix');
+});
+
+test('no Repos group when none match', async () => {
+    let resolve;
+    searchSuggestions.mockReturnValue(new Promise(res => resolve = res));
+    renderProbe();
+    act(() => {
+        hook.setSearchStr('foo');
+    });
+    act(() => {
+        jest.advanceTimersByTime(DEBOUNCE);
+    });
+    await act(async () => {
+        resolve({channels: [], domains: [], authors: [], subjects: [], repos: []});
+    });
+    expect(results().repos).toBeUndefined();
+});

app/src/components/ThemeSamplePage.js

@@ -49,6 +49,9 @@ import {contrastingColor} from './Common';
 import {navBarStyle, navColorNames, useNavColors} from '../themes/navColors';
 import {IconMenu2} from '@tabler/icons-react';
 import {BookmarkAddForms, BookmarkMenuItems, BookmarkTable} from './Bookmarks';
+import {
+    RepoBreadcrumb, RepoHistoryTable, RepoImportSummary, RepoStatusIcon, RepoSummary, RepoTreeTable,
+} from './Repos';
 
 /*
  * A gallery of every component in the library, in the current theme.
@@ -170,6 +173,28 @@ const sampleBookmarks = [
 ];
 const noBookmarkActions = {edit: () => null, move: () => null, remove: () => null};
 
+const sampleRepo = {
+    id: 1, name: 'kiwix-tools', tag_name: null, url: 'https://github.com/kiwix/kiwix-tools',
+    description: 'Command line tools for Kiwix and Zim files.', mode: 'full', branch: null, default_branch: 'main',
+    head_sha: '3f9c2a1e7b', head_message: 'Serve Zim files with the correct mimetype', head_date: '2026-09-30T14:02:00Z',
+    size: 18_400_000, directory: 'repos/kiwix-tools', download_error: null,
+};
+const sampleFailedRepo = {
+    ...sampleRepo, name: 'gone-upstream', branch: 'release', mode: 'snapshot',
+    download_error: "git ls-remote failed (128): remote: Repository not found.\nfatal: repository " +
+        "'https://github.com/example/gone-upstream/' not found",
+};
+const sampleRepoCommits = [
+    {sha: '3f9c2a1e7b', author: 'Ada Example', date: '2026-09-30T14:02:00Z',
+        message: 'Serve Zim files with the correct mimetype'},
+    {sha: '91b0d44c02', author: 'Ben Example', date: '2026-09-12T09:15:00Z', message: 'Merge pull request #42'},
+];
+const sampleRepoEntries = [
+    {name: 'docs', path: 'src/docs', is_dir: true, size: null},
+    {name: 'kiwix-serve.cpp', path: 'src/kiwix-serve.cpp', is_dir: false, size: 48_210},
+    {name: 'README.md', path: 'src/README.md', is_dir: false, size: 2_104},
+];
+
 export function ThemeSamplePage() {
     const {theme, mediaFilterEnabled} = useContext(ThemeContext);
     const [confirmOpen, setConfirmOpen] = useState(false);
@@ -1127,6 +1152,36 @@ export function ThemeSamplePage() {
             </Panel>
         </Section>
 
+        <Section label='Repos'>
+            <Panel>
+                <p style={{fontSize: '0.75rem', color: 'var(--muted)', marginTop: 0}}>
+                    A git repo's page: its summary, then its files.  A repo whose last update failed keeps
+                    its files and explains why.  The status icons appear beside a repo's name in the Repos
+                    table: <RepoStatusIcon repo={sampleFailedRepo}/> failed,{' '}
+                    <RepoStatusIcon repo={{...sampleRepo, head_sha: null}}/> not downloaded yet.
+                </p>
+                <RepoSummary repo={sampleRepo}/>
+                <div style={{marginTop: 12}}>
+                    <RepoBreadcrumb name={sampleRepo.name} path='src' onNavigate={() => null}/>
+                    <RepoTreeTable path='src' entries={sampleRepoEntries} onOpenDirectory={() => null}
+                                   onOpenFile={() => null}/>
+                </div>
+                <div style={{marginTop: 12}}>
+                    <RepoHistoryTable commits={sampleRepoCommits}/>
+                </div>
+                <div style={{marginTop: 12}}>
+                    <RepoImportSummary inspection={{
+                        origin: 'git@github.com:kiwix/kiwix-tools.git', branch: 'main', head_sha: '3f9c2a1e7b',
+                        head_message: 'Local work', head_date: '2026-09-30T14:02:00Z', local_commits: 2,
+                        shallow: true, directory: 'code/kiwix-tools', ignored: false,
+                    }}/>
+                </div>
+                <div style={{marginTop: 12}}>
+                    <RepoSummary repo={sampleFailedRepo}/>
+                </div>
+            </Panel>
+        </Section>
+
         <Section label='Danger zone'>
             <Panel danger>
                 <h2 style={{color: 'var(--danger)', fontSize: '0.875rem', margin: '0 0 4px'}}>Danger zone</h2>

app/src/components/Vars.js

@@ -3,6 +3,7 @@ export const VIDEOS_API = `${API_URI}/videos`;
 export const ARCHIVES_API = `${API_URI}/archive`;
 export const COLLECTIONS_API = `${API_URI}/collections`;
 export const ZIM_API = `${API_URI}/zim`;
+export const REPOS_API = `${API_URI}/repos`;
 export const DEFAULT_LIMIT = 20;
 export const NAME = process.env && process.env.REACT_APP_NAME ? process.env.REACT_APP_NAME : null;
 

app/src/components/admin/Settings.js

@@ -499,6 +499,7 @@ export function SettingsPage() {
             videos_destination: settings.videos_destination,
             zims_destination: settings.zims_destination,
             playlists_destination: settings.playlists_destination,
+            repos_destination: settings.repos_destination,
             save_ffprobe_json: settings.save_ffprobe_json,
         });
     }, [JSON.stringify(settings), settingsLoaded, settingsFailed]);
@@ -849,6 +850,32 @@ export function SettingsPage() {
                             onChange={e => handleInputChange('playlists_destination', e.currentTarget.value)}
                         />
                     </Grid.Col>
+                    <Grid.Col span={{base: 12, sm: 6}}>
+                        <PathInput
+                            label={<>
+                                Repos Directory
+                                <InfoPopup
+                                    content={<>
+                                        <p>Variables:</p>
+                                        <ul>
+                                            <li><code>%(repo_name)s</code> - Repo name</li>
+                                            <li><code>%(repo_tag)s</code> - Tag name for the repo (empty if no tag)
+                                            </li>
+                                            <li><code>%(repo_owner)s</code> - Owner from the repo URL</li>
+                                            <li><code>%(repo_host)s</code> - Host from the repo URL</li>
+                                        </ul>
+                                        <p>Must start with a fixed directory (like <code>repos/</code>), which
+                                            is never indexed, and contain <code>%(repo_name)s</code>.</p>
+                                    </>}
+                                    position='top'
+                                />
+                            </>}
+                            prefix={mediaDirectoryLabel}
+                            value={state.repos_destination}
+                            disabled={!editSpecialDirectories}
+                            onChange={e => handleInputChange('repos_destination', e.currentTarget.value)}
+                        />
+                    </Grid.Col>
                     <Grid.Col span={12}>
                         <Toggle
                             label='Edit Directories'

app/src/components/collections/CollectionTable.js

@@ -77,7 +77,9 @@ function renderRow(collection, columns, routes, SingleTag, onRowClick) {
         }
 
         // Format the value based on column configuration
-        if (col.format === 'bytes') {
+        if (col.render) {
+            value = col.render(collection);
+        } else if (col.format === 'bytes') {
             value = humanFileSize(value);
         } else if (col.format === 'frequency') {
             value = formatFrequency(value);
@@ -134,7 +136,9 @@ function MobileCollectionRow({collection, mobileColumns, routes}) {
                 .filter(col => col.type !== 'actions' && col.key !== primaryColumn.key && col.key !== 'tag_name')
                 .map(col => {
                     let value = collection[col.key];
-                    if (col.format === 'bytes') {
+                    if (col.render) {
+                        value = col.render(collection);
+                    } else if (col.format === 'bytes') {
                         value = humanFileSize(value);
                     } else if (col.format === 'frequency') {
                         value = formatFrequency(value);
@@ -157,7 +161,8 @@ function MobileCollectionRow({collection, mobileColumns, routes}) {
  * Reusable table component for displaying collections (Domains, Channels, etc).
  *
  * @param {Array} collections - Array of collection objects
- * @param {Array} columns - Column configurations for the table
+ * @param {Array} columns - Column configurations for the table.  A column's optional `render(collection)` displays
+ *  the cell; the column still sorts by `collection[key]`.
  * @param {Object} routes - Routes configuration for navigation (edit, search, etc.)
  * @param {String} searchStr - Search filter string (managed by parent)
  * @param {Function} onRowClick - Optional callback when a row is clicked

app/src/components/collections/CollectionTable.test.js

@@ -68,6 +68,27 @@ const DOMAIN_ROUTES = {
 describe('CollectionTable', () => {
     const mockCollections = createMockDomains(3);
 
+    describe('Column render', () => {
+        it('displays what a column renders, in the desktop and mobile rows', () => {
+            const columns = [
+                {key: 'name', label: 'Name', render: (c) => <em data-testid='rendered-name'>{c.name}!</em>},
+                {key: 'branch', label: 'Branch', render: (c) => c.branch || c.default_branch},
+            ];
+            render(<CollectionTable
+                collections={[{id: 1, name: 'kiwix-tools', branch: null, default_branch: 'main'}]}
+                columns={columns}
+                routes={{search: '/repos/:id', id_field: 'id'}}
+            />);
+
+            // Desktop: the rendered name is still the link to the collection.
+            expect(screen.getByTestId('rendered-name')).toHaveTextContent('kiwix-tools!');
+            expect(screen.getByTestId('rendered-name').closest('a')).toHaveAttribute('href', '/repos/1');
+            expect(screen.getByText('main')).toBeInTheDocument();
+            // Mobile lists the other columns by label.
+            expect(screen.getByText('Branch: main')).toBeInTheDocument();
+        });
+    });
+
     describe('Loading and Error States', () => {
         it('renders loading placeholder when collections is null', () => {
             const {container} = render(

app/src/hooks/customHooks.js

@@ -38,6 +38,7 @@ import {
     searchChannels,
     searchDirectories,
     searchDocs,
+    searchRepos,
     searchVideos,
     searchZim,
     updateChannel,
@@ -2156,6 +2157,27 @@ export const useSearchChannels = (defaultTagNames) => {
     }
 }
 
+// Repos matching the search (their name and README) and/or the tags.  `repos`: null = pending,
+// undefined = fetch failed, [] = none.  Only the newest request may write state.
+export const useSearchRepos = (searchStr, tagNames) => {
+    const [repos, setRepos] = useState(null);
+    const requestGen = useRef(0);
+    const tagKey = JSON.stringify(tagNames || []);
+
+    useEffect(() => {
+        const gen = ++requestGen.current;
+        setRepos(null);
+        searchRepos(searchStr, JSON.parse(tagKey))
+            .then(result => gen === requestGen.current && setRepos(result ? result.repos : undefined))
+            .catch(e => {
+                console.error(e);
+                if (gen === requestGen.current) setRepos(undefined);
+            });
+    }, [searchStr, tagKey]);
+
+    return {repos};
+}
+
 
 /**
  * Detect whether the WROLPi browser extension is installed AND has this

app/src/test-fixtures.js

@@ -71,6 +71,7 @@ export const settingsFixture = (overrides = {}) => ({
     media_directory: '/media/wrolpi',
     nav_color: 'violet',
     playlists_destination: 'playlists',
+    repos_destination: 'repos/%(repo_tag)s/%(repo_name)s',
     require_cookies_unlocked: false,
     require_media_mounted: false,
     save_ffprobe_json: true,

conftest.py

@@ -1,5 +1,6 @@
 from modules.archive.conftest import *  # noqa
 from modules.inventory.conftest import *  # noqa
+from modules.repos.conftest import *  # noqa
 from modules.videos.conftest import *  # noqa
 from modules.zim.conftest import *  # noqa
 from wrolpi.conftest import *  # noqa

docker/help/wrolpi-help

@@ -1 +1 @@
-Subproject commit b61bae309cf253e080824ac12eb8b4fb690b8456
+Subproject commit 1b49385d674bbe4d98d2708f3f1e79297c9b0c00

modules/__init__.py

@@ -2,5 +2,6 @@ from .archive import api
 from .docs import api
 from .inventory import api
 from .map import api
+from .repos import api
 from .videos import api
 from .zim import api, test

modules/repos/__init__.py

@@ -0,0 +1,3 @@
+from .downloader import GitDownloader  # noqa
+
+__all__ = []

modules/repos/api.py

@@ -0,0 +1,182 @@
+from http import HTTPStatus
+
+from sanic import Request, Blueprint, response
+from sanic_ext.extensions.openapi import openapi
+
+from wrolpi.api_utils import json_response
+from wrolpi.common import wrol_mode_check, logger
+from wrolpi.errors import ValidationError
+from . import lib, schema
+from .models import Repository
+
+repos_bp = Blueprint('Repos', '/api/repos')
+
+logger = logger.getChild(__name__)
+
+
+@repos_bp.get('/')
+@openapi.definition(summary='List all git Repos')
+async def get_repos(request: Request):
+    repos = lib.get_repositories(request.ctx.session)
+    return json_response(dict(repos=repos))
+
+
+@repos_bp.post('/')
+@openapi.definition(
+    summary='Add a git Repo; it will be cloned, then kept up to date',
+    body=schema.RepoCreateRequest,
+    validate=True,
+)
+@wrol_mode_check
+async def post_repo(request: Request, body: schema.RepoCreateRequest):
+    lib.get_repos_config().check_imported()
+    session = request.ctx.session
+    repo = lib.create_repository(
+        session,
+        body.url,
+        tag_name=body.tag_name or None,
+        frequency=lib.DEFAULT_REPO_FREQUENCY if body.frequency is None else body.frequency,
+        mode=body.mode or 'full',
+        branch=body.branch or None,
+        name=body.name or None,
+        description=body.description or None,
+        submodules=body.submodules,
+        directory=(body.directory or '').strip() or None,
+    )
+    repo = Repository.find_by_id(session, repo.id, options=Repository.json_options())
+    return json_response(dict(repo=repo.__json__()), status=HTTPStatus.CREATED)
+
+
+@repos_bp.post('/import/inspect')
+@openapi.definition(
+    summary='What importing an existing git clone (in the media directory) as a Repo would do.  Changes nothing.',
+    body=schema.RepoImportInspectRequest,
+    validate=True,
+)
+async def post_inspect_import(request: Request, body: schema.RepoImportInspectRequest):
+    info = lib.inspect_import(request.ctx.session, body.directory, body.name or None, body.tag_name or None,
+                              body.url or None)
+    return json_response(dict(inspection=info))
+
+
+@repos_bp.post('/import')
+@openapi.definition(
+    summary='Import an existing git clone as a Repo.  It stays where it is, and becomes a mirror of its origin '
+            '(local changes are discarded by its updates).',
+    body=schema.RepoImportRequest,
+    validate=True,
+)
+@wrol_mode_check
+async def post_import(request: Request, body: schema.RepoImportRequest):
+    lib.get_repos_config().check_imported()
+    session = request.ctx.session
+    repo = await lib.import_repository(
+        session,
+        body.directory,
+        url=body.url or None,
+        confirm=body.confirm,
+        tag_name=body.tag_name or None,
+        frequency=lib.DEFAULT_REPO_FREQUENCY if body.frequency is None else body.frequency,
+        mode=body.mode or 'full',
+        branch=body.branch or None,
+        submodules=body.submodules,
+        name=body.name or None,
+        description=body.description or None,
+    )
+    repo = Repository.find_by_id(session, repo.id, options=Repository.json_options())
+    return json_response(dict(repo=repo.__json__()), status=HTTPStatus.CREATED)
+
+
+@repos_bp.post('/search')
+@openapi.definition(
+    summary='Search git Repos by their name and README, and/or by Tag',
+    body=schema.RepoSearchRequest,
+    validate=True,
+)
+async def post_search_repos(request: Request, body: schema.RepoSearchRequest):
+    limit = min(max(body.limit, 1), 100)
+    repos, total = lib.search_repos(request.ctx.session, body.search_str, body.tag_names, limit=limit,
+                                    offset=max(body.offset, 0))
+    return json_response(dict(repos=repos, totals=dict(repos=total)))
+
+
+@repos_bp.get('/<repo_id:int>')
+@openapi.definition(summary='Get a git Repo')
+async def get_repo(request: Request, repo_id: int):
+    repo = Repository.find_by_id(request.ctx.session, repo_id, options=Repository.json_options())
+    return json_response(dict(repo=repo.__json__()))
+
+
+@repos_bp.put('/<repo_id:int>')
+@openapi.definition(
+    summary="Change a git Repo's settings",
+    body=schema.RepoUpdateRequest,
+    validate=True,
+)
+@wrol_mode_check
+async def put_repo(request: Request, repo_id: int, body: schema.RepoUpdateRequest):
+    lib.get_repos_config().check_imported()
+    session = request.ctx.session
+    lib.update_repository(session, repo_id, description=body.description, frequency=body.frequency,
+                          mode=body.mode, branch=body.branch, submodules=body.submodules)
+    repo = Repository.find_by_id(session, repo_id, options=Repository.json_options())
+    return json_response(dict(repo=repo.__json__()))
+
+
+@repos_bp.delete('/<repo_id:int>')
+@openapi.definition(summary='Delete a git Repo.  Its files are kept unless `delete_files=true`.')
+@wrol_mode_check
+async def delete_repo(request: Request, repo_id: int):
+    lib.get_repos_config().check_imported()
+    delete_files = request.args.get('delete_files', '').lower() in ('1', 'true', 'yes')
+    await lib.delete_repository(request.ctx.session, repo_id, delete_files=delete_files)
+    return response.empty()
+
+
+@repos_bp.post('/<repo_id:int>/update')
+@openapi.definition(summary='Update a git Repo now')
+@wrol_mode_check
+async def post_update_repo(request: Request, repo_id: int):
+    session = request.ctx.session
+    repo = Repository.find_by_id(session, repo_id)
+    lib.request_repository_update(session, repo)
+    session.commit()
+    return response.empty(HTTPStatus.NO_CONTENT)
+
+
+@repos_bp.get('/<repo_id:int>/tree')
+@openapi.definition(summary="List a directory of a git Repo's files")
+@openapi.parameter('path', str, 'query', description='Directory in the repo, relative to its root', required=False)
+async def get_repo_tree(request: Request, repo_id: int):
+    repo = Repository.find_by_id(request.ctx.session, repo_id)
+    tree = lib.list_repo_tree(repo, request.args.get('path', ''))
+    return json_response(tree)
+
+
+@repos_bp.get('/<repo_id:int>/log')
+@openapi.definition(summary="The commits of a git Repo's checked out branch, newest first")
+@openapi.parameter('limit', int, 'query', description='At most 100', required=False)
+@openapi.parameter('offset', int, 'query', required=False)
+async def get_repo_log(request: Request, repo_id: int):
+    repo = Repository.find_by_id(request.ctx.session, repo_id)
+    try:
+        limit = int(request.args.get('limit', 50))
+        offset = int(request.args.get('offset', 0))
+    except ValueError:
+        raise ValidationError('limit and offset must be numbers')
+    log = await lib.get_repo_log(repo, limit=limit, offset=offset)
+    return json_response(log)
+
+
+@repos_bp.get('/<repo_id:int>/archive.zip')
+@openapi.definition(summary="Download a ZIP of a git Repo's checked out files")
+async def get_repo_archive(request: Request, repo_id: int):
+    repo = Repository.find_by_id(request.ctx.session, repo_id)
+    # Read everything from the database before the response starts; the request's session closes then.
+    directory = lib.get_repo_root(repo)
+    name = lib.repo_archive_name(repo)
+    response = await request.respond(content_type='application/zip', headers={
+        'Content-Disposition': f'attachment; filename="{name}.zip"',
+    })
+    await lib.stream_repo_archive(directory, name, response.send)
+    await response.eof()

modules/repos/conftest.py

@@ -0,0 +1,104 @@
+import os
+import pathlib
+import subprocess
+from typing import Callable
+
+import pytest
+from mock import mock
+
+from modules.repos import lib
+from modules.repos.downloader import GitDownloader
+from wrolpi.downloader import DownloadManager
+
+# Commits made by the tests have a fixed identity and do not read the developer's git config.
+TEST_GIT_ENV = dict(
+    GIT_AUTHOR_NAME='WROLPi Test', GIT_AUTHOR_EMAIL='test@example.com',
+    GIT_COMMITTER_NAME='WROLPi Test', GIT_COMMITTER_EMAIL='test@example.com',
+    GIT_CONFIG_NOSYSTEM='1', GIT_CONFIG_GLOBAL=os.devnull,
+)
+
+
+class GitRemote:
+    """A local repo which the Repos module sees as `https://git.example.com/<owner>/<name>`."""
+
+    def __init__(self, path: pathlib.Path, url: str):
+        self.path = path
+        self.url = url
+
+    def git(self, *args: str) -> str:
+        env = dict(os.environ, **TEST_GIT_ENV)
+        result = subprocess.run(['git', *args], cwd=self.path, env=env, check=True, capture_output=True)
+        return result.stdout.decode().strip()
+
+    def commit(self, files: dict, message: str = 'Commit') -> str:
+        """Write `files` ({relative path: contents}), commit them, return the commit's sha."""
+        for name, contents in files.items():
+            file = self.path / name
+            file.parent.mkdir(parents=True, exist_ok=True)
+            file.write_text(contents)
+        self.git('add', '--all')
+        self.git('commit', '--quiet', '-m', message)
+        return self.git('rev-parse', 'HEAD')
+
+    @property
+    def head(self) -> str:
+        return self.git('rev-parse', 'HEAD')
+
+    def add_submodule(self, path: str, submodule: 'GitRemote', url: str = None) -> str:
+        """Add `submodule` (at its current commit) at `path`, and commit it.  `url` overrides the submodule's URL."""
+        gitmodules = self.path / '.gitmodules'
+        existing = gitmodules.read_text() if gitmodules.exists() else ''
+        gitmodules.write_text(f'{existing}[submodule "{path}"]\n\tpath = {path}\n\turl = {url or submodule.url}\n')
+        self.git('update-index', '--add', '--cacheinfo', f'160000,{submodule.head},{path}')
+        self.git('add', '.gitmodules')
+        self.git('commit', '--quiet', '-m', f'Add submodule {path}')
+        return self.head
+
+
+@pytest.fixture
+def git_remote_factory(tmp_path_factory) -> Callable[..., GitRemote]:
+    """Create local git repos which are cloned over `https://git.example.com/...` URLs.
+
+    git's `url.<base>.insteadOf` rewrites the https URL to the local path, and `file://` is allowed for the test,
+    so the Repos module runs the same commands it runs in production."""
+    remotes = dict()
+    # Outside the media directory (which is `tmp_path`).
+    remotes_directory = tmp_path_factory.mktemp('remotes')
+
+    def git_env():
+        env = original_git_env()
+        env['GIT_CONFIG_COUNT'] = str(len(remotes))
+        for idx, (url, path) in enumerate(remotes.items()):
+            env[f'GIT_CONFIG_KEY_{idx}'] = f'url.file://{path}.insteadOf'
+            env[f'GIT_CONFIG_VALUE_{idx}'] = url
+        return env
+
+    original_git_env = lib.git_env
+
+    def factory(name: str = 'example', owner: str = 'owner', branch: str = 'main',
+                files: dict = None) -> GitRemote:
+        path = remotes_directory / owner / name
+        path.mkdir(parents=True)
+        url = f'https://git.example.com/{owner}/{name}'
+        remote = GitRemote(path, url)
+        remote.git('init', '--quiet', f'--initial-branch={branch}')
+        remote.commit(files or {'README.md': f'# {name}\n\nThe {name} repo.\n'}, 'Initial commit')
+        remotes[url] = path
+        return remote
+
+    def resolve_host_addresses(host):
+        # The remotes' host has a fixed public address; no test uses real DNS.
+        if host == 'git.example.com':
+            return ['93.184.215.14']
+        raise OSError(f'Unknown host {host}')
+
+    with mock.patch('modules.repos.lib.ALLOWED_PROTOCOLS', ('https', 'file')), \
+            mock.patch('modules.repos.downloader.git_env', git_env), \
+            mock.patch('modules.repos.lib.resolve_host_addresses', resolve_host_addresses):
+        yield factory
+
+
+@pytest.fixture
+def repos_download_manager(test_download_manager) -> DownloadManager:
+    test_download_manager.register_downloader(GitDownloader())
+    yield test_download_manager

modules/repos/downloader.py

@@ -0,0 +1,338 @@
+import asyncio
+import pathlib
+import re
+from dataclasses import dataclass
+from datetime import datetime
+from typing import Optional
+
+from sqlalchemy.orm import Session
+
+from wrolpi.common import logger
+from wrolpi.dates import now
+from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult, make_progress_callback
+from wrolpi.errors import DownloadError
+from .lib import git_command, git_env, find_readme, get_repo_size, is_valid_branch, read_readme_text, \
+    submodule_fetch_config, resolve_repo_addresses, pin_config, HostAddresses, clone_ownership_error, mark_clone, \
+    sanitize_git_config, new_clone_token, working_tree_git_error, repo_directory_error, get_repository_by_url
+from .models import Repository
+
+__all__ = ['GitDownloader', 'git_downloader', 'PreparedRepo', 'ExecutedRepo', 'parse_git_progress']
+
+logger = logger.getChild(__name__)
+
+# e.g. "Receiving objects:  45% (123/456), 1.20 MiB | 1.00 MiB/s"
+GIT_PROGRESS_RE = re.compile(r'^(?:remote: )?(?P<stage>[A-Za-z ]+):\s+(?P<percent>\d+)%')
+
+# Submodules of submodules are checked out this many levels deep.
+MAXIMUM_SUBMODULE_DEPTH = 5
+
+# Full mode keeps every branch of the remote as `origin/<branch>`.
+FULL_REFSPEC = '+refs/heads/*:refs/remotes/origin/*'
+
+
+def parse_git_progress(line: str) -> Optional[dict]:
+    if match := GIT_PROGRESS_RE.match(line):
+        return dict(stage=match.group('stage').strip(), percent=int(match.group('percent')))
+    return None
+
+
+@dataclass
+class PreparedRepo:
+    repository_id: int
+    url: str
+    directory: pathlib.Path
+    mode: str
+    branch: Optional[str]
+    submodules: bool = False
+    clone_token: Optional[str] = None
+    # The repo's host, resolved once by execute_download; every fetch from it connects to these addresses.
+    addresses: Optional[HostAddresses] = None
+
+
+@dataclass
+class ExecutedRepo:
+    repository_id: int
+    default_branch: Optional[str]
+    head_sha: str
+    head_date: datetime
+    head_message: str
+    size: int
+    readme_path: Optional[str]
+    readme_text: Optional[str]
+    # The repo was updated, but its submodules were not.
+    submodules_error: Optional[str] = None
+
+
+class GitCancelled(Exception):
+    """A git command was cancelled; carries the cancel_wrapper's DownloadResult."""
+
+    def __init__(self, result: DownloadResult):
+        super().__init__(result.error)
+        self.result = result
+
+
+class GitDownloader(Downloader):
+    """Clones a git repo into its Repo's directory, or updates the existing clone.
+
+    Repos are mirrors: local changes are discarded by every update.  Upstream disappearing never deletes the
+    local copy; the download is deferred with git's error."""
+    name = 'git'
+    pretty_name = 'Git'
+    # Repos are added on the Repos page, not the generic download form.
+    listable = False
+
+    def prepare_download(self, session: Session, download: Download) -> PreparedRepo:
+        if download.collection_id:
+            repo = Repository.get_by_collection_id(session, download.collection_id)
+        else:
+            # The one-time Download of a Repo which is never updated.
+            repo = get_repository_by_url(session, download.url)
+        if not repo:
+            raise DownloadError(f'No Repo for this download, add it on the Repos page: {download.url}')
+        directory = repo.collection.get_or_set_directory(session)
+        if not repo.clone_token:
+            repo.clone_token = new_clone_token()
+        return PreparedRepo(repository_id=repo.id, url=repo.url, directory=directory, mode=repo.mode,
+                            branch=repo.branch, submodules=repo.submodules, clone_token=repo.clone_token)
+
+    async def _git(self, download: Download, ctx: DownloadContext, cwd: pathlib.Path, *args,
+                   config: tuple = (), progress: bool = False, check: bool = True,
+                   prepared: PreparedRepo = None) -> str:
+        """Run a git command, return its output (stdout and stderr).
+
+        Pass `prepared` for a command which connects to the repo's host, so it connects to the resolved addresses."""
+        if prepared and prepared.addresses and (pin := pin_config(prepared.addresses)):
+            config = (pin, *config)
+        cmd = git_command(*args, config=config)
+        stdout_callback = make_progress_callback(ctx.report_progress, parse_git_progress) if progress else None
+        result = await self.process_runner(download, cmd, cwd, env=git_env(), merge_stderr=True,
+                                           stdout_callback=stdout_callback, start_new_session=True, ctx=ctx)
+        if isinstance(result, DownloadResult):
+            raise GitCancelled(result)
+        output = result.stdout.decode(errors='replace').strip()
+        if check and result.return_code != 0:
+            raise DownloadError(f'git {args[0]} failed ({result.return_code}): {output}')
+        return output
+
+    async def _default_branch(self, download: Download, ctx: DownloadContext, prepared: PreparedRepo) -> str:
+        """Ask the remote for its default branch.  This fails quickly when the upstream is gone."""
+        output = await self._git(download, ctx, prepared.directory.parent, 'ls-remote', '--symref', prepared.url,
+                                 'HEAD', prepared=prepared)
+        for line in output.splitlines():
+            if line.startswith('ref: refs/heads/') and line.endswith('\tHEAD'):
+                return line[len('ref: refs/heads/'):-len('\tHEAD')]
+        raise DownloadError(f'Repo has no default branch (is it empty?): {prepared.url}')
+
+    async def _clone(self, download: Download, ctx: DownloadContext, prepared: PreparedRepo, branch: str):
+        directory = prepared.directory
+        if directory.is_dir() and any(directory.iterdir()):
+            raise DownloadError(f'Refusing to clone into a directory which is not empty: {directory}')
+        directory.mkdir(parents=True, exist_ok=True)
+
+        args = ['clone', '--progress', '--branch', branch]
+        if prepared.mode == 'snapshot':
+            args.extend(['--depth', '1', '--single-branch', '--no-tags'])
+        args.extend(['--', prepared.url, str(directory)])
+        await self._git(download, ctx, directory.parent, *args, progress=True, prepared=prepared)
+        mark_clone(directory, prepared.clone_token)
+
+    async def _update(self, download: Download, ctx: DownloadContext, prepared: PreparedRepo, branch: str):
+        directory = prepared.directory
+        git = lambda *args, **kwargs: self._git(download, ctx, directory, *args, prepared=prepared,  # noqa: E731
+                                                **kwargs)
+
+        # Only a clone WROLPi made (or imported) for this Repo is reset and cleaned; its config cannot change what
+        # git runs or where it connects, and it is fetched from the Repo's URL, whatever its origin says.
+        if error := clone_ownership_error(directory, prepared.clone_token):
+            raise DownloadError(error)
+        try:
+            await asyncio.to_thread(sanitize_git_config, directory)
+        except ValueError as e:
+            raise DownloadError(str(e))
+        await git('config', '--replace-all', 'remote.origin.url', prepared.url)
+        refspec = FULL_REFSPEC if prepared.mode == 'full' else f'+refs/heads/{branch}:refs/remotes/origin/{branch}'
+        await git('config', '--replace-all', 'remote.origin.fetch', refspec)
+
+        old_head = await git('rev-parse', '--verify', '--quiet', 'HEAD', check=False)
+        if prepared.mode == 'snapshot':
+            await git('fetch', '--progress', '--depth', '1', '--no-tags', 'origin', branch, progress=True)
+            target = 'FETCH_HEAD'
+        else:
+            # Fetch every branch and tag.  Branches deleted upstream are kept (no --prune).
+            args = ['fetch', '--progress', '--tags', '--force', 'origin']
+            if (directory / '.git/shallow').exists():
+                # This was a snapshot; get the whole history.
+                args.insert(1, '--unshallow')
+            await git(*args, progress=True)
+            target = f'refs/remotes/origin/{branch}'
+
+        new_head = await git('rev-parse', '--verify', target)
+        if prepared.mode == 'full' and old_head and old_head != new_head:
+            if not await self._is_ancestor(download, ctx, directory, old_head, new_head):
+                # Upstream rewrote history (force push).  Keep what we had.
+                backup = f'refs/wrolpi/backup/{now().strftime("%Y%m%d%H%M%S")}'
+                await git('update-ref', backup, old_head)
+                logger.warning(f'Repo {prepared.url} history was rewritten; saved old HEAD as {backup}')
+
+        await git('checkout', '--force', '-B', branch, new_head)
+        await git('clean', '-ffdx')
+        if prepared.mode == 'snapshot':
+            # Shallow fetches leave the previous commits behind; drop them.
+            await git('gc', '--prune=now', '--quiet',
+                      config=('gc.reflogExpire=now', 'gc.reflogExpireUnreachable=now'))
+
+    async def _update_submodules(self, download: Download, ctx: DownloadContext, prepared: PreparedRepo) \
+            -> Optional[str]:
+        """Check out (or remove) the submodules of the checked out commit.  Returns an error, if any.
+
+        Submodules are fetched with the same restrictions as the repo (`git_command` options are inherited by the
+        git commands that git runs)."""
+        directory = prepared.directory
+        git = lambda *args, **kwargs: self._git(download, ctx, directory, *args, **kwargs)  # noqa: E731
+        # Every submodule command uses the git directory a submodule's `.git` names; it must be one that was checked.
+        if (error := await asyncio.to_thread(working_tree_git_error, directory)) and (
+                prepared.submodules or (directory / '.git/modules').is_dir()):
+            return error
+        if not prepared.submodules:
+            if (directory / '.git/modules').is_dir():
+                # Submodules were turned off; the repo is a mirror of only its own files now.
+                await git('submodule', 'deinit', '--all', '--force', check=False)
+            return None
+        if not (directory / '.gitmodules').is_file():
+            return None
+
+        try:
+            await self._check_out_submodules(download, ctx, directory, prepared)
+        except DownloadError as e:
+            logger.warning(f'Failed to update submodules of {prepared.url}', exc_info=e)
+            return str(e)
+        return None
+
+    async def _check_out_submodules(self, download: Download, ctx: DownloadContext, directory: pathlib.Path,
+                                    prepared: PreparedRepo, depth: int = 0):
+        """Check out the submodules of the repo in `directory`, then theirs, one level at a time.
+
+        Each level's URLs are checked before git fetches them (git's `--recursive` would fetch nested submodules
+        unchecked)."""
+        if not (directory / '.gitmodules').is_file():
+            return
+        if depth >= MAXIMUM_SUBMODULE_DEPTH:
+            raise DownloadError(f'Submodules are nested more than {MAXIMUM_SUBMODULE_DEPTH} levels deep')
+        git = lambda *args, **kwargs: self._git(download, ctx, directory, *args, prepared=prepared,  # noqa: E731
+                                                **kwargs)
+
+        paths = await self._git_config_values(download, ctx, directory, '.path', '--file', '.gitmodules')
+
+        # git resolves each submodule's URL (a relative URL is resolved against this repo's remote, and may be
+        # another host) into this repo's config.  Those are the URLs git fetches, so those are checked.
+        await git('submodule', 'init')
+        await git('submodule', 'sync')
+        urls = await self._git_config_values(download, ctx, directory, '.url', '--local')
+        try:
+            fetch_config = await asyncio.to_thread(submodule_fetch_config, urls, prepared.addresses)
+        except ValueError as e:
+            raise DownloadError(str(e))
+
+        args = ['submodule', 'update', '--force', '--progress']
+        if prepared.mode == 'snapshot':
+            args.extend(['--depth', '1'])
+        await git(*args, progress=True, config=fetch_config)
+
+        for path in paths:
+            submodule = (directory / path).resolve()
+            if submodule.is_dir() and directory.resolve() in submodule.parents:
+                await self._check_out_submodules(download, ctx, submodule, prepared, depth + 1)
+
+    async def _git_config_values(self, download: Download, ctx: DownloadContext, directory: pathlib.Path,
+                                 suffix: str, *source: str) -> list:
+        """The values of the `submodule.<name><suffix>` keys in a git config.  Refuses anything it cannot read."""
+        # NUL-delimited: a submodule's name and URL can contain spaces and newlines.  Each record is "key\nvalue".
+        output = await self._git(download, ctx, directory, 'config', *source, '-z', '--get-regexp',
+                                 rf'^submodule\..*\{suffix}$', check=False)
+        values = []
+        for record in output.split('\0'):
+            if not record:
+                continue
+            key, newline, value = record.partition('\n')
+            if not newline or not key.startswith('submodule.') or not key.endswith(suffix):
+                raise DownloadError(f'Refusing to download submodules, cannot read their config: {record!r}')
+            values.append(value)
+        return values
+
+    async def _is_ancestor(self, download: Download, ctx: DownloadContext, directory: pathlib.Path,
+                           ancestor: str, descendant: str) -> bool:
+        cmd = git_command('merge-base', '--is-ancestor', ancestor, descendant)
+        result = await self.process_runner(download, cmd, directory, env=git_env(), merge_stderr=True, ctx=ctx)
+        if isinstance(result, DownloadResult):
+            raise GitCancelled(result)
+        return result.return_code == 0
+
+    async def execute_download(self, prepared: PreparedRepo, ctx: DownloadContext,
+                               download: Download = None) -> ExecutedRepo | DownloadResult:
+        download = download if download is not None else Download(url=prepared.url)
+        directory = prepared.directory
+        if error := repo_directory_error(directory):
+            raise DownloadError(error)
+        directory.parent.mkdir(parents=True, exist_ok=True)
+
+        # Resolved once; the repo and its same-host submodules are all fetched from these addresses, so a DNS
+        # answer cannot change in between.
+        prepared.addresses = await asyncio.to_thread(resolve_repo_addresses, prepared.url)
+        try:
+            default_branch = await self._default_branch(download, ctx, prepared)
+            branch = prepared.branch or default_branch
+            # The remote chooses its default branch's name, and branches are passed to git as arguments.
+            if not is_valid_branch(branch):
+                raise DownloadError(f'Invalid branch name: {branch!r}')
+            if (directory / '.git').exists() or (directory / '.git').is_symlink():
+                await self._update(download, ctx, prepared, branch)
+            else:
+                await self._clone(download, ctx, prepared, branch)
+            submodules_error = await self._update_submodules(download, ctx, prepared)
+
+            head = await self._git(download, ctx, directory, 'log', '-1', '--format=%H%x00%cI%x00%s')
+        except GitCancelled as e:
+            return e.result
+        finally:
+            ctx.clear_progress()
+
+        head_sha, head_date, head_message = head.split('\x00', 2)
+        size = await asyncio.to_thread(get_repo_size, directory)
+        readme_path = find_readme(directory)
+        readme_text = await asyncio.to_thread(read_readme_text, directory / readme_path) if readme_path else None
+        return ExecutedRepo(
+            repository_id=prepared.repository_id,
+            default_branch=default_branch,
+            head_sha=head_sha,
+            head_date=datetime.fromisoformat(head_date),
+            head_message=head_message,
+            size=size,
+            readme_path=readme_path,
+            readme_text=readme_text,
+            submodules_error=submodules_error,
+        )
+
+    def finalize_download(self, session: Session, download: Download, executed: ExecutedRepo) -> DownloadResult:
+        repo = session.query(Repository).filter_by(id=executed.repository_id).one_or_none()
+        if not repo:
+            return DownloadResult(success=False, error='Repo was deleted while downloading')
+        repo.default_branch = executed.default_branch
+        repo.head_sha = executed.head_sha
+        repo.head_date = executed.head_date
+        repo.head_message = executed.head_message
+        repo.size = executed.size
+        repo.readme_path = executed.readme_path
+        if repo.readme_text != executed.readme_text:
+            # Only write a changed README; every write re-indexes it.
+            repo.readme_text = executed.readme_text
+        repo.last_fetch = now()
+        logger.info(f'Updated repo {repo.url} to {executed.head_sha}')
+        if executed.submodules_error:
+            # The repo is current, but the download is retried so its submodules are too.
+            return DownloadResult(success=False, location=repo.location,
+                                  error='The repo was updated, but its submodules were not: '
+                                        f'{executed.submodules_error}')
+        return DownloadResult(success=True, location=repo.location)
+
+
+git_downloader = GitDownloader()

modules/repos/errors.py

@@ -0,0 +1,21 @@
+from http import HTTPStatus
+
+from wrolpi.errors import APIError
+
+
+class UnknownRepo(APIError):
+    code = 'UNKNOWN_REPO'
+    summary = 'Unable to find the repo'
+    status_code = HTTPStatus.NOT_FOUND
+
+
+class InvalidRepo(APIError):
+    code = 'INVALID_REPO'
+    summary = 'The repo is invalid'
+    status_code = HTTPStatus.BAD_REQUEST
+
+
+class RepoConflict(APIError):
+    code = 'REPO_CONFLICT'
+    summary = 'The repo conflicts with an existing repo'
+    status_code = HTTPStatus.CONFLICT

modules/repos/models.py

@@ -0,0 +1,146 @@
+import pathlib
+from typing import Optional
+
+from sqlalchemy import Column, Integer, String, ForeignKey, BigInteger, Text, Boolean, false
+from sqlalchemy.orm import relationship, Session, joinedload, selectinload, deferred
+
+from wrolpi.collections.models import Collection
+from wrolpi.common import Base, ModelHelper, get_relative_to_media_directory
+from wrolpi.dates import TZDateTime
+from wrolpi.db import serializer
+from wrolpi.downloader import Download
+from .errors import UnknownRepo
+
+__all__ = ['Repository', 'REPO_MODES']
+
+# full: every branch and tag, with all history.  snapshot: only the latest commit of one branch.
+REPO_MODES = ('full', 'snapshot')
+
+
+class Repository(ModelHelper, Base):
+    """A git repository mirrored into a `kind='repo'` Collection's directory.
+
+    The name, directory and Tag live on the Collection; the Collection's recurring Download keeps the mirror
+    up to date."""
+    __tablename__ = 'repository'
+    id = Column(Integer, primary_key=True)
+    # The https clone URL; unique (compared without a trailing `.git`, see `repo_url_key`).
+    url = Column(String, nullable=False, unique=True)
+    host = Column(String)
+    owner = Column(String)
+    mode = Column(String, nullable=False, default='full')
+    # The branch to check out; None follows the remote's default branch.
+    branch = Column(String)
+    # Random; written in the clone's .git (see lib.mark_clone).  Only a clone holding it is updated, so a clone which
+    # WROLPi did not make for this Repo is never reset or cleaned.
+    clone_token = Column(String)
+    # Also download the repo's submodules.
+    submodules = Column(Boolean, nullable=False, default=False, server_default=false())
+    default_branch = Column(String)
+    head_sha = Column(String)
+    head_date = Column(TZDateTime)
+    head_message = Column(Text)
+    last_fetch = Column(TZDateTime)
+    size = Column(BigInteger)
+    # The README in the repo's root directory, relative to the repo.
+    readme_path = Column(String)
+    # Searched by `repository_fts` (see wrolpi.fts); kept current by `update_search_name` and the downloader.
+    search_name = Column(String)
+    readme_text = deferred(Column(Text))
+
+    collection_id = Column(Integer, ForeignKey('collection.id', ondelete='CASCADE'), nullable=False, unique=True)
+    # Joined: a Repository's name, directory and Tag live on its Collection.
+    collection = relationship('Collection', foreign_keys=[collection_id], lazy='joined')
+    # The Download of this Repo's URL: its recurring Download, or the one-time Download of a Repo which is never
+    # updated (which cannot belong to its Collection).
+    url_download = relationship('Download', primaryjoin='Repository.url == foreign(Download.url)', viewonly=True,
+                                uselist=False)
+
+    def __repr__(self):
+        return f'<Repository id={self.id} url={self.url} directory={self.directory}>'
+
+    @property
+    def name(self) -> Optional[str]:
+        return self.collection.name if self.collection else None
+
+    @property
+    def directory(self) -> Optional[pathlib.Path]:
+        return self.collection.directory if self.collection else None
+
+    @property
+    def tag_name(self) -> Optional[str]:
+        return self.collection.tag_name if self.collection else None
+
+    @property
+    def download(self) -> Optional[Download]:
+        """The recurring Download which keeps this repo up to date."""
+        downloads = [i for i in self.collection.downloads if i.frequency] if self.collection else []
+        return downloads[0] if downloads else None
+
+    @property
+    def once_download(self) -> Optional[Download]:
+        """The one-time Download which clones (or updates, on request) a Repo which is never updated."""
+        download = self.url_download
+        return download if download and not download.frequency else None
+
+    def update_search_name(self):
+        """The Repo is found by its name and its owner."""
+        self.search_name = ' '.join(i for i in (self.name, self.owner) if i)
+
+    @property
+    def location(self) -> str:
+        return f'/repos/{self.id}'
+
+    @staticmethod
+    def json_options() -> tuple:
+        """Loader options for a query whose Repositories will be serialized."""
+        return (
+            joinedload(Repository.collection).joinedload(Collection.tag),
+            joinedload(Repository.collection).selectinload(Collection.downloads).defer(Download.info_json),
+            selectinload(Repository.url_download).defer(Download.info_json),
+        )
+
+    @serializer
+    def __json__(self) -> dict:
+        download = self.download
+        # The status of the last update; a Repo which is never updated is only updated on request.
+        latest = download or self.once_download
+        return {
+            'id': self.id,
+            'collection_id': self.collection_id,
+            'name': self.name,
+            'description': self.collection.description if self.collection else None,
+            'directory': str(get_relative_to_media_directory(self.directory)) if self.directory else None,
+            'tag_name': self.tag_name,
+            'url': self.url,
+            'host': self.host,
+            'owner': self.owner,
+            'mode': self.mode,
+            'branch': self.branch,
+            'submodules': self.submodules,
+            'default_branch': self.default_branch,
+            'head_sha': self.head_sha,
+            'head_date': self.head_date,
+            'head_message': self.head_message,
+            'last_fetch': self.last_fetch,
+            'size': self.size,
+            'readme_path': self.readme_path,
+            'location': self.location,
+            # 0: never updated.
+            'frequency': download.frequency if download else 0,
+            'download_id': latest.id if latest else None,
+            'download_status': latest.status if latest else None,
+            'download_error': latest.error if latest else None,
+            'next_download': download.next_download if download else None,
+        }
+
+    @staticmethod
+    def find_by_id(session: Session, id_: int, options: tuple = ()) -> 'Repository':
+        repo = session.query(Repository).options(*options).filter_by(id=id_).one_or_none()
+        if not repo:
+            raise UnknownRepo(f'Cannot find repo with id {id_}')
+        return repo
+
+    @staticmethod
+    def get_by_collection_id(session: Session, collection_id: int) -> Optional['Repository']:
+        return session.query(Repository).filter_by(collection_id=collection_id).one_or_none()

modules/repos/schema.py

@@ -0,0 +1,55 @@
+from dataclasses import dataclass, field
+from typing import Optional, List
+
+
+@dataclass
+class RepoCreateRequest:
+    url: str
+    # Relative to the media directory; empty clones it into the Repos directory.
+    directory: Optional[str] = None
+    tag_name: Optional[str] = None
+    frequency: Optional[int] = None
+    mode: Optional[str] = None
+    branch: Optional[str] = None
+    name: Optional[str] = None
+    description: Optional[str] = None
+    submodules: bool = False
+
+
+@dataclass
+class RepoSearchRequest:
+    search_str: Optional[str] = None
+    tag_names: List[str] = field(default_factory=list)
+    limit: int = 20
+    offset: int = 0
+
+
+@dataclass
+class RepoUpdateRequest:
+    description: Optional[str] = None
+    frequency: Optional[int] = None
+    mode: Optional[str] = None
+    branch: Optional[str] = None
+    submodules: Optional[bool] = None
+
+
+@dataclass
+class RepoImportInspectRequest:
+    directory: str
+    name: Optional[str] = None
+    tag_name: Optional[str] = None
+    url: Optional[str] = None
+
+
+@dataclass
+class RepoImportRequest:
+    directory: str
+    confirm: bool = False
+    url: Optional[str] = None
+    tag_name: Optional[str] = None
+    frequency: Optional[int] = None
+    mode: Optional[str] = None
+    branch: Optional[str] = None
+    submodules: bool = False
+    name: Optional[str] = None
+    description: Optional[str] = None

modules/repos/test/test_api.py

@@ -0,0 +1,235 @@
+import json
+from http import HTTPStatus
+
+import pytest
+
+from modules.repos import lib
+from modules.repos.models import Repository
+from wrolpi.conftest import await_file_worker
+from wrolpi.downloader import Download, DownloadFrequency, DownloadStatus
+from wrolpi.files.models import FileGroup
+
+
+@pytest.mark.asyncio
+async def test_repos_crud(test_session, test_directory, async_client, test_wrolpi_config):
+    body = dict(url='https://github.com/kiwix/kiwix-tools', tag_name='software', frequency=DownloadFrequency.daily)
+    request, response = await async_client.post('/api/repos', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.CREATED, response.json
+    repo = response.json['repo']
+    assert repo['name'] == 'kiwix-tools'
+    assert repo['tag_name'] == 'software'
+    assert repo['directory'] == 'repos/software/kiwix-tools'
+    assert repo['mode'] == 'full'
+    assert repo['frequency'] == DownloadFrequency.daily
+    assert repo['download_status'] == DownloadStatus.new
+    assert repo['location'] == f'/repos/{repo["id"]}'
+    repo_id = repo['id']
+
+    request, response = await async_client.post('/api/repos', content=json.dumps(dict(url='https://github.com/a/b')))
+    assert response.status_code == HTTPStatus.CREATED
+
+    request, response = await async_client.get('/api/repos')
+    assert response.status_code == HTTPStatus.OK
+    assert [i['name'] for i in response.json['repos']] == ['b', 'kiwix-tools']
+
+    body = dict(description='Kiwix tools', mode='snapshot', branch='dev', frequency=DownloadFrequency.weekly)
+    request, response = await async_client.put(f'/api/repos/{repo_id}', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.OK, response.json
+    request, response = await async_client.get(f'/api/repos/{repo_id}')
+    repo = response.json['repo']
+    assert (repo['description'], repo['mode'], repo['branch'], repo['frequency']) == \
+           ('Kiwix tools', 'snapshot', 'dev', DownloadFrequency.weekly)
+
+    request, response = await async_client.delete(f'/api/repos/{repo_id}')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    request, response = await async_client.get(f'/api/repos/{repo_id}')
+    assert response.status_code == HTTPStatus.NOT_FOUND
+    assert response.json['code'] == 'UNKNOWN_REPO'
+
+
+@pytest.mark.asyncio
+async def test_create_repo_errors(test_session, async_client, test_wrolpi_config):
+    request, response = await async_client.post('/api/repos', content=json.dumps(dict(url='git@github.com:a/b')))
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+    assert response.json['code'] == 'INVALID_REPO'
+
+    request, response = await async_client.post('/api/repos', content=json.dumps(dict(url='https://github.com/a/b')))
+    assert response.status_code == HTTPStatus.CREATED
+    request, response = await async_client.post('/api/repos',
+                                                content=json.dumps(dict(url='https://github.com/a/b.git')))
+    assert response.status_code == HTTPStatus.CONFLICT
+    assert response.json['code'] == 'REPO_CONFLICT'
+
+
+@pytest.mark.asyncio
+async def test_repo_tree_and_update(test_session, async_client, git_remote_factory, repos_download_manager):
+    remote = git_remote_factory(files={'README.md': '# Example', 'docs/guide.md': 'guide'})
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/tree')
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['readme_path'] == 'README.md'
+    assert [i['path'] for i in response.json['entries']] == ['docs', 'README.md']
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/tree?path=docs')
+    assert [i['path'] for i in response.json['entries']] == ['docs/guide.md']
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/tree?path=../..')
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+
+    # "Update now" renews the download.
+    new_head = remote.commit({'new.txt': 'new'}, 'New')
+    request, response = await async_client.post(f'/api/repos/{repo.id}/update')
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    assert test_session.query(Repository).one().head_sha == new_head
+
+
+@pytest.mark.asyncio
+async def test_tag_repo_moves_clone(test_session, test_directory, async_client, git_remote_factory,
+                                    repos_download_manager, await_background_tasks):
+    """Tagging a Repo (through the collections API) moves the whole clone, including `.git`.  Its files are not
+    indexed (repos/ is ignored), even ones the file mover could not group."""
+    remote = git_remote_factory(files={'README.md': '# Example', 'web/.npmrc': 'a', 'web/.gitignore': 'b'})
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    old_directory = test_directory / 'repos/example'
+    assert (old_directory / '.git').is_dir()
+
+    body = dict(tag_name='software', directory='repos/software/example')
+    request, response = await async_client.post(f'/api/collections/{repo.collection_id}/tag',
+                                                content=json.dumps(body))
+    assert response.status_code == HTTPStatus.OK, response.json
+    await await_background_tasks()
+    test_session.expire_all()
+
+    new_directory = test_directory / 'repos/software/example'
+    assert repo.directory == new_directory
+    assert repo.tag_name == 'software'
+    assert (new_directory / '.git').is_dir()
+    assert (new_directory / 'README.md').is_file()
+    assert (new_directory / 'web/.npmrc').is_file()
+    assert not old_directory.exists()
+    assert test_session.query(Download).one().destination == new_directory
+    assert test_session.query(FileGroup).count() == 0
+
+    # The moved clone still updates.
+    new_head = remote.commit({'new.txt': 'new'}, 'New')
+    download = test_session.query(Download).one()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    assert repo.head_sha == new_head
+    assert (new_directory / 'new.txt').is_file()
+
+
+@pytest.mark.asyncio
+async def test_tag_repo_in_custom_directory(test_session, test_directory, async_client, git_remote_factory,
+                                            repos_download_manager, await_background_tasks, refresh_files):
+    """A Repo in a directory which is not ignored is indexed where it is moved to, and not where it was."""
+    remote = git_remote_factory(files={'README.md': '# Example', 'guide.md': 'A guide'})
+    repo = lib.create_repository(test_session, remote.url, directory='projects/example')
+    await repos_download_manager.wait_for_all_downloads()
+    await refresh_files()
+    old_directory = test_directory / 'projects/example'
+    assert test_session.query(FileGroup).filter(FileGroup.primary_path == str(old_directory / 'guide.md')).count()
+
+    body = dict(tag_name='software', directory='projects/moved')
+    request, response = await async_client.post(f'/api/collections/{repo.collection_id}/tag',
+                                                content=json.dumps(body))
+    assert response.status_code == HTTPStatus.OK, response.json
+    await await_background_tasks()
+    await await_file_worker()
+    test_session.expire_all()
+
+    new_directory = test_directory / 'projects/moved'
+    assert repo.directory == new_directory
+    assert (new_directory / '.git').is_dir()
+    assert not old_directory.exists()
+    paths = {i.primary_path for i in test_session.query(FileGroup)}
+    assert new_directory / 'guide.md' in paths
+    assert not [i for i in paths if old_directory in i.parents]
+
+
+@pytest.mark.parametrize('directory', ['../escape/example', 'repos/../../escape/example', '{media}/../escape/example',
+                                       'linked/example'])
+@pytest.mark.asyncio
+async def test_tag_repo_cannot_leave_media_directory(test_session, test_directory, async_client, git_remote_factory,
+                                                     repos_download_manager, await_background_tasks,
+                                                     tmp_path_factory, directory):
+    """Tagging cannot move a Repo's clone out of the media directory, by `..` or through a link."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    outside = tmp_path_factory.mktemp('outside')
+    (test_directory / 'linked').symlink_to(outside)
+
+    body = dict(tag_name='software', directory=directory.format(media=test_directory))
+    request, response = await async_client.post(f'/api/collections/{repo.collection_id}/tag',
+                                                content=json.dumps(body))
+    await await_background_tasks()
+    test_session.expire_all()
+
+    assert (test_directory / 'repos/example/.git').is_dir()
+    assert repo.directory == test_directory / 'repos/example'
+    assert not (test_directory.parent / 'escape').exists()
+    assert list(outside.iterdir()) == []
+
+
+@pytest.mark.asyncio
+async def test_repos_destination_setting(async_client, test_wrolpi_config):
+    request, response = await async_client.get('/api/settings')
+    assert response.json['repos_destination'] == 'repos/%(repo_tag)s/%(repo_name)s'
+
+    body = dict(repos_destination='/repos')
+    request, response = await async_client.patch('/api/settings', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+
+    body = dict(repos_destination='repos/%(repo_owner)s/%(repo_name)s')
+    request, response = await async_client.patch('/api/settings', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    request, response = await async_client.get('/api/settings')
+    assert response.json['repos_destination'] == 'repos/%(repo_owner)s/%(repo_name)s'
+
+
+@pytest.mark.asyncio
+async def test_repo_changes_wait_for_config_import(test_session, test_directory, async_client, git_remote_factory,
+                                                   repos_download_manager, test_wrolpi_config):
+    """Until repos.yaml is imported (e.g. while WROLPi starts), Repos cannot change; the import would undo it."""
+    remote = git_remote_factory()
+    other = git_remote_factory('other')
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    config = lib.get_repos_config()
+    config.dump_config()
+    assert config.get_file().is_file()
+    config.successful_import = False
+
+    requests = [
+        ('post', '/api/repos', dict(url=other.url)),
+        ('post', '/api/repos/import', dict(directory='repos/example', confirm=True)),
+        ('put', f'/api/repos/{repo.id}', dict(description='changed')),
+        ('delete', f'/api/repos/{repo.id}', None),
+        ('post', f'/api/collections/{repo.collection_id}/tag', dict(tag_name='software',
+                                                                     directory='repos/software/example')),
+        ('put', f'/api/collections/{repo.collection_id}', dict(description='changed')),
+        ('delete', f'/api/collections/{repo.collection_id}', None),
+    ]
+    for method, url, body in requests:
+        kwargs = dict(content=json.dumps(body)) if body else {}
+        request, response = await getattr(async_client, method)(url, **kwargs)
+        assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE, (url, response.json)
+        assert response.json['code'] == 'CONFIG_NOT_IMPORTED'
+    test_session.expire_all()
+    assert test_session.query(Repository).count() == 1
+    assert repo.collection.description is None
+    assert repo.tag_name is None
+    assert (test_directory / 'repos/example/.git').is_dir()
+
+    # Once imported, Repos can change again.
+    lib.import_repos_config()
+    request, response = await async_client.put(f'/api/repos/{repo.id}', content=json.dumps(dict(description='x')))
+    assert response.status_code == HTTPStatus.OK, response.json

modules/repos/test/test_config.py

@@ -0,0 +1,91 @@
+import pytest
+
+from modules.repos import lib
+from modules.repos.models import Repository
+from wrolpi.collections.models import Collection
+from wrolpi.downloader import Download, DownloadFrequency
+
+
+@pytest.mark.asyncio
+async def test_dump_and_import(test_session, test_directory, async_client, test_wrolpi_config, await_switches):
+    """repos.yaml is the source of truth; a lost database is rebuilt from it."""
+    lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software',
+                          description='Tools', frequency=DownloadFrequency.daily)
+    lib.create_repository(test_session, 'https://github.com/a/b', mode='snapshot', branch='dev')
+    await await_switches()
+
+    config = lib.get_repos_config()
+    assert config.get_file().is_file()
+    # Each Repo's random clone token is kept, so a restored Repo still owns its clone.
+    dumped = [dict(i) for i in config.repos]
+    assert all(len(i.pop('clone_token')) == 32 for i in dumped)
+    assert dumped == [
+        dict(name='b', url='https://github.com/a/b', directory='repos/b', tag_name=None, description=None,
+             mode='snapshot', branch='dev', submodules=False, frequency=DownloadFrequency.weekly),
+        dict(name='kiwix-tools', url='https://github.com/kiwix/kiwix-tools', directory='repos/software/kiwix-tools',
+             tag_name='software', description='Tools', mode='full', branch=None, submodules=False,
+             frequency=DownloadFrequency.daily),
+    ]
+
+    # Lose the database.
+    for repo in test_session.query(Repository).all():
+        lib._delete_repository(test_session, repo)
+    test_session.commit()
+    assert test_session.query(Collection).count() == 0
+
+    config.import_config()
+    test_session.expire_all()
+    repos = {i.name: i for i in test_session.query(Repository)}
+    assert repos['kiwix-tools'].directory == test_directory / 'repos/software/kiwix-tools'
+    assert repos['kiwix-tools'].tag_name == 'software'
+    assert repos['kiwix-tools'].collection.description == 'Tools'
+    assert repos['kiwix-tools'].download.frequency == DownloadFrequency.daily
+    assert (repos['b'].mode, repos['b'].branch) == ('snapshot', 'dev')
+    assert {i.downloader for i in test_session.query(Download)} == {'git'}
+
+    # A repo removed from the config is deleted.
+    data = config.read_config_file()
+    data['repos'] = data['repos'][1:]
+    data['version'] += 1
+    config.write_config_data(data, config.get_file())
+    config.import_config()
+    test_session.expire_all()
+    assert [i.name for i in test_session.query(Repository)] == ['kiwix-tools']
+    assert test_session.query(Download).count() == 1
+
+
+@pytest.mark.asyncio
+async def test_import_links_existing_download(test_session, async_client, test_wrolpi_config):
+    """A Repo imported after download_manager.yaml re-uses its Download."""
+    lib.create_repository(test_session, 'https://github.com/a/b')
+    test_session.query(Repository).delete()
+    test_session.query(Collection).delete()
+    test_session.commit()
+    download = test_session.query(Download).one()
+    assert download.collection_id is None
+
+    config = lib.get_repos_config()
+    config.write_config_data(dict(version=1, repos=[dict(name='b', url='https://github.com/a/b', frequency=3600)]),
+                             config.get_file())
+    config.import_config()
+    test_session.expire_all()
+
+    repo = test_session.query(Repository).one()
+    assert test_session.query(Download).one().id == download.id
+    assert download.collection_id == repo.collection_id
+    assert download.frequency == 3600
+
+
+@pytest.mark.asyncio
+async def test_delete_last_repo(test_session, async_client, test_wrolpi_config, await_switches):
+    """Deleting the last Repo empties the config, so it does not return on the next import."""
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    await await_switches()
+    config = lib.get_repos_config()
+    assert len(config.read_config_file()['repos']) == 1
+
+    request, response = await async_client.delete(f'/api/repos/{repo.id}')
+    assert config.read_config_file()['repos'] == []
+
+    config.import_config()
+    assert test_session.query(Repository).count() == 0

modules/repos/test/test_downloader.py

@@ -0,0 +1,226 @@
+import shutil
+import subprocess
+
+import pytest
+from mock import mock
+
+from modules.repos import lib
+from modules.repos.downloader import parse_git_progress
+from modules.repos.models import Repository
+from wrolpi.downloader import Download, DownloadStatus
+
+
+def git(directory, *args) -> str:
+    return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True,
+                          capture_output=True).stdout.decode().strip()
+
+
+@pytest.mark.asyncio
+async def test_clone_full(test_session, test_directory, git_remote_factory, repos_download_manager):
+    """A new Repo is cloned with its whole history, and the Repo describes the checked out commit."""
+    remote = git_remote_factory('kiwix-tools', owner='kiwix', files={'README.md': '# Kiwix\n', 'src/main.c': 'int'})
+    remote.commit({'CHANGELOG': 'v2'}, 'Second commit')
+    remote.git('tag', 'v2')
+
+    repo = lib.create_repository(test_session, remote.url, tag_name='software')
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    directory = test_directory / 'repos/software/kiwix-tools'
+    assert repo.directory == directory
+    assert (directory / 'README.md').read_text() == '# Kiwix\n'
+    assert (directory / 'src/main.c').is_file()
+    # Full mode has all the history and tags.
+    assert git(directory, 'rev-list', '--count', 'HEAD') == '2'
+    assert git(directory, 'tag') == 'v2'
+    assert not (directory / '.git/shallow').exists()
+
+    assert repo.head_sha == remote.head
+    assert repo.head_message == 'Second commit'
+    assert repo.head_date is not None
+    assert repo.default_branch == 'main'
+    assert repo.readme_path == 'README.md'
+    assert repo.size > 0
+    assert repo.last_fetch is not None
+
+    download, = test_session.query(Download).all()
+    assert download.status == DownloadStatus.complete
+    assert download.location == f'/repos/{repo.id}'
+    assert download.error is None
+
+
+@pytest.mark.asyncio
+async def test_update(test_session, test_directory, git_remote_factory, repos_download_manager):
+    """An update fetches new commits, and discards local changes (a Repo is a mirror)."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    directory = repo.directory
+
+    # Local changes.
+    (directory / 'README.md').write_text('local change')
+    (directory / 'untracked.txt').write_text('untracked')
+
+    new_head = remote.commit({'new.txt': 'new'}, 'New file')
+    download, = test_session.query(Download).all()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    assert (directory / 'new.txt').read_text() == 'new'
+    assert (directory / 'README.md').read_text() == '# example\n\nThe example repo.\n'
+    assert not (directory / 'untracked.txt').exists()
+    assert repo.head_sha == new_head
+    assert repo.head_message == 'New file'
+
+
+@pytest.mark.asyncio
+async def test_update_keeps_history(test_session, git_remote_factory, repos_download_manager):
+    """Branches deleted upstream are kept, and history rewritten by a force push is saved in a backup ref."""
+    remote = git_remote_factory()
+    remote.git('branch', 'feature')
+    old_head = remote.commit({'a.txt': 'a'}, 'Will be rewritten')
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    directory = repo.directory
+    assert 'origin/feature' in git(directory, 'branch', '--remotes')
+
+    # Upstream deletes a branch, and rewrites main.
+    remote.git('branch', '--delete', 'feature')
+    remote.git('reset', '--hard', 'HEAD~1')
+    new_head = remote.commit({'b.txt': 'b'}, 'Rewritten')
+
+    download, = test_session.query(Download).all()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    assert repo.head_sha == new_head
+    assert 'origin/feature' in git(directory, 'branch', '--remotes')
+    backups = git(directory, 'for-each-ref', '--format=%(objectname)', 'refs/wrolpi/backup/')
+    assert backups == old_head
+
+
+@pytest.mark.asyncio
+async def test_snapshot(test_session, git_remote_factory, repos_download_manager):
+    """A snapshot only has the latest commit; switching to full fetches the whole history."""
+    remote = git_remote_factory()
+    remote.commit({'a.txt': 'a'}, 'Second')
+    repo = lib.create_repository(test_session, remote.url, mode='snapshot')
+    await repos_download_manager.wait_for_all_downloads()
+    directory = repo.directory
+    assert (directory / '.git/shallow').is_file()
+    assert git(directory, 'rev-list', '--count', 'HEAD') == '1'
+
+    # Snapshot updates stay shallow.
+    remote.commit({'b.txt': 'b'}, 'Third')
+    download, = test_session.query(Download).all()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    assert (directory / 'b.txt').is_file()
+    assert git(directory, 'rev-list', '--count', 'HEAD') == '1'
+
+    lib.update_repository(test_session, repo.id, mode='full')
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    assert not (directory / '.git/shallow').exists()
+    assert git(directory, 'rev-list', '--count', 'HEAD') == '3'
+
+
+@pytest.mark.asyncio
+async def test_branch(test_session, git_remote_factory, repos_download_manager):
+    """A Repo can follow a branch other than the default branch."""
+    remote = git_remote_factory()
+    remote.git('checkout', '--quiet', '-b', 'dev')
+    dev_head = remote.commit({'dev.txt': 'dev'}, 'Dev')
+    remote.git('checkout', '--quiet', 'main')
+
+    repo = lib.create_repository(test_session, f'{remote.url}/tree/dev')
+    assert repo.branch == 'dev'
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    assert repo.head_sha == dev_head
+    assert repo.default_branch == 'main'
+    assert git(repo.directory, 'branch', '--show-current') == 'dev'
+
+
+@pytest.mark.asyncio
+async def test_upstream_gone(test_session, git_remote_factory, repos_download_manager):
+    """When the upstream disappears the download is deferred with git's error; the local copy is kept."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    directory = repo.directory
+    head = repo.head_sha
+
+    shutil.rmtree(remote.path)
+    download, = test_session.query(Download).all()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    assert download.status == DownloadStatus.deferred
+    assert 'git ls-remote failed' in download.error
+    assert (directory / 'README.md').is_file()
+    assert repo.head_sha == head
+
+
+@pytest.mark.asyncio
+async def test_symlinks_are_not_followed(test_session, git_remote_factory, repos_download_manager):
+    """A symlink in a repo is checked out as a plain file, so it cannot expose files outside the repo."""
+    remote = git_remote_factory()
+    (remote.path / 'shadow').symlink_to('/etc/passwd')
+    remote.git('add', 'shadow')
+    remote.git('commit', '--quiet', '-m', 'Symlink')
+
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    shadow = repo.directory / 'shadow'
+    assert not shadow.is_symlink()
+    assert shadow.read_text() == '/etc/passwd'
+
+
+@pytest.mark.asyncio
+async def test_refuses_non_empty_directory(test_session, test_directory, git_remote_factory, repos_download_manager):
+    """A Repo is never cloned over existing files."""
+    remote = git_remote_factory()
+    directory = test_directory / 'repos/example'
+    directory.mkdir(parents=True)
+    (directory / 'important.txt').write_text('important')
+
+    lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    download, = test_session.query(Download).all()
+    assert download.status == DownloadStatus.deferred
+    assert 'not empty' in download.error
+    assert (directory / 'important.txt').read_text() == 'important'
+    assert test_session.query(Repository).one().head_sha is None
+
+
+def test_parse_git_progress():
+    assert parse_git_progress('Receiving objects:  45% (123/456), 1.20 MiB | 1.00 MiB/s') == \
+           dict(stage='Receiving objects', percent=45)
+    assert parse_git_progress('remote: Counting objects: 100% (5/5), done.') == \
+           dict(stage='Counting objects', percent=100)
+    assert parse_git_progress("Cloning into 'example'...") is None
+
+
+@pytest.mark.asyncio
+async def test_invalid_remote_default_branch(test_session, git_remote_factory, repos_download_manager):
+    """The remote chooses its default branch name; an invalid one is never passed to git."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    with mock.patch('modules.repos.downloader.GitDownloader._default_branch', return_value='--upload-pack=x'):
+        await repos_download_manager.wait_for_all_downloads()
+
+    download, = test_session.query(Download).all()
+    assert download.status == DownloadStatus.deferred
+    assert 'Invalid branch' in download.error
+    assert not repo.directory.exists()

modules/repos/test/test_extras.py

@@ -0,0 +1,435 @@
+import io
+import subprocess
+import zipfile
+from http import HTTPStatus
+
+import pytest
+from mock import mock
+
+from modules.repos import lib
+from modules.repos.models import Repository
+from wrolpi.downloader import Download, DownloadStatus
+
+# The repo the submodules belong to, and the (public) address its host resolved to.
+REPO = lib.HostAddresses('git.example.com', 443, ['93.184.215.14'])
+
+
+def git(directory, *args) -> str:
+    return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True,
+                          capture_output=True).stdout.decode().strip()
+
+
+async def update(test_session, repos_download_manager):
+    download = test_session.query(Download).one()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    return test_session.query(Download).one()
+
+
+@pytest.mark.asyncio
+async def test_log(test_session, async_client, git_remote_factory, repos_download_manager):
+    """The history of the checked out branch, newest first, a page at a time."""
+    remote = git_remote_factory()
+    second = remote.commit({'a.txt': 'a'}, 'Second')
+    third = remote.commit({'b.txt': 'b'}, 'Third: with a | and a\ttab')
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/log?limit=2')
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['total'] == 3
+    assert [(i['sha'], i['message']) for i in response.json['commits']] == \
+           [(third, 'Third: with a | and a\ttab'), (second, 'Second')]
+    assert response.json['commits'][0]['author'] == 'WROLPi Test'
+    assert response.json['commits'][0]['date']
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/log?limit=2&offset=2')
+    assert [i['message'] for i in response.json['commits']] == ['Initial commit']
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/log?limit=nope')
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+
+
+@pytest.mark.asyncio
+async def test_log_not_downloaded(test_session, async_client, test_wrolpi_config):
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    request, response = await async_client.get(f'/api/repos/{repo.id}/log')
+    assert response.status_code == HTTPStatus.NOT_FOUND
+    request, response = await async_client.get(f'/api/repos/{repo.id}/archive.zip')
+    assert response.status_code == HTTPStatus.NOT_FOUND
+
+
+@pytest.mark.asyncio
+async def test_archive(test_session, async_client, git_remote_factory, repos_download_manager):
+    """A ZIP of the checked out commit; local changes and `.git` are not in it."""
+    remote = git_remote_factory(files={'README.md': '# Example', 'src/main.c': 'int main;'})
+    repo = lib.create_repository(test_session, remote.url, name='my "repo"\r\nX-Evil: 1')
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    (repo.directory / 'untracked.txt').write_text('local')
+
+    request, response = await async_client.get(f'/api/repos/{repo.id}/archive.zip')
+    assert response.status_code == HTTPStatus.OK
+    assert response.headers['content-type'] == 'application/zip'
+    # The name is safe in a header.
+    name = f'my_repo_X-Evil_1-{repo.head_sha[:7]}'
+    assert response.headers['content-disposition'] == f'attachment; filename="{name}.zip"'
+    assert 'x-evil' not in response.headers
+
+    with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
+        assert sorted(archive.namelist()) == [f'{name}/', f'{name}/README.md', f'{name}/src/', f'{name}/src/main.c']
+        assert archive.read(f'{name}/src/main.c') == b'int main;'
+
+
+@pytest.mark.asyncio
+async def test_submodules(test_session, git_remote_factory, repos_download_manager):
+    """Submodules are only downloaded when the Repo asks for them, and are removed when it no longer does."""
+    library = git_remote_factory('library', files={'lib.c': 'v1'})
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library)
+
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    submodule = repo.directory / 'vendor/library'
+    assert not (submodule / 'lib.c').exists()
+
+    lib.update_repository(test_session, repo.id, submodules=True)
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.complete
+    assert (submodule / 'lib.c').read_text() == 'v1'
+    # The submodule has its history, like the Repo.
+    assert git(submodule, 'rev-list', '--count', 'HEAD') == '1'
+
+    # A new submodule commit is checked out when the Repo points at it.
+    library.commit({'lib.c': 'v2'}, 'v2')
+    remote.git('update-index', '--cacheinfo', f'160000,{library.head},vendor/library')
+    remote.git('commit', '--quiet', '-m', 'Bump library')
+    await update(test_session, repos_download_manager)
+    assert (submodule / 'lib.c').read_text() == 'v2'
+
+    lib.update_repository(test_session, repo.id, submodules=False)
+    await update(test_session, repos_download_manager)
+    assert not (submodule / 'lib.c').exists()
+
+
+@pytest.mark.asyncio
+async def test_submodule_failure(test_session, git_remote_factory, repos_download_manager):
+    """A submodule which cannot be downloaded (e.g. an ssh URL) does not stop the Repo from updating."""
+    library = git_remote_factory('library')
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library)
+    # Rewrite the submodule's URL to one git may not use.
+    (remote.path / '.gitmodules').write_text(
+        '[submodule "vendor/library"]\n\tpath = vendor/library\n\turl = ssh://git@git.example.com/owner/library\n')
+    (remote.path / 'new.txt').write_text('new')
+    # Not `remote.commit`, whose `add --all` would delete the submodule (it is not checked out in the remote).
+    remote.git('add', '.gitmodules', 'new.txt')
+    remote.git('commit', '--quiet', '-m', 'ssh submodule')
+    head = remote.head
+
+    repo = lib.create_repository(test_session, remote.url, submodules=True)
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'its submodules were not' in download.error
+    # The Repo itself is current.
+    assert repo.head_sha == head
+    assert (repo.directory / 'new.txt').is_file()
+
+
+@pytest.mark.asyncio
+async def test_submodules_api_and_config(test_session, async_client, test_wrolpi_config, await_switches):
+    body = '{"url": "https://github.com/a/b", "submodules": true}'
+    request, response = await async_client.post('/api/repos', content=body)
+    assert response.status_code == HTTPStatus.CREATED
+    assert response.json['repo']['submodules'] is True
+    repo_id = response.json['repo']['id']
+
+    await await_switches()
+    assert lib.get_repos_config().repos[0]['submodules'] is True
+
+    request, response = await async_client.put(f'/api/repos/{repo_id}', content='{"submodules": false}')
+    assert response.json['repo']['submodules'] is False
+
+    # Restored from the config.
+    config = lib.get_repos_config()
+    data = config.read_config_file()
+    data['repos'][0]['submodules'] = True
+    data['version'] += 1
+    config.write_config_data(data, config.get_file())
+    config.import_config()
+    test_session.expire_all()
+    assert test_session.query(Repository).one().submodules is True
+
+
+@pytest.mark.asyncio
+async def test_archive_after_session_closes(test_session, async_client, git_remote_factory, repos_download_manager):
+    """In production the request's session is closed once a streamed response starts; the ZIP must not need it."""
+    from sanic import Request
+
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    original_respond = Request.respond
+
+    async def respond_and_close_session(self, *args, **kwargs):
+        response = await original_respond(self, *args, **kwargs)
+        test_session.expunge_all()
+        return response
+
+    with mock.patch.object(Request, 'respond', respond_and_close_session):
+        request, response = await async_client.get(f'/api/repos/{repo.id}/archive.zip')
+    assert response.status_code == HTTPStatus.OK
+    with zipfile.ZipFile(io.BytesIO(response.content)) as archive:
+        assert any(i.endswith('/README.md') for i in archive.namelist())
+
+
+@pytest.mark.parametrize('url,resolved,allowed', [
+    # Relative URLs are resolved by git (against the repo's remote) before they are checked.
+    ('../library.git', None, False),
+    ('./library', None, False),
+    ('https://git.example.com/owner/library', None, True),  # The repo's own host.
+    ('https://codeberg.org/owner/library', ['217.197.91.145'], True),
+    ('https://localhost/library', ['127.0.0.1'], False),
+    ('https://127.0.0.1/library', None, False),
+    ('https://10.0.0.5/library', None, False),
+    ('https://192.168.1.1:8443/library', None, False),
+    ('https://[::1]/library', None, False),
+    ('https://169.254.169.254/latest', None, False),
+    ('https://gitea.lan/library', ['192.168.1.20'], False),
+    ('https://mixed.example/library', ['93.184.215.14', '10.0.0.1'], False),
+    ('https://unresolvable.example/library', OSError('no such host'), False),
+    ('http://codeberg.org/owner/library', None, False),
+    ('ssh://git@codeberg.org/owner/library', None, False),
+    ('file:///etc', None, False),
+    # URLs which parsers disagree about are refused.
+    ('https://codeberg.org\\@10.0.0.5/library', ['217.197.91.145'], False),
+    ('https://user@codeberg.org/owner/library', ['217.197.91.145'], False),
+    ('https://10.0.0.5#@codeberg.org/library', ['217.197.91.145'], False),
+    ('https://codeberg.org/owner/library?x=@10.0.0.5', ['217.197.91.145'], False),
+    ('https://codeberg.org/owner/lib rary', ['217.197.91.145'], False),
+    ('https://codeberg.org:443/owner/library', ['217.197.91.145'], True),
+    ('../../evil@10.0.0.5/x', None, False),
+    ('..\\evil', None, False),
+])
+def test_submodule_url_error(url, resolved, allowed):
+    """A submodule URL comes from the (untrusted) repo; it may not make WROLPi fetch from its own network."""
+    side_effect = resolved if isinstance(resolved, Exception) else None
+    with mock.patch('modules.repos.lib.resolve_host_addresses', side_effect=side_effect,
+                    return_value=resolved or []):
+        error = lib.submodule_url_error(url, REPO)
+    assert (error is None) is allowed, error
+
+
+@pytest.mark.asyncio
+async def test_submodule_to_private_address(test_session, git_remote_factory, repos_download_manager):
+    """A submodule which points into WROLPi's network is never fetched; the repo itself still updates."""
+    library = git_remote_factory('library')
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library, url='https://10.0.0.5/admin/library')
+
+    repo = lib.create_repository(test_session, remote.url, submodules=True)
+    with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['93.184.215.14']) as resolve:
+        await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'not a public address' in download.error
+    assert repo.head_sha == remote.head
+    assert not (repo.directory / 'vendor/library/README.md').exists()
+    # Only the repo's own host was resolved; an IP address is checked without DNS.
+    assert [i.args for i in resolve.call_args_list] == [('git.example.com',)]
+
+
+@pytest.mark.asyncio
+async def test_nested_submodule_to_private_address(test_session, git_remote_factory, repos_download_manager):
+    """Each level of submodules is checked before it is fetched."""
+    evil = git_remote_factory('evil')
+    library = git_remote_factory('library')
+    library.add_submodule('nested', evil, url='https://127.0.0.1/evil')
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library)
+
+    repo = lib.create_repository(test_session, remote.url, submodules=True)
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'not a public address' in download.error
+    # The first level (the repo's own host) was fetched; the nested submodule was not.
+    assert (repo.directory / 'vendor/library/README.md').is_file()
+    assert not (repo.directory / 'vendor/library/nested/README.md').exists()
+
+
+@pytest.mark.parametrize('gitmodules', [
+    # A space in the name must not hide the URL.
+    '[submodule "a b"]\n\tpath = vendor/library\n\turl = https://10.0.0.5/library\n',
+    # Nor a newline in the URL.
+    '[submodule "library"]\n\tpath = vendor/library\n\turl = "https://10.0.0.5/library\\nsubmodule.x.path y"\n',
+])
+@pytest.mark.asyncio
+async def test_submodule_url_cannot_hide(test_session, git_remote_factory, repos_download_manager, gitmodules):
+    """Every submodule URL is checked, however the .gitmodules is written."""
+    library = git_remote_factory('library')
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library)
+    (remote.path / '.gitmodules').write_text(gitmodules)
+    remote.git('add', '.gitmodules')
+    remote.git('commit', '--quiet', '-m', 'Tricky .gitmodules')
+
+    repo = lib.create_repository(test_session, remote.url, submodules=True)
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'Refusing' in download.error
+    assert not (repo.directory / 'vendor/library/README.md').exists()
+
+
+
+def test_submodule_fetch_config():
+    """Checked hosts are pinned to the addresses that were checked, so DNS cannot change between the check and git's
+    fetch."""
+    with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['217.197.91.145', '2606:4700::1111']):
+        config = lib.submodule_fetch_config(['https://codeberg.org/a/b', 'https://codeberg.org:8443/c/d',
+                                             'https://git.example.com/own/host'], REPO)
+    assert config == (
+        'http.followRedirects=false',
+        'http.curloptResolve=codeberg.org:443:217.197.91.145,[2606:4700::1111]',
+        'http.curloptResolve=codeberg.org:8443:217.197.91.145,[2606:4700::1111]',
+    )
+
+
+
+@pytest.mark.parametrize('relative_url,allowed', [
+    ('../library', True),  # https://git.example.com/owner/library
+    ('../../../10.0.0.5/library', False),  # git resolves this to https://10.0.0.5/library
+])
+@pytest.mark.asyncio
+async def test_relative_submodule_url(test_session, git_remote_factory, repos_download_manager, relative_url,
+                                      allowed):
+    """A relative submodule URL is checked where git resolves it, which may be another host."""
+    library = git_remote_factory('library')
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library, url=relative_url)
+
+    repo = lib.create_repository(test_session, remote.url, submodules=True)
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    download = test_session.query(Download).one()
+    if allowed:
+        assert download.status == DownloadStatus.complete, download.error
+        assert (repo.directory / 'vendor/library/README.md').is_file()
+    else:
+        assert download.status == DownloadStatus.deferred
+        assert 'Refusing to download submodule https://10.0.0.5/library' in download.error
+        assert not (repo.directory / 'vendor/library/README.md').exists()
+
+
+@pytest.mark.parametrize('address,public', [
+    ('8.8.8.8', True),
+    ('2606:4700::1111', True),
+    ('64:ff9b::808:808', True),  # NAT64 of 8.8.8.8
+    ('2002:808:808::1', True),  # 6to4 of 8.8.8.8
+    ('10.0.0.5', False),
+    ('100.64.0.1', False),  # Carrier-grade NAT
+    ('127.0.0.1', False),
+    ('169.254.169.254', False),
+    ('224.0.0.1', False),  # Multicast
+    ('255.255.255.255', False),
+    ('0.0.0.0', False),
+    ('::1', False),
+    ('::', False),
+    ('fe80::1', False),
+    ('fc00::1', False),
+    ('fec0::1', False),  # Deprecated site-local
+    ('ff02::1', False),  # Multicast
+    ('::ffff:169.254.169.254', False),
+    ('64:ff9b::169.254.169.254', False),  # NAT64 of the metadata address
+    ('64:ff9b::a9fe:a9fe', False),
+    ('64:ff9b:1::808:808', False),  # Local-use NAT64
+    ('2002:a00:5::1', False),  # 6to4 of 10.0.0.5
+    ('2001:0:4136:e378:8000:63bf:3fff:fdd2', False),  # Teredo
+])
+def test_is_public_address(address, public):
+    import ipaddress
+    assert lib.is_public_address(ipaddress.ip_address(address)) is public
+
+
+@pytest.mark.parametrize('url', [
+    'https://[64:ff9b::169.254.169.254]/latest/meta-data',
+    'https://[64:ff9b::a9fe:a9fe]/',
+    'https://[fec0::1]/',
+    'https://224.0.0.1/',
+])
+def test_submodule_url_error_rejects_non_public_literals(url):
+    assert lib.submodule_url_error(url, REPO)
+
+
+
+def test_submodule_on_repo_host():
+    """The repo's own host is fetched from the addresses the repo was fetched from, never resolved again (so its
+    DNS cannot change in between).  The same name on another port is checked like any other host."""
+    lan_forge = lib.HostAddresses('git.lan', 443, ['192.168.1.20'])
+    with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['127.0.0.1']) as resolve:
+        # A LAN forge the user chose: its own submodules work.
+        assert lib.submodule_url_error('https://git.lan/owner/library', lan_forge) is None
+        assert lib.submodule_url_error('https://git.lan:443/owner/library', lan_forge) is None
+        resolve.assert_not_called()
+        # Another port on that name (e.g. a Docker API) is not the repo.
+        assert 'not a public address' in lib.submodule_url_error('https://git.lan:2375/x', lan_forge)
+
+    # A repo host which could not be resolved cannot vouch for anything.
+    unresolved = lib.HostAddresses('git.lan', 443, None)
+    assert lib.submodule_url_error('https://git.lan/owner/library', unresolved)
+    # An address needs no DNS.
+    literal = lib.HostAddresses('192.168.1.20', 443, None)
+    assert lib.submodule_url_error('https://192.168.1.20/owner/library', literal) is None
+
+
+@pytest.mark.parametrize('url,expected', [
+    ('https://git.example.com/owner/repo', lib.HostAddresses('git.example.com', 443, ['93.184.215.14'])),
+    ('https://git.example.com:8443/repo', lib.HostAddresses('git.example.com', 8443, ['93.184.215.14'])),
+    ('https://192.168.1.20/repo', lib.HostAddresses('192.168.1.20', 443, None)),
+])
+def test_resolve_repo_addresses(url, expected):
+    with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['93.184.215.14']):
+        assert lib.resolve_repo_addresses(url) == expected
+    assert lib.pin_config(lib.HostAddresses('h', 443, ['1.2.3.4', '2606:4700::1111'])) == \
+           'http.curloptResolve=h:443:1.2.3.4,[2606:4700::1111]'
+    assert lib.pin_config(lib.HostAddresses('h', 443, None)) is None
+
+
+@pytest.mark.asyncio
+async def test_repo_fetches_are_pinned(test_session, git_remote_factory, repos_download_manager):
+    """Every git command which connects to the repo's host connects to the address resolved at the start."""
+    from modules.repos.downloader import GitDownloader
+    remote = git_remote_factory()
+    remote.commit({'a.txt': 'a'}, 'Second')
+    lib.create_repository(test_session, remote.url)
+
+    commands = []
+    original = GitDownloader.process_runner
+
+    async def record(self, download, cmd, *args, **kwargs):
+        commands.append([str(i) for i in cmd])
+        return await original(self, download, cmd, *args, **kwargs)
+
+    with mock.patch('modules.repos.lib.resolve_host_addresses', return_value=['93.184.215.14']), \
+            mock.patch.object(GitDownloader, 'process_runner', record):
+        await repos_download_manager.wait_for_all_downloads()
+        await update(test_session, repos_download_manager)
+
+    pin = 'http.curloptResolve=git.example.com:443:93.184.215.14'
+    network = [i for i in commands if {'ls-remote', 'clone', 'fetch'} & set(i)]
+    assert {'ls-remote', 'clone', 'fetch'} <= {j for i in network for j in i}
+    assert all(pin in i for i in network), network

modules/repos/test/test_import.py

@@ -0,0 +1,287 @@
+import json
+import subprocess
+from http import HTTPStatus
+
+import pytest
+
+from modules.repos import lib
+from modules.repos.errors import InvalidRepo, RepoConflict
+from modules.repos.models import Repository
+from wrolpi.downloader import Download, DownloadStatus
+from wrolpi.errors import UnknownDirectory, ValidationError
+from wrolpi.files.models import FileGroup
+
+
+def git(directory, *args) -> str:
+    env = dict(GIT_AUTHOR_NAME='t', GIT_AUTHOR_EMAIL='t@t', GIT_COMMITTER_NAME='t', GIT_COMMITTER_EMAIL='t@t',
+               PATH='/usr/bin:/bin:/usr/local/bin:/opt/homebrew/bin')
+    return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True, env=env,
+                          capture_output=True).stdout.decode().strip()
+
+
+@pytest.fixture
+def user_clone(test_directory, git_remote_factory):
+    """A clone the user made themselves, in their media directory, with an `origin`."""
+
+    def factory(relative='code/kiwix-tools', remote=None):
+        remote = remote or git_remote_factory('kiwix-tools', owner='kiwix', files={'README.md': '# Kiwix tools'})
+        directory = test_directory / relative
+        directory.parent.mkdir(parents=True, exist_ok=True)
+        git(directory.parent, 'clone', '--quiet', str(remote.path), directory.name)
+        git(directory, 'remote', 'set-url', 'origin', remote.url)
+        return directory, remote
+
+    return factory
+
+
+@pytest.mark.parametrize('origin,expected', [
+    ('https://github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools'),
+    ('https://github.com/kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools.git'),
+    ('git@github.com:kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools'),
+    ('ssh://git@github.com:22/kiwix/kiwix-tools.git', 'https://github.com/kiwix/kiwix-tools'),
+    ('git://github.com/kiwix/kiwix-tools', 'https://github.com/kiwix/kiwix-tools'),
+    ('/home/user/kiwix-tools', None),
+    ('../kiwix-tools', None),
+])
+def test_remote_https_url(origin, expected):
+    assert lib.remote_https_url(origin) == expected
+
+
+def test_remote_url_key():
+    assert lib.remote_url_key('git@GitHub.com:kiwix/kiwix-tools.git') == \
+           lib.remote_url_key('https://github.com/kiwix/kiwix-tools/') == 'github.com/kiwix/kiwix-tools'
+    assert lib.remote_url_key('https://github.com/kiwix/other') != lib.remote_url_key('https://github.com/kiwix/x')
+    assert lib.remote_url_key('/local/path') is None
+
+
+@pytest.mark.asyncio
+async def test_inspect_import(test_session, async_client, test_wrolpi_config, user_clone):
+    """Inspecting a clone reports what an import would do, and changes nothing."""
+    directory, remote = user_clone()
+    git(directory, 'commit', '--quiet', '--allow-empty', '-m', 'Local work')
+    config_before = (directory / '.git/config').read_text()
+
+    request, response = await async_client.post('/api/repos/import/inspect',
+                                                content=json.dumps(dict(directory='code/kiwix-tools')))
+    assert response.status_code == HTTPStatus.OK, response.json
+    info = response.json['inspection']
+    assert info['directory'] == 'code/kiwix-tools'
+    assert info['origin'] == remote.url
+    assert info['url'] == remote.url
+    assert info['name'] == 'kiwix-tools'
+    assert info['branch'] == 'main'
+    assert info['head_message'] == 'Local work'
+    assert info['head_sha'] == git(directory, 'rev-parse', 'HEAD')
+    assert info['head_date']
+    assert info['local_commits'] == 1
+    assert info['shallow'] is False
+    # The clone stays where it is, where its files are indexed.
+    assert info['ignored'] is False
+
+    assert (directory / '.git/config').read_text() == config_before
+    assert not (directory / '.git' / lib.REPO_MARKER).exists()
+
+    # A clone in the Repos directory is ignored.
+    user_clone('repos/other', remote=remote)
+    assert lib.inspect_import(test_session, 'repos/other')['ignored'] is True
+
+
+@pytest.mark.asyncio
+async def test_inspect_import_refuses(test_session, test_directory, async_client, test_wrolpi_config, user_clone):
+    (test_directory / 'not-git').mkdir()
+    with pytest.raises(InvalidRepo):
+        lib.inspect_import(test_session, 'not-git')
+    with pytest.raises(UnknownDirectory):
+        lib.inspect_import(test_session, 'missing')
+    for directory in ('', '/etc', '../outside', 'config', 'tags', 'videos'):
+        (test_directory / 'videos').mkdir(exist_ok=True)
+        with pytest.raises((InvalidRepo, UnknownDirectory)):
+            lib.inspect_import(test_session, directory)
+
+    # `.git` must be the clone's own directory.
+    (test_directory / 'gitfile').mkdir()
+    (test_directory / 'gitfile/.git').write_text('gitdir: /etc\n')
+    with pytest.raises(InvalidRepo):
+        lib.inspect_import(test_session, 'gitfile')
+
+    # A partial clone's missing objects would come from the network.
+    directory, _ = user_clone('code/partial')
+    git(directory, 'config', 'remote.origin.promisor', 'true')
+    with pytest.raises(InvalidRepo, match='partial clone'):
+        lib.inspect_import(test_session, 'code/partial')
+
+    # Already a Repo, or inside one.
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    repo.directory.mkdir(parents=True)
+    git(repo.directory, 'init', '--quiet')
+    with pytest.raises(RepoConflict):
+        lib.inspect_import(test_session, 'repos/b')
+
+
+@pytest.mark.asyncio
+async def test_import(test_session, test_directory, async_client, test_wrolpi_config, user_clone,
+                      repos_download_manager, refresh_files):
+    """An imported clone stays where it is, is owned by WROLPi, and is updated like any Repo."""
+    directory, remote = user_clone()
+    (directory / 'local-note.txt').write_text('untracked')
+    with (directory / '.git/config').open('a') as fh:
+        fh.write('[http]\n\tproxy = http://127.0.0.1:9\n[user]\n\tname = Someone\n')
+    await refresh_files()
+    indexed = test_session.query(FileGroup).filter(FileGroup.primary_path.contains('code/kiwix-tools')).count()
+    assert indexed
+
+    body = dict(directory='code/kiwix-tools', tag_name='Software', confirm=True, frequency=86400)
+    request, response = await async_client.post('/api/repos/import', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.CREATED, response.json
+    imported = response.json['repo']
+    assert imported['name'] == 'kiwix-tools'
+    assert imported['url'] == remote.url
+    assert imported['directory'] == 'code/kiwix-tools'
+    assert imported['tag_name'] == 'Software'
+    assert imported['frequency'] == 86400
+    assert imported['branch'] is None  # Follows the default branch, like a new Repo.
+    assert imported['head_sha'] == remote.head
+    assert imported['readme_path'] == 'README.md'
+
+    assert (directory / 'README.md').is_file()
+    assert (directory / '.git' / lib.REPO_MARKER).is_file()
+    config = (directory / '.git/config').read_text()
+    assert 'proxy' not in config and 'Someone' not in config
+    # It was not moved (or ignored); its files are still indexed.
+    assert not (test_directory / 'repos').exists()
+    await refresh_files()
+    assert test_session.query(FileGroup).filter(FileGroup.primary_path.contains('code/kiwix-tools')).count()
+
+    # From now on it is a mirror: an update fetches the origin, and discards local files.
+    await repos_download_manager.wait_for_all_downloads()
+    head = remote.commit({'new.txt': 'new'}, 'New upstream')
+    download = test_session.query(Download).one()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.complete, download.error
+    assert test_session.query(Repository).one().head_sha == head
+    assert (directory / 'new.txt').is_file()
+    assert not (directory / 'local-note.txt').exists()
+
+
+@pytest.mark.asyncio
+async def test_import_refuses(test_session, test_directory, async_client, test_wrolpi_config, user_clone,
+                              git_remote_factory):
+    directory, remote = user_clone()
+
+    # Without confirmation.
+    with pytest.raises(ValidationError):
+        await lib.import_repository(test_session, 'code/kiwix-tools')
+    # A URL which is not the clone's origin.
+    with pytest.raises(InvalidRepo, match="not this clone's origin"):
+        await lib.import_repository(test_session, 'code/kiwix-tools', url='https://github.com/evil/other',
+                                    confirm=True)
+    test_session.rollback()
+    assert test_session.query(Repository).count() == 0
+    assert not (directory / '.git' / lib.REPO_MARKER).exists()
+
+    # Without an https origin, the URL must be given (and must still be the origin).
+    other, _ = user_clone('code/ssh-clone', remote=git_remote_factory('library'))
+    git(other, 'remote', 'set-url', 'origin', 'git@git.example.com:owner/library.git')
+    assert lib.inspect_import(test_session, 'code/ssh-clone')['url'] == 'https://git.example.com/owner/library'
+    git(other, 'remote', 'remove', 'origin')
+    assert lib.inspect_import(test_session, 'code/ssh-clone')['url'] is None
+    with pytest.raises(InvalidRepo, match='https:// URL'):
+        await lib.import_repository(test_session, 'code/ssh-clone', confirm=True)
+    repo = await lib.import_repository(test_session, 'code/ssh-clone', url='https://git.example.com/owner/library',
+                                       confirm=True)
+    assert repo.url == 'https://git.example.com/owner/library'
+    assert repo.directory == other
+
+
+@pytest.mark.asyncio
+async def test_refused_import_changes_nothing(test_session, test_directory, async_client, test_wrolpi_config,
+                                              user_clone):
+    """Everything which can refuse an import is checked before the clone is changed."""
+    directory, _ = user_clone()
+    (directory / 'other').mkdir()
+    (directory / 'other/.git').write_text(f'gitdir: {test_directory}\n')
+    config_before = (directory / '.git/config').read_text()
+    with pytest.raises(InvalidRepo, match='outside'):
+        await lib.import_repository(test_session, 'code/kiwix-tools', confirm=True)
+    test_session.rollback()
+    assert (directory / '.git/config').read_text() == config_before
+    assert not (directory / '.git' / lib.REPO_MARKER).exists()
+    assert test_session.query(Repository).count() == 0
+
+
+@pytest.mark.asyncio
+async def test_inspect_never_reads_the_clones_config(test_session, test_directory, async_client, test_wrolpi_config,
+                                                     user_clone):
+    """Inspecting runs no git command with the clone's (unsanitized) config: one git refuses to read still inspects."""
+    directory, remote = user_clone()
+    with (directory / '.git/config').open('a') as fh:
+        fh.write('[core]\n\trepositoryformatversion = 1\n[extensions]\n\tnotAnExtension = true\n'
+                 f'[include]\n\tpath = {test_directory}/included.gitconfig\n')
+    # git itself refuses this repository.
+    assert subprocess.run(['git', '-c', 'safe.directory=*', 'rev-parse', 'HEAD'], cwd=directory,
+                          capture_output=True).returncode != 0
+
+    info = lib.inspect_import(test_session, 'code/kiwix-tools')
+    assert info['head_sha'] == remote.head
+    assert info['head_message'] == 'Initial commit'
+    assert info['local_commits'] == 0
+
+
+def test_working_tree_git_file_is_read_like_git(tmp_path):
+    """git trims only the line ending from a `.git` file, not spaces; so does WROLPi."""
+    repo = tmp_path / 'repo'
+    (repo / '.git/modules/library/objects').mkdir(parents=True)
+    (repo / '.git/modules/library/HEAD').write_text('ref: refs/heads/main\n')
+    (repo / 'vendor/library').mkdir(parents=True)
+    # git reads this as "<...>/library " (with the space): not the checked git directory.
+    (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library/../../../.. \n')
+    assert lib.working_tree_git_error(repo)
+    (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library\r\n')
+    assert lib.working_tree_git_error(repo) is None
+
+
+
+@pytest.mark.asyncio
+async def test_create_repository_in_a_directory(test_session, test_directory, async_client, test_wrolpi_config,
+                                                git_remote_factory, repos_download_manager, refresh_files):
+    """A Repo can be cloned into a directory the user chooses.  It is not ignored: its files are indexed."""
+    remote = git_remote_factory(files={'README.md': '# Example', 'guide.md': 'A guide'})
+    body = dict(url=remote.url, directory='projects/example', tag_name='Software')
+    request, response = await async_client.post('/api/repos', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.CREATED, response.json
+    assert response.json['repo']['directory'] == 'projects/example'
+    await repos_download_manager.wait_for_all_downloads()
+
+    directory = test_directory / 'projects/example'
+    assert (directory / 'guide.md').is_file()
+    await refresh_files()
+    assert test_session.query(FileGroup).filter(FileGroup.primary_path.contains('projects/example')).count()
+
+
+@pytest.mark.parametrize('directory,error', [
+    ('/etc/example', 'relative'),
+    ('../example', 'relative'),
+    ('.', 'media directory'),
+    ('videos', 'cannot be used'),
+    ('config/example', 'cannot be used'),
+    ('tags', 'cannot be used'),
+    ('not-empty', 'not empty'),
+    ('existing-repo/inside', 'Collection'),
+])
+@pytest.mark.asyncio
+async def test_create_repository_directory_refused(test_session, test_directory, async_client, test_wrolpi_config,
+                                                   directory, error):
+    (test_directory / 'videos').mkdir()
+    (test_directory / 'not-empty').mkdir()
+    (test_directory / 'not-empty/file.txt').write_text('mine')
+    lib.create_repository(test_session, 'https://github.com/a/existing', directory='existing-repo')
+
+    body = dict(url='https://github.com/a/b', directory=directory)
+    request, response = await async_client.post('/api/repos', content=json.dumps(body))
+    assert response.status_code in (HTTPStatus.BAD_REQUEST, HTTPStatus.CONFLICT), response.json
+    assert error in response.json['error'], response.json
+    assert (test_directory / 'not-empty/file.txt').read_text() == 'mine'

modules/repos/test/test_lib.py

@@ -0,0 +1,380 @@
+import json
+import os
+from http import HTTPStatus
+
+import pytest
+from mock import mock
+
+from modules.repos import lib
+from modules.repos.errors import InvalidRepo, RepoConflict
+from modules.repos.models import Repository
+from wrolpi.collections.models import Collection
+from wrolpi.common import get_wrolpi_config
+from wrolpi.downloader import Download, DownloadFrequency
+from wrolpi.errors import UnknownDirectory, ValidationError
+from wrolpi.files.lib import get_normalized_ignored_directories
+from wrolpi.files.models import FileGroup
+
+
+@pytest.mark.parametrize('url,expected', [
+    ('https://github.com/kiwix/kiwix-tools',
+     lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'kiwix-tools')),
+    ('https://GitHub.com/kiwix/kiwix-tools.git/',
+     lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools.git', 'github.com', 'kiwix', 'kiwix-tools')),
+    ('https://github.com/kiwix/kiwix-tools/tree/feature/x',
+     lib.ParsedRepoURL('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'kiwix-tools', 'feature/x')),
+    ('https://gitlab.com/group/sub/project/-/tree/dev',
+     lib.ParsedRepoURL('https://gitlab.com/group/sub/project', 'gitlab.com', 'group/sub', 'project', 'dev')),
+    ('https://git.example.com:8443/project',
+     lib.ParsedRepoURL('https://git.example.com:8443/project', 'git.example.com', '', 'project')),
+])
+def test_parse_repo_url(url, expected):
+    assert lib.parse_repo_url(url) == expected
+
+
+@pytest.mark.parametrize('url', [
+    'http://github.com/kiwix/kiwix-tools',
+    'git@github.com:kiwix/kiwix-tools.git',
+    'ssh://git@github.com/kiwix/kiwix-tools',
+    'file:///etc',
+    'https://user:token@github.com/kiwix/kiwix-tools',
+    'https://github.com/kiwix/kiwix-tools?tab=readme',
+    'https://github.com',
+    'https://github.com/a/..',
+    'https://github.com/../b',
+    'https://github.com/a/./b',
+    '',
+])
+def test_parse_repo_url_invalid(url):
+    with pytest.raises(InvalidRepo):
+        lib.parse_repo_url(url)
+
+
+def test_repo_url_key():
+    assert lib.repo_url_key('https://GitHub.com/a/b.git/') == lib.repo_url_key('https://github.com/a/b')
+    assert lib.repo_url_key('https://github.com/a/b') != lib.repo_url_key('https://github.com/a/c')
+
+
+def test_git_command_is_restricted():
+    cmd = lib.git_command('clone', 'https://example.com/a')
+    assert 'protocol.allow=never' in cmd
+    assert 'protocol.https.allow=always' in cmd
+    assert 'core.symlinks=false' in cmd
+    assert 'core.hooksPath=/dev/null' in cmd
+    assert cmd[-2:] == ('clone', 'https://example.com/a')
+    assert lib.git_env()['GIT_TERMINAL_PROMPT'] == '0'
+
+
+@pytest.mark.asyncio
+async def test_create_repository(test_session, test_directory, test_wrolpi_config, async_client):
+    repo = lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software')
+
+    collection = test_session.query(Collection).one()
+    assert collection.kind == 'repo'
+    assert collection.name == 'kiwix-tools'
+    assert collection.tag_name == 'software'
+    assert collection.directory == test_directory / 'repos/software/kiwix-tools'
+    assert repo.collection_id == collection.id
+    assert (repo.url, repo.host, repo.owner, repo.mode, repo.branch) == \
+           ('https://github.com/kiwix/kiwix-tools', 'github.com', 'kiwix', 'full', None)
+
+    # Repos are always kept up to date; weekly by default.
+    download = test_session.query(Download).one()
+    assert download.url == repo.url
+    assert download.downloader == 'git'
+    assert download.frequency == DownloadFrequency.weekly
+    assert download.collection_id == collection.id
+    assert repo.download == download
+
+    # Without a tag.
+    repo = lib.create_repository(test_session, 'https://github.com/kiwix/libkiwix')
+    assert repo.directory == test_directory / 'repos/libkiwix'
+
+
+@pytest.mark.asyncio
+async def test_create_repository_conflicts(test_session, test_wrolpi_config, async_client):
+    lib.create_repository(test_session, 'https://github.com/a/utils')
+
+    # The same repo, even with `.git`.
+    with pytest.raises(RepoConflict):
+        lib.create_repository(test_session, 'https://github.com/a/utils.git')
+
+    # Another owner's repo with the same name; a new name is suggested.
+    with pytest.raises(RepoConflict) as e:
+        lib.create_repository(test_session, 'https://github.com/b/utils')
+    assert "'b-utils'" in str(e.value)
+    test_session.rollback()
+
+    repo = lib.create_repository(test_session, 'https://github.com/b/utils', name='b-utils')
+    assert repo.name == 'b-utils'
+    assert test_session.query(Repository).count() == 2
+
+
+@pytest.mark.asyncio
+async def test_create_repository_invalid(test_session, test_wrolpi_config, async_client):
+    with pytest.raises(InvalidRepo):
+        lib.create_repository(test_session, 'http://github.com/a/b')
+    with pytest.raises(Exception):
+        lib.create_repository(test_session, 'https://github.com/a/b', mode='bare')
+    # 0 is never updated.
+    with pytest.raises(Exception):
+        lib.create_repository(test_session, 'https://github.com/a/b', frequency=-1)
+
+
+@pytest.mark.asyncio
+async def test_repos_destination(test_session, test_directory, test_wrolpi_config, async_client):
+    """The repos destination can use the repo's owner and host."""
+    get_wrolpi_config().repos_destination = 'repos/%(repo_host)s/%(repo_owner)s/%(repo_name)s'
+    repo = lib.create_repository(test_session, 'https://gitlab.com/group/sub/project')
+    assert repo.directory == test_directory / 'repos/gitlab.com/group⧸sub/project'
+
+    get_wrolpi_config().repos_destination = 'repos/%(tag_name)s/%(repo_name)s'
+    repo = lib.create_repository(test_session, 'https://github.com/a/b', tag_name='Tag')
+    assert repo.directory == test_directory / 'repos/Tag/b'
+
+
+@pytest.mark.asyncio
+async def test_update_repository(test_session, test_wrolpi_config, async_client):
+    repo = lib.create_repository(test_session, 'https://github.com/a/b', branch='dev')
+    lib.update_repository(test_session, repo.id, description='My repo', frequency=DownloadFrequency.daily,
+                          mode='snapshot', branch='')
+    assert repo.collection.description == 'My repo'
+    assert repo.download.frequency == DownloadFrequency.daily
+    assert repo.mode == 'snapshot'
+    # An empty branch follows the default branch.
+    assert repo.branch is None
+
+
+@pytest.mark.asyncio
+async def test_delete_repository(test_session, test_directory, test_wrolpi_config, await_background_tasks,
+                                 async_client):
+    keep = lib.create_repository(test_session, 'https://github.com/a/keep')
+    delete = lib.create_repository(test_session, 'https://github.com/a/delete')
+    for repo in (keep, delete):
+        repo.directory.mkdir(parents=True)
+        (repo.directory / 'README.md').write_text('readme')
+    keep_directory, delete_directory = keep.directory, delete.directory
+
+    await lib.delete_repository(test_session, keep.id)
+    await lib.delete_repository(test_session, delete.id, delete_files=True)
+    await await_background_tasks()
+
+    assert test_session.query(Repository).count() == 0
+    assert test_session.query(Collection).count() == 0
+    assert test_session.query(Download).count() == 0
+    # Files are kept by default.
+    assert (keep_directory / 'README.md').is_file()
+    assert not delete_directory.exists()
+
+
+@pytest.mark.asyncio
+async def test_list_repo_tree(test_session, test_directory, test_wrolpi_config, async_client):
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    with pytest.raises(UnknownDirectory):
+        lib.list_repo_tree(repo)
+
+    root = repo.directory
+    (root / '.git').mkdir(parents=True)
+    (root / 'src/lib').mkdir(parents=True)
+    (root / 'src/Readme.rst').write_text('src')
+    (root / 'README.md').write_text('# B')
+    (root / 'a.txt').write_text('aaa')
+    (root / 'Docs').mkdir()
+
+    tree = lib.list_repo_tree(repo)
+    assert tree == dict(
+        path='',
+        readme_path='README.md',
+        entries=[
+            dict(name='Docs', path='Docs', is_dir=True, size=None),
+            dict(name='src', path='src', is_dir=True, size=None),
+            dict(name='a.txt', path='a.txt', is_dir=False, size=3),
+            dict(name='README.md', path='README.md', is_dir=False, size=3),
+        ],
+    )
+
+    tree = lib.list_repo_tree(repo, '/src/')
+    assert tree['path'] == 'src'
+    assert tree['readme_path'] == 'src/Readme.rst'
+    assert [i['path'] for i in tree['entries']] == ['src/lib', 'src/Readme.rst']
+
+    with pytest.raises(InvalidRepo):
+        lib.list_repo_tree(repo, '../..')
+    with pytest.raises(UnknownDirectory):
+        lib.list_repo_tree(repo, 'missing')
+
+
+@pytest.mark.asyncio
+async def test_repos_directory_is_ignored(test_session, test_directory, test_wrolpi_config, refresh_files):
+    """Files in the repos directory are never indexed, even when an old config does not ignore it."""
+    config = get_wrolpi_config()
+    config.ignored_directories = ['config']
+    config.import_config()
+    assert 'repos' in config.ignored_directories
+
+    (test_directory / 'repos/a/b').mkdir(parents=True)
+    (test_directory / 'repos/a/b/README.md').write_text('# B')
+    (test_directory / 'other.txt').write_text('other')
+    await refresh_files()
+
+    assert [i.primary_path.name for i in test_session.query(FileGroup)] == ['other.txt']
+
+
+@pytest.mark.parametrize('branch,valid', [
+    ('main', True),
+    ('feature/x-1', True),
+    ('release-1.2', True),
+    ('--upload-pack=touch /tmp/pwned', False),
+    ('-b', False),
+    ('a..b', False),
+    ('a b', False),
+    ('a~1', False),
+    ('a^', False),
+    ('a:b', False),
+    ('a?', False),
+    ('a*', False),
+    ('a[', False),
+    ('a\\b', False),
+    ('a@{1}', False),
+    ('@', False),
+    ('a/', False),
+    ('/a', False),
+    ('a//b', False),
+    ('a.lock', False),
+    ('a/.b', False),
+    ('a.', False),
+    ('a\x00b', False),
+    ('', False),
+])
+def test_is_valid_branch(branch, valid):
+    assert lib.is_valid_branch(branch) is valid
+
+
+@pytest.mark.asyncio
+async def test_invalid_branch_rejected(test_session, async_client, test_wrolpi_config):
+    """A branch is passed to git, it must never be read as an option."""
+    with pytest.raises(ValidationError):
+        lib.create_repository(test_session, 'https://github.com/a/b', branch='--upload-pack=x')
+    with pytest.raises(ValidationError):
+        lib.create_repository(test_session, 'https://github.com/a/b/tree/a..b')
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    with pytest.raises(ValidationError):
+        lib.update_repository(test_session, repo.id, branch='-x')
+
+
+@pytest.mark.asyncio
+async def test_repo_directory_traversal(test_session, test_directory, async_client, test_wrolpi_config):
+    """A Repo's directory is always strictly inside the media directory, and never escapes its template."""
+    for name in ('..', '.', ' '):
+        with pytest.raises(InvalidRepo):
+            lib.create_repository(test_session, 'https://github.com/a/b', tag_name='tag', name=name)
+        test_session.rollback()
+
+    with pytest.raises(InvalidRepo):
+        lib.create_repository(test_session, 'https://github.com/a/b', directory=test_directory / 'repos/../../etc')
+    test_session.rollback()
+    with pytest.raises(InvalidRepo):
+        lib.create_repository(test_session, 'https://github.com/a/b', directory=test_directory)
+    test_session.rollback()
+
+    # The config can not move an existing Repo outside the media directory.
+    lib.create_repository(test_session, 'https://github.com/a/b')
+    with pytest.raises(InvalidRepo):
+        lib.ReposConfig._import_repo(test_session, dict(url='https://github.com/a/b', directory='../outside'))
+    test_session.rollback()
+    assert test_session.query(Repository).one().directory == test_directory / 'repos/b'
+
+
+def test_git_env_is_not_inherited():
+    """The process environment cannot undo git_command's restrictions."""
+    hostile = dict(
+        GIT_ALLOW_PROTOCOL='file', GIT_CONFIG_COUNT='1', GIT_CONFIG_KEY_0='url.https://127.0.0.1/.insteadOf',
+        GIT_CONFIG_VALUE_0='https://example.com/', GIT_CONFIG_PARAMETERS="'protocol.allow'='always'",
+        GIT_SSL_NO_VERIFY='1', GIT_PROXY_COMMAND='evil', GIT_SSH_COMMAND='evil', HTTPS_PROXY='http://127.0.0.1:9',
+        https_proxy='http://127.0.0.1:9', CURL_CA_BUNDLE='/tmp/evil.pem', GIT_DIR='/tmp/evil',
+    )
+    with mock.patch.dict(os.environ, dict(hostile, PATH='/usr/bin:/bin', SSL_CERT_FILE='/etc/ssl/cert.pem')):
+        env = lib.git_env()
+    assert not set(hostile) & set(env), set(hostile) & set(env)
+    assert env['PATH'] == '/usr/bin:/bin'
+    assert env['SSL_CERT_FILE'] == '/etc/ssl/cert.pem'
+    assert env['GIT_TERMINAL_PROMPT'] == '0'
+
+
+def test_git_command_ignores_hostile_environment(tmp_path):
+    """With git's environment variables set against it, git still refuses file:// (only https is allowed)."""
+    import subprocess
+    origin = tmp_path / 'origin'
+    subprocess.run(['git', 'init', '--quiet', str(origin)], check=True)
+    with mock.patch.dict(os.environ, dict(GIT_ALLOW_PROTOCOL='file')):
+        result = subprocess.run(lib.git_command('ls-remote', f'file://{origin}'), env=lib.git_env(),
+                                capture_output=True)
+    assert result.returncode != 0
+    assert b"transport 'file' not allowed" in result.stderr
+
+
+@pytest.mark.parametrize('destination,error', [
+    ('repos/%(repo_tag)s/%(repo_name)s', None),
+    ('library/code/%(repo_owner)s/%(repo_name)s', None),
+    ('repos/%(tag)s/%(name)s', None),
+    ('%(repo_name)s', 'fixed directory'),  # Would put repos in the media directory itself.
+    ('%(repo_tag)s/%(repo_name)s', 'fixed directory'),
+    ('repos', 'repo name'),  # Every repo in one directory.
+    ('repos/%(repo_tag)s', 'repo name'),
+    ('repos/%(bogus)s/%(repo_name)s', 'Unknown variable'),
+    ('repos/%(repo_name)d', 'Unknown variable'),
+    ('/repos/%(repo_name)s', 'relative'),
+    ('../repos/%(repo_name)s', '".."'),
+    ('repos/./%(repo_name)s', '".."'),
+    ('videos/%(repo_name)s', 'videos_destination'),
+    ('archive/x/%(repo_name)s', 'archive_destination'),
+    ('playlists/%(repo_name)s', 'playlists_destination'),
+    ('tags/%(repo_name)s', 'tags directory'),
+    ('config/%(repo_name)s', 'config directory'),
+    ('', 'empty'),
+])
+@pytest.mark.asyncio
+async def test_validate_repos_destination(async_client, test_wrolpi_config, destination, error):
+    result = lib.validate_repos_destination(destination)
+    if error is None:
+        assert result == ''
+    else:
+        assert error in result, result
+
+
+def test_repos_destination_root():
+    assert lib.repos_destination_root('repos/%(repo_tag)s/%(repo_name)s') == 'repos'
+    assert lib.repos_destination_root('library/code/%(repo_name)s') == 'library/code'
+    assert lib.repos_destination_root('%(repo_name)s') == ''
+
+
+@pytest.mark.asyncio
+async def test_repos_destination_is_ignored(test_session, test_directory, async_client, test_wrolpi_config,
+                                            refresh_files):
+    """Wherever repos are saved, their files are not indexed."""
+    body = dict(repos_destination='%(repo_name)s')
+    request, response = await async_client.patch('/api/settings', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.BAD_REQUEST
+
+    body = dict(repos_destination='library/code/%(repo_tag)s/%(repo_name)s')
+    request, response = await async_client.patch('/api/settings', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.NO_CONTENT
+    assert str(test_directory / 'library/code') in get_normalized_ignored_directories()
+
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    assert repo.directory == test_directory / 'library/code/b'
+    repo.directory.mkdir(parents=True)
+    (repo.directory / 'README.md').write_text('# B')
+    (test_directory / 'other.txt').write_text('other')
+    await refresh_files()
+    assert [i.primary_path.name for i in test_session.query(FileGroup)] == ['other.txt']
+
+    # A hand-edited config which breaks the rules falls back to the default.
+    config = get_wrolpi_config()
+    config.save(overwrite=True)
+    data = config.read_config_file()
+    data['repos_destination'] = '%(repo_name)s'
+    data['version'] = (data.get('version') or 0) + 1
+    config.write_config_data(data, config.get_file())
+    config.import_config()
+    assert config.repos_destination == 'repos/%(repo_tag)s/%(repo_name)s'

modules/repos/test/test_never.py

@@ -0,0 +1,175 @@
+"""A Repo which is never updated: its clone is kept as it is (e.g. its upstream is gone), and only updated on
+request."""
+import json
+import subprocess
+from http import HTTPStatus
+
+import pytest
+import yaml
+
+from modules.repos import lib
+from modules.repos.models import Repository
+from wrolpi.downloader import Download, DownloadFrequency, DownloadStatus
+
+
+def edit_config(config, change):
+    """Hand-edit each Repo in repos.yaml."""
+    data = yaml.safe_load(config.get_file().read_text())
+    data['repos'] = [change(i) for i in data['repos']]
+    config.get_file().write_text(yaml.dump(data))
+
+
+async def get_repo(async_client, repo_id: int) -> dict:
+    request, response = await async_client.get(f'/api/repos/{repo_id}')
+    assert response.status_code == HTTPStatus.OK, response.json
+    return response.json['repo']
+
+
+@pytest.mark.asyncio
+async def test_never_stops_updates(test_session, test_directory, async_client, test_wrolpi_config,
+                                   git_remote_factory, repos_download_manager):
+    """Changing a Repo to Never deletes its recurring Download (and its error); its clone is kept."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    download = test_session.query(Download).one()
+    download.error = 'fatal: repository not found'
+    download.status = DownloadStatus.deferred
+    test_session.commit()
+
+    request, response = await async_client.put(f'/api/repos/{repo.id}', content=json.dumps(dict(frequency=0)))
+    assert response.status_code == HTTPStatus.OK, response.json
+    assert response.json['repo']['frequency'] == 0
+    assert response.json['repo']['download_error'] is None
+    assert test_session.query(Download).count() == 0
+    assert (repo.directory / '.git').is_dir()
+
+    # Back to weekly.
+    request, response = await async_client.put(f'/api/repos/{repo.id}',
+                                               content=json.dumps(dict(frequency=DownloadFrequency.weekly)))
+    assert response.status_code == HTTPStatus.OK, response.json
+    assert response.json['repo']['frequency'] == DownloadFrequency.weekly
+    download = test_session.query(Download).one()
+    assert download.frequency == DownloadFrequency.weekly
+    assert download.collection_id == repo.collection_id
+
+
+@pytest.mark.asyncio
+async def test_never_new_repo_clones_once(test_session, test_directory, async_client, test_wrolpi_config,
+                                          git_remote_factory, repos_download_manager):
+    """A new Repo which is never updated is cloned once."""
+    remote = git_remote_factory()
+    body = dict(url=remote.url, frequency=0)
+    request, response = await async_client.post('/api/repos', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.CREATED, response.json
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+
+    repo = test_session.query(Repository).one()
+    assert repo.head_sha == remote.head
+    assert repo.download is None
+    download = test_session.query(Download).one()
+    assert download.frequency is None and download.collection_id is None
+    assert download.status == DownloadStatus.complete
+    assert (await get_repo(async_client, repo.id))['frequency'] == 0
+
+    # Back to weekly; the clone's (one-time) Download becomes the recurring Download.
+    request, response = await async_client.put(f'/api/repos/{repo.id}',
+                                               content=json.dumps(dict(frequency=DownloadFrequency.weekly)))
+    assert response.status_code == HTTPStatus.OK, response.json
+    download = test_session.query(Download).one()
+    assert download.frequency == DownloadFrequency.weekly
+    assert download.collection_id == repo.collection_id
+
+
+@pytest.mark.asyncio
+async def test_never_update_now(test_session, test_directory, async_client, test_wrolpi_config,
+                                git_remote_factory, repos_download_manager):
+    """A Repo which is never updated is updated on request, as often as requested; a failure is shown."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url, frequency=0)
+    await repos_download_manager.wait_for_all_downloads()
+
+    for _ in range(2):
+        head = remote.commit({'new.txt': remote.head}, 'New')
+        request, response = await async_client.post(f'/api/repos/{repo.id}/update')
+        assert response.status_code == HTTPStatus.NO_CONTENT, response.json
+        await repos_download_manager.wait_for_all_downloads()
+        test_session.expire_all()
+        assert repo.head_sha == head
+        assert repo.download is None
+
+    # The upstream is gone.
+    subprocess.run(['rm', '-rf', str(remote.path)], check=True)
+    request, response = await async_client.post(f'/api/repos/{repo.id}/update')
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    data = await get_repo(async_client, repo.id)
+    assert data['download_error']
+    assert data['frequency'] == 0
+    assert repo.head_sha == head
+
+
+@pytest.mark.asyncio
+async def test_never_import(test_session, test_directory, async_client, test_wrolpi_config, git_remote_factory,
+                            repos_download_manager):
+    """An imported clone which is never updated is never fetched."""
+    remote = git_remote_factory()
+    clone = test_directory / 'code/example'
+    subprocess.run(['git', 'clone', '--quiet', str(remote.path), str(clone)], check=True, capture_output=True)
+    subprocess.run(['git', '-C', str(clone), 'remote', 'set-url', 'origin', remote.url], check=True)
+    await lib.import_repository(test_session, 'code/example', url=remote.url, confirm=True, frequency=0)
+    test_session.commit()
+    assert test_session.query(Download).count() == 0
+    assert test_session.query(Repository).one().head_sha == remote.head
+
+
+@pytest.mark.asyncio
+async def test_never_config(test_session, test_directory, async_client, test_wrolpi_config, git_remote_factory,
+                            repos_download_manager):
+    """Never is saved in repos.yaml.  A restored Repo is cloned once if its clone is missing, and is otherwise left
+    as it is."""
+    kept, missing = git_remote_factory('kept'), git_remote_factory('missing')
+    lib.create_repository(test_session, kept.url, frequency=0)
+    lib.create_repository(test_session, missing.url, frequency=DownloadFrequency.daily)
+    await repos_download_manager.wait_for_all_downloads()
+    config = lib.get_repos_config()
+    config.dump_config()
+    assert {i['name']: i['frequency'] for i in config.repos} == {'kept': 'never', 'missing': DownloadFrequency.daily}
+
+    # A Repo whose frequency is missing from the config is updated weekly.
+    edit_config(config, lambda i: {k: v for k, v in i.items() if not (i['name'] == 'missing' and k == 'frequency')})
+
+    # Lose the database, and one clone.
+    for repo in test_session.query(Repository).all():
+        lib._delete_repository(test_session, repo)
+    for download in test_session.query(Download).all():
+        test_session.delete(download)
+    test_session.commit()
+    subprocess.run(['rm', '-rf', str(test_directory / 'repos/missing')], check=True)
+
+    config.import_config()
+    test_session.expire_all()
+    repos = {i.name: i for i in test_session.query(Repository)}
+    assert repos['kept'].download is None
+    assert repos['missing'].download.frequency == DownloadFrequency.weekly
+    # Only the missing clone is downloaded.
+    assert [i.url for i in test_session.query(Download)] == [repos['missing'].url]
+
+    # Never, in a hand-edited config.
+    edit_config(config, lambda i: dict(i, frequency='never'))
+    config.import_config()
+    test_session.expire_all()
+    assert test_session.query(Download).count() == 0
+
+
+@pytest.mark.asyncio
+async def test_never_delete(test_session, test_directory, async_client, test_wrolpi_config, git_remote_factory,
+                            repos_download_manager):
+    """Deleting a Repo which is never updated deletes its one-time Download."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url, frequency=0)
+    await repos_download_manager.wait_for_all_downloads()
+    assert test_session.query(Download).count() == 1
+    await lib.delete_repository(test_session, repo.id)
+    assert test_session.query(Download).count() == 0

modules/repos/test/test_ownership.py

@@ -0,0 +1,577 @@
+import subprocess
+
+import pytest
+
+from modules.repos import lib
+from wrolpi.downloader import Download, DownloadStatus
+
+
+def git(directory, *args) -> str:
+    return subprocess.run(['git', '-c', 'safe.directory=*', *args], cwd=directory, check=True,
+                          capture_output=True).stdout.decode().strip()
+
+
+async def update(test_session, repos_download_manager):
+    download = test_session.query(Download).one()
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    return test_session.query(Download).one()
+
+
+@pytest.mark.asyncio
+async def test_clone_is_marked(test_session, git_remote_factory, repos_download_manager):
+    """WROLPi marks the clones it makes, so it never mistakes another clone for one of its own."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    # The marker holds the Repo's own random token, which nothing else can know (unlike its URL).
+    assert len(repo.clone_token) == 32
+    assert (repo.directory / '.git' / lib.REPO_MARKER).read_text().strip() == repo.clone_token
+
+
+@pytest.mark.asyncio
+async def test_update_refuses_unmarked_clone(test_session, test_directory, git_remote_factory,
+                                             repos_download_manager):
+    """A clone WROLPi did not make (or made for another repo) is not updated, so it is never reset or cleaned."""
+    remote = git_remote_factory()
+    directory = test_directory / 'repos/example'
+    directory.parent.mkdir(parents=True)
+    git(directory.parent, 'clone', '--quiet', str(remote.path), str(directory))
+    (directory / 'my-work.txt').write_text('uncommitted')
+
+    lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'was not created by WROLPi' in download.error
+    assert (directory / 'my-work.txt').read_text() == 'uncommitted'
+
+    # A marker for another repo is not this repo's.
+    (directory / '.git' / lib.REPO_MARKER).write_text('git.example.com/owner/other')
+    download = await update(test_session, repos_download_manager)
+    assert 'was not created by WROLPi' in download.error
+    assert (directory / 'my-work.txt').exists()
+
+
+@pytest.mark.asyncio
+async def test_update_refuses_git_file(test_session, test_directory, git_remote_factory, repos_download_manager):
+    """`.git` must be the repo's own directory, not a file or link which points elsewhere."""
+    remote = git_remote_factory()
+    directory = test_directory / 'repos/example'
+    directory.mkdir(parents=True)
+    (directory / '.git').write_text('gitdir: /somewhere/else\n')
+
+    lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'not a git directory' in download.error
+
+
+@pytest.mark.asyncio
+async def test_update_fetches_the_repo_url(test_session, git_remote_factory, repos_download_manager):
+    """The Repo's URL is fetched, whatever the clone's origin says."""
+    remote = git_remote_factory()
+    other = git_remote_factory('other')
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    git(repo.directory, 'config', 'remote.origin.url', other.url)
+    head = remote.commit({'new.txt': 'new'}, 'New')
+    await update(test_session, repos_download_manager)
+    assert repo.head_sha == head
+    assert git(repo.directory, 'config', 'remote.origin.url') == remote.url
+
+
+@pytest.mark.asyncio
+async def test_update_sanitizes_git_config(test_session, git_remote_factory, repos_download_manager):
+    """Only the settings WROLPi needs survive in a clone's config; a planted setting cannot change what git does."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    config = repo.directory / '.git/config'
+    config.write_text(config.read_text() + (
+        '[url "https://127.0.0.1/"]\n\tinsteadOf = https://git.example.com/\n'
+        '[http]\n\tproxy = http://127.0.0.1:9\n'
+        '[include]\n\tpath = /etc/evil.gitconfig\n'
+        '[core]\n\tsshCommand = evil\n\tworktree = /etc\n\tfsmonitor = evil\n'
+        '[branch "main"]\n\tremote = https://127.0.0.1/evil\n'
+    ))
+    head = remote.commit({'new.txt': 'new'}, 'New')
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.complete, download.error
+    assert repo.head_sha == head
+
+    keys = set(git(repo.directory, 'config', '--local', '--name-only', '--list').splitlines())
+    assert {'remote.origin.url', 'remote.origin.fetch', 'core.repositoryformatversion'} <= keys
+    assert not {'url.https://127.0.0.1/.insteadof', 'http.proxy', 'include.path', 'core.sshcommand',
+                'core.worktree', 'core.fsmonitor', 'branch.main.remote'} & keys
+
+
+def test_sanitize_git_config_submodules(tmp_path):
+    """A submodule's config may point its work tree inside the repo, and nowhere else."""
+    repo = tmp_path / 'repo'
+    git_dir = repo / '.git'
+    module = git_dir / 'modules/library'
+    module.mkdir(parents=True)
+    (repo / 'vendor/library').mkdir(parents=True)
+    (git_dir / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tbare = false\n')
+    (module / 'config').write_text('[core]\n\trepositoryformatversion = 0\n\tworktree = ../../../vendor/library\n'
+                                   '[remote "origin"]\n\turl = https://git.example.com/owner/library\n')
+    evil = git_dir / 'modules/evil'
+    evil.mkdir()
+    (evil / 'config').write_text('[core]\n\tworktree = /etc\n')
+    for gitdir in (module, evil):
+        (gitdir / 'HEAD').write_text('ref: refs/heads/main\n')
+        (gitdir / 'objects').mkdir()
+
+    lib.sanitize_git_config(repo)
+    assert 'worktree = ../../../vendor/library' in (module / 'config').read_text()
+    assert 'worktree' not in (evil / 'config').read_text()
+
+
+@pytest.mark.asyncio
+async def test_update_with_duplicate_and_option_like_settings(test_session, git_remote_factory,
+                                                             repos_download_manager):
+    """Planted extra or option-like values in the kept settings cannot break or redirect an update."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+
+    config = repo.directory / '.git/config'
+    config.write_text(config.read_text() + (
+        '[remote "origin"]\n\turl = --file=/tmp/evil\n\turl = https://127.0.0.1/evil\n'
+        '\tfetch = +refs/heads/*:refs/heads/*\n'
+        '[submodule "lib"]\n\turl = "quoted \\"value\\" \\\\ and\\ttab"\n'
+    ))
+    head = remote.commit({'new.txt': 'new'}, 'New')
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.complete, download.error
+    assert repo.head_sha == head
+    assert git(repo.directory, 'config', '--get-all', 'remote.origin.url') == remote.url
+    assert git(repo.directory, 'config', '--get-all', 'remote.origin.fetch') == '+refs/heads/*:refs/remotes/origin/*'
+
+
+def test_sanitized_config_round_trips(tmp_path):
+    """Kept values are written back exactly, whatever they contain."""
+    repo = tmp_path / 'repo'
+    (repo / '.git').mkdir(parents=True)
+    values = ['--file=/tmp/evil', 'a "quoted" value', 'back\\slash', 'tab\there', ' leading space', 'semi;colon#hash']
+    subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), 'http.proxy', 'evil'], check=True)
+    for value in values:
+        subprocess.run(['git', 'config', '--file', str(repo / '.git/config'), '--add', 'submodule.lib.url', value],
+                       check=True)
+    lib.sanitize_git_config(repo)
+    result = git(repo, 'config', '--file', '.git/config', '--get-all', 'submodule.lib.url')
+    assert result.split('\n') == values
+    assert 'proxy' not in (repo / '.git/config').read_text()
+
+
+def test_marker_and_config_do_not_follow_links(tmp_path):
+    """A clone (e.g. one being imported) can hold links; WROLPi never writes through them."""
+    repo = tmp_path / 'repo'
+    git_dir = repo / '.git'
+    git_dir.mkdir(parents=True)
+    victim = tmp_path / 'victim.txt'
+    victim.write_text('precious')
+
+    (git_dir / lib.REPO_MARKER).symlink_to(victim)
+    lib.mark_clone(repo, 'a1b2c3')
+    assert victim.read_text() == 'precious'
+    assert not (git_dir / lib.REPO_MARKER).is_symlink()
+    assert (git_dir / lib.REPO_MARKER).read_text().strip() == 'a1b2c3'
+
+    # A git directory holding any link is refused.
+    (git_dir / 'config').write_text('[http]\n\tproxy = evil\n[core]\n\tbare = false\n')
+    (git_dir / 'config.wrolpi').symlink_to(victim)
+    with pytest.raises(ValueError, match='link'):
+        lib.sanitize_git_config(repo)
+    assert victim.read_text() == 'precious'
+    (git_dir / 'config.wrolpi').unlink()
+    lib.sanitize_git_config(repo)
+    assert 'proxy' not in (git_dir / 'config').read_text()
+
+    # A config which is a link is refused.
+    (git_dir / 'config').unlink()
+    (git_dir / 'config').symlink_to(victim)
+    with pytest.raises(ValueError):
+        lib.sanitize_git_config(repo)
+    assert victim.read_text() == 'precious'
+
+
+@pytest.mark.asyncio
+async def test_import_refuses_linked_config(test_session, test_directory, async_client, test_wrolpi_config):
+    from modules.repos.errors import InvalidRepo
+    directory = test_directory / 'code/linked'
+    git(test_directory, 'init', '--quiet', 'code/linked')
+    (directory / '.git/config').unlink()
+    (directory / '.git/config').symlink_to(test_directory / 'elsewhere')
+    with pytest.raises(InvalidRepo, match='config'):
+        lib.inspect_import(test_session, 'code/linked')
+
+
+
+@pytest.mark.asyncio
+async def test_import_directory_is_not_an_oracle(test_session, test_directory, async_client, test_wrolpi_config,
+                                                tmp_path_factory):
+    """A link in the media directory cannot reveal whether a path outside it exists."""
+    from modules.repos.errors import InvalidRepo
+    outside = tmp_path_factory.mktemp('outside')
+    (outside / 'exists').mkdir()
+    (test_directory / 'code').mkdir()
+    (test_directory / 'code/link').symlink_to(outside)
+    for directory in ('code/link/exists', 'code/link/missing'):
+        with pytest.raises(InvalidRepo, match='must be in the media directory'):
+            lib.resolve_import_directory(test_session, directory)
+
+
+def test_git_command_disables_automatic_programs():
+    cmd = lib.git_command('rev-parse', 'HEAD')
+    for setting in ('gc.auto=0', 'maintenance.auto=false', 'core.alternateRefsCommand=', 'core.fsmonitor=false',
+                    'core.hooksPath=/dev/null'):
+        assert setting in cmd, setting
+
+
+@pytest.mark.parametrize('planted', ['commondir', 'objects/info/alternates'])
+@pytest.mark.asyncio
+async def test_clone_cannot_borrow_another_git_directory(test_session, test_directory, async_client,
+                                                         test_wrolpi_config, git_remote_factory,
+                                                         repos_download_manager, planted):
+    """`.git/commondir` makes git read another directory's config (bypassing the sanitized one), and alternates
+    read its objects; neither is in a clone WROLPi makes, so neither is imported or updated."""
+    from modules.repos.errors import InvalidRepo
+    elsewhere = test_directory / 'elsewhere'
+    elsewhere.mkdir()
+
+    # Import.
+    git(test_directory, 'init', '--quiet', 'code/borrowing')
+    (test_directory / 'code/borrowing/.git' / planted).write_text(f'{elsewhere}\n')
+    with pytest.raises(InvalidRepo, match='another git directory'):
+        lib.inspect_import(test_session, 'code/borrowing')
+
+    # Update.
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    (repo.directory / '.git' / planted).write_text(f'{elsewhere}\n')
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.deferred
+    assert 'another git directory' in download.error
+
+
+@pytest.mark.parametrize('plant', ['config-link', 'commondir', 'alternates', 'modules-link', 'gitdir-link'])
+def test_submodule_git_directories_are_checked_too(tmp_path, plant):
+    """A submodule's git directory is held to the same rules as the repo's: no linked config, no borrowing."""
+    repo = tmp_path / 'repo'
+    git_dir = repo / '.git'
+    git_dir.mkdir(parents=True)
+    (git_dir / 'config').write_text('[core]\n\tbare = false\n')
+    victim = tmp_path / 'victim'
+    victim.mkdir()
+    (victim / 'config').write_text('[core]\n\tfsmonitor = evil\n')
+
+    module = git_dir / 'modules/library'
+    (module / 'objects').mkdir(parents=True)
+    (module / 'HEAD').write_text('ref: refs/heads/main\n')
+    (module / 'config').write_text('[core]\n\tbare = false\n')
+    # A ref named HEAD is not a git directory.
+    (module / 'refs/remotes/origin').mkdir(parents=True)
+    (module / 'refs/remotes/origin/HEAD').write_text('ref: refs/remotes/origin/main\n')
+
+    if plant == 'config-link':
+        (module / 'config').unlink()
+        (module / 'config').symlink_to(victim / 'config')
+    elif plant == 'commondir':
+        (module / 'commondir').write_text(f'{victim}\n')
+    elif plant == 'alternates':
+        (module / 'objects/info').mkdir()
+        (module / 'objects/info/alternates').write_text(f'{victim}\n')
+    elif plant == 'modules-link':
+        import shutil
+        shutil.rmtree(git_dir / 'modules')
+        (git_dir / 'modules').symlink_to(victim)
+    elif plant == 'gitdir-link':
+        (git_dir / 'modules/other').symlink_to(victim)
+
+    with pytest.raises(ValueError):
+        lib.sanitize_git_config(repo)
+    assert (victim / 'config').read_text() == '[core]\n\tfsmonitor = evil\n'
+
+
+def test_submodule_git_directories_pass(tmp_path):
+    repo = tmp_path / 'repo'
+    git_dir = repo / '.git'
+    module = git_dir / 'modules/library'
+    (module / 'objects').mkdir(parents=True)
+    (module / 'HEAD').write_text('ref: refs/heads/main\n')
+    (module / 'config').write_text('[core]\n\tbare = false\n[http]\n\tproxy = evil\n')
+    (module / 'refs/remotes/origin').mkdir(parents=True)
+    (module / 'refs/remotes/origin/HEAD').write_text('ref: refs/remotes/origin/main\n')
+    (git_dir / 'config').write_text('[core]\n\tbare = false\n')
+    lib.sanitize_git_config(repo)
+    assert 'proxy' not in (module / 'config').read_text()
+
+
+@pytest.mark.parametrize('name', ['library', 'refs', 'vendor/library'])
+def test_submodule_git_directory_without_objects_is_refused(tmp_path, name):
+    """A git directory which borrows another (commondir) has no objects of its own; it is refused, not skipped,
+    whatever the submodule is named."""
+    repo = tmp_path / 'repo'
+    git_dir = repo / '.git'
+    module = git_dir / 'modules' / name
+    module.mkdir(parents=True)
+    (git_dir / 'config').write_text('[core]\n\tbare = false\n')
+    (module / 'HEAD').write_text('ref: refs/heads/main\n')
+    (module / 'commondir').write_text(f'{tmp_path}\n')
+    with pytest.raises(ValueError):
+        lib.sanitize_git_config(repo)
+
+
+def test_nested_submodule_git_directories(tmp_path):
+    """A nested submodule's git directory is checked too; refs and logs named HEAD are not git directories."""
+    repo = tmp_path / 'repo'
+    git_dir = repo / '.git'
+    (git_dir).mkdir(parents=True)
+    (git_dir / 'config').write_text('[core]\n\tbare = false\n')
+    outer = git_dir / 'modules/library'
+    inner = outer / 'modules/nested'
+    for gitdir in (outer, inner):
+        (gitdir / 'objects').mkdir(parents=True)
+        (gitdir / 'HEAD').write_text('ref: refs/heads/main\n')
+        (gitdir / 'config').write_text('[core]\n\tbare = false\n[http]\n\tproxy = evil\n')
+        (gitdir / 'refs/remotes/origin').mkdir(parents=True)
+        (gitdir / 'refs/remotes/origin/HEAD').write_text('ref: refs/remotes/origin/main\n')
+        (gitdir / 'logs').mkdir()
+        (gitdir / 'logs/HEAD').write_text('0000 1111 t <t@t> 0 +0000\tclone\n')
+    lib.sanitize_git_config(repo)
+    assert 'proxy' not in (outer / 'config').read_text()
+    assert 'proxy' not in (inner / 'config').read_text()
+
+    (inner / 'commondir').write_text(f'{tmp_path}\n')
+    with pytest.raises(ValueError):
+        lib.sanitize_git_config(repo)
+
+
+
+@pytest.mark.asyncio
+async def test_copied_clone_is_not_owned(test_session, test_directory, git_remote_factory, repos_download_manager):
+    """A clone with a marker WROLPi did not write (e.g. copied from another WROLPi, or made by hand) is not owned:
+    the marker must hold this Repo's own token."""
+    remote = git_remote_factory()
+    directory = test_directory / 'repos/example'
+    directory.parent.mkdir(parents=True)
+    git(directory.parent, 'clone', '--quiet', str(remote.path), str(directory))
+    # What a marker used to hold, which anyone could write.
+    (directory / '.git' / lib.REPO_MARKER).write_text(lib.repo_url_key(remote.url))
+    (directory / 'my-work.txt').write_text('uncommitted')
+
+    lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'was not created by WROLPi' in download.error
+    assert (directory / 'my-work.txt').exists()
+
+
+@pytest.mark.asyncio
+async def test_clone_token_survives_the_config(test_session, async_client, git_remote_factory,
+                                               repos_download_manager, await_switches):
+    """The token is in repos.yaml, so a Repo restored from the config still owns its clone."""
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    await await_switches()
+    token = repo.clone_token
+    config = lib.get_repos_config()
+    assert config.repos[0]['clone_token'] == token
+
+    # Lose the database, restore it from the config.
+    lib._delete_repository(test_session, repo)
+    test_session.commit()
+    config.import_config()
+    test_session.expire_all()
+    restored = test_session.query(lib.Repository).one()
+    assert restored.clone_token == token
+
+    head = remote.commit({'new.txt': 'new'}, 'New')
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.complete, download.error
+    assert test_session.query(lib.Repository).one().head_sha == head
+
+
+@pytest.mark.parametrize('plant', ['outside', 'directory', 'link', 'garbage'])
+def test_working_tree_git_files(tmp_path, plant):
+    """A submodule's `.git` (in the working tree) decides which git directory git uses for it; it must be a file
+    pointing inside the repo's own .git/modules."""
+    repo = tmp_path / 'repo'
+    _real_submodule_git_dir(repo / '.git/modules/library')
+    (repo / 'vendor/library').mkdir(parents=True)
+    (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/library\n')
+    assert lib.working_tree_git_error(repo) is None
+
+    evil = tmp_path / 'evil'
+    evil.mkdir()
+    (repo / 'other').mkdir()
+    target = repo / 'other/.git'
+    if plant == 'outside':
+        target.write_text(f'gitdir: {evil}\n')
+    elif plant == 'directory':
+        target.mkdir()
+    elif plant == 'link':
+        target.symlink_to(evil)
+    elif plant == 'garbage':
+        target.write_text('not a gitdir line\n')
+    assert lib.working_tree_git_error(repo)
+
+
+@pytest.mark.parametrize('submodules', [True, False])
+@pytest.mark.asyncio
+async def test_update_refuses_planted_submodule_git_file(test_session, test_directory, git_remote_factory,
+                                                         repos_download_manager, submodules):
+    """Submodule commands (update, and deinit when submodules are turned off) never use a git directory outside
+    the repo."""
+    library = git_remote_factory('library')
+    remote = git_remote_factory()
+    remote.add_submodule('vendor/library', library)
+    repo = lib.create_repository(test_session, remote.url, submodules=True)
+    await repos_download_manager.wait_for_all_downloads()
+    assert (repo.directory / 'vendor/library/lib.c').exists() or (repo.directory / 'vendor/library/README.md').exists()
+
+    evil = test_directory / 'evil-gitdir'
+    evil.mkdir()
+    (repo.directory / 'vendor/library/.git').write_text(f'gitdir: {evil}\n')
+    lib.update_repository(test_session, repo.id, submodules=submodules)
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.deferred
+    assert 'outside' in download.error
+    assert list(evil.iterdir()) == []
+
+
+def _real_submodule_git_dir(path):
+    (path / 'objects').mkdir(parents=True)
+    (path / 'refs').mkdir()
+    (path / 'HEAD').write_text('ref: refs/heads/main\n')
+    (path / 'config').write_text('[core]\n\tbare = false\n')
+
+
+def test_planted_head_does_not_hide_a_submodule_git_dir(tmp_path):
+    """git finds a submodule's git directory at .git/modules/<name>, however deep; a git directory planted on a path
+    prefix (or anywhere else) cannot hide it from being sanitized."""
+    repo = tmp_path / 'repo'
+    (repo / '.git').mkdir(parents=True)
+    (repo / '.git/config').write_text('[core]\n\tbare = false\n')
+    _real_submodule_git_dir(repo / '.git/modules/vendor')
+    real = repo / '.git/modules/vendor/library'
+    _real_submodule_git_dir(real)
+    (real / 'config').write_text('[protocol "file"]\n\tallow = always\n[url "file:///tmp/evil/"]\n'
+                                 '\tinsteadOf = https://example.com/\n')
+    hidden = repo / '.git/modules/vendor/refs/library'
+    _real_submodule_git_dir(hidden)
+    (hidden / 'config').write_text('[http]\n\tproxy = evil\n')
+
+    lib.sanitize_git_config(repo)
+    assert 'protocol' not in (real / 'config').read_text()
+    assert 'insteadOf' not in (real / 'config').read_text()
+    assert 'proxy' not in (hidden / 'config').read_text()
+
+    # A submodule's `.git` must point at one of the (sanitized) git directories.
+    (repo / 'vendor/library').mkdir(parents=True)
+    (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/vendor/library\n')
+    assert lib.working_tree_git_error(repo) is None
+    (repo / 'vendor/library/.git').write_text('gitdir: ../../.git/modules/vendor/library/objects\n')
+    assert lib.working_tree_git_error(repo)
+
+
+@pytest.mark.parametrize('link', ['objects', 'refs', 'logs', 'packed-refs', 'refs/remotes', 'objects/pack'])
+@pytest.mark.asyncio
+async def test_links_in_git_dir_are_refused(test_session, test_directory, async_client, test_wrolpi_config,
+                                            git_remote_factory, repos_download_manager, link):
+    """git writes through a link in a git directory (fetch writes objects, refs and logs), so a git directory
+    holding one is neither imported nor updated."""
+    from modules.repos.errors import InvalidRepo
+    import shutil
+
+    def plant(git_dir):
+        target = git_dir / link
+        outside = test_directory / f'outside-{link.replace("/", "-")}-{git_dir.parent.name}'
+        if target.exists():
+            shutil.move(target, outside)
+        else:
+            outside.write_text('')
+        target.symlink_to(outside)
+
+    git(test_directory, 'init', '--quiet', 'code/linked')
+    git(test_directory / 'code/linked', 'commit', '--quiet', '--allow-empty', '-m', 'x')
+    git(test_directory / 'code/linked', 'pack-refs', '--all')
+    git(test_directory / 'code/linked', 'gc', '--quiet')
+    plant(test_directory / 'code/linked/.git')
+    with pytest.raises(InvalidRepo, match='link'):
+        lib.inspect_import(test_session, 'code/linked')
+
+    remote = git_remote_factory()
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    git(repo.directory, 'pack-refs', '--all')
+    git(repo.directory, 'gc', '--quiet')
+    plant(repo.directory / '.git')
+    download = await update(test_session, repos_download_manager)
+    assert download.status == DownloadStatus.deferred
+    assert 'link' in download.error
+
+
+@pytest.mark.asyncio
+async def test_control_characters_in_directories_are_refused(test_session, test_directory, async_client,
+                                                             test_wrolpi_config):
+    """A newline would split a line git reads (e.g. alternates)."""
+    from modules.repos.errors import InvalidRepo
+    for directory in ('foo\n/tmp/abs', 'foo\rbar', 'tab\there'):
+        with pytest.raises(InvalidRepo, match='character'):
+            lib.resolve_new_repo_directory(test_session, directory)
+
+
+@pytest.mark.parametrize('link_at', ['directory', 'parent'])
+@pytest.mark.asyncio
+async def test_clone_refuses_links_made_after_adding(test_session, test_directory, async_client, test_wrolpi_config,
+                                                     git_remote_factory, repos_download_manager, tmp_path_factory,
+                                                     link_at):
+    """A repo's directory (or a parent) which became a link after the repo was added is not cloned into."""
+    remote = git_remote_factory()
+    lib.create_repository(test_session, remote.url, directory='projects/example')
+    outside = tmp_path_factory.mktemp('outside')
+    if link_at == 'directory':
+        (test_directory / 'projects').mkdir()
+        (test_directory / 'projects/example').symlink_to(outside)
+    else:
+        (test_directory / 'projects').symlink_to(outside)
+    await repos_download_manager.wait_for_all_downloads()
+
+    download = test_session.query(Download).one()
+    assert download.status == DownloadStatus.deferred
+    assert 'link' in download.error
+    assert list(outside.iterdir()) == []
+
+
+@pytest.mark.asyncio
+async def test_refused_sanitize_changes_nothing(test_session, test_directory, async_client, test_wrolpi_config):
+    """Every refusal is found before any config is rewritten."""
+    from modules.repos.errors import InvalidRepo
+    git(test_directory, 'init', '--quiet', 'code/clone')
+    directory = test_directory / 'code/clone'
+    git(directory, 'commit', '--quiet', '--allow-empty', '-m', 'x')
+    git(directory, 'remote', 'add', 'origin', 'https://git.example.com/owner/clone')
+    with (directory / '.git/config').open('a') as fh:
+        fh.write('[http]\n\tproxy = http://127.0.0.1:9\n')
+    module = directory / '.git/modules/library'
+    _real_submodule_git_dir(module)
+    # Found only after the clone's own config was checked.
+    (module / 'commondir').write_text('../..\n')
+    before = (directory / '.git/config').read_text()
+
+    with pytest.raises(InvalidRepo):
+        await lib.import_repository(test_session, 'code/clone', confirm=True)
+    assert (directory / '.git/config').read_text() == before

modules/repos/test/test_search.py

@@ -0,0 +1,193 @@
+import json
+import sqlite3
+import time
+from http import HTTPStatus
+
+import pytest
+
+from modules.repos import lib
+from modules.repos.models import Repository
+
+
+def set_readme(session, repo: Repository, readme_text: str):
+    repo.readme_text = readme_text
+    session.commit()
+
+
+@pytest.mark.asyncio
+async def test_search_repos(test_session, async_client, test_wrolpi_config):
+    """Repos are found by their name, owner, and README; a name match outranks a README mention."""
+    kiwix = lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software')
+    maps = lib.create_repository(test_session, 'https://github.com/protomaps/basemaps')
+    set_readme(test_session, kiwix, 'Kiwix tools serve Zim files offline.')
+    set_readme(test_session, maps, 'Vector basemaps.  Works with kiwix-tools and offline maps.')
+
+    def names(search_str=None, tag_names=None):
+        repos, total = lib.search_repos(test_session, search_str, tag_names)
+        assert total == len(repos)
+        return [i['name'] for i in repos]
+
+    assert names('kiwix') == ['kiwix-tools', 'basemaps']
+    assert names('protomaps') == ['basemaps']  # The owner.
+    assert names('vector') == ['basemaps']
+    assert set(names('offline')) == {'kiwix-tools', 'basemaps'}
+    assert names('zim -vector') == ['kiwix-tools']
+    assert names('nothing') == []
+
+    repos, _ = lib.search_repos(test_session, 'vector')
+    assert repos[0]['readme_headline'] == '<b>Vector</b> basemaps.  Works with kiwix-tools and offline maps.'
+
+    # Tags.
+    assert names('offline', ['software']) == ['kiwix-tools']
+    assert names(tag_names=['software']) == ['kiwix-tools']
+    assert names('offline', ['software', 'other']) == []
+    assert names('offline', ['other']) == []
+
+    # Nothing to search.
+    assert names() == []
+    assert names('!!!') == []
+
+    assert lib.count_repos(test_session, 'offline') == 2
+    assert lib.count_repos(test_session, 'offline', ['software']) == 1
+    assert lib.count_repos(test_session) == 0
+
+
+@pytest.mark.asyncio
+async def test_search_index_follows_changes(test_session, test_directory, async_client, test_wrolpi_config):
+    """The search index follows README changes, renames from the config, and deletes."""
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    set_readme(test_session, repo, 'first readme')
+    assert lib.count_repos(test_session, 'first') == 1
+
+    set_readme(test_session, repo, 'second readme')
+    assert lib.count_repos(test_session, 'first') == 0
+    assert lib.count_repos(test_session, 'second') == 1
+
+    lib.ReposConfig._import_repo(test_session, dict(url='https://github.com/a/b', name='renamed'))
+    test_session.commit()
+    assert lib.count_repos(test_session, 'renamed') == 1
+
+    await lib.delete_repository(test_session, repo.id)
+    assert lib.count_repos(test_session, 'second') == 0
+    assert lib.count_repos(test_session, 'renamed') == 0
+
+    # The index matches its content.
+    with sqlite3.connect(test_directory / 'config/wrolpi.db') as conn:
+        conn.execute("INSERT INTO repository_fts(repository_fts, rank) VALUES('integrity-check', 1)")
+
+
+@pytest.mark.asyncio
+async def test_downloader_indexes_readme(test_session, git_remote_factory, repos_download_manager):
+    """The downloader stores the README as plain text, and keeps it current."""
+    readme = '# Example\n\n[![CI](https://ci/badge.svg)](https://ci) An **offline** [library](https://docs).\n'
+    remote = git_remote_factory(files={'README.md': readme})
+    repo = lib.create_repository(test_session, remote.url)
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    assert repo.readme_text == 'Example\nCI An offline library.'
+    assert lib.count_repos(test_session, 'library') == 1
+
+    remote.git('rm', '--quiet', 'README.md')
+    remote.commit({'readme.txt': 'Plain *text*'}, 'Plain README')
+    download = repo.download
+    download.renew()
+    test_session.commit()
+    await repos_download_manager.wait_for_all_downloads()
+    test_session.expire_all()
+    assert repo.readme_path == 'readme.txt'
+    assert repo.readme_text == 'Plain *text*'
+    assert lib.count_repos(test_session, 'library') == 0
+
+
+@pytest.mark.asyncio
+async def test_search_api(test_session, async_client, test_wrolpi_config):
+    kiwix = lib.create_repository(test_session, 'https://github.com/kiwix/kiwix-tools', tag_name='software')
+    set_readme(test_session, kiwix, 'Serve Zim files offline.')
+    lib.create_repository(test_session, 'https://github.com/a/b')
+
+    body = dict(search_str='zim')
+    request, response = await async_client.post('/api/repos/search', content=json.dumps(body))
+    assert response.status_code == HTTPStatus.OK
+    assert response.json['totals'] == dict(repos=1)
+    repo, = response.json['repos']
+    assert repo['name'] == 'kiwix-tools'
+    assert repo['readme_headline'] == 'Serve <b>Zim</b> files offline.'
+    assert 'readme_text' not in repo
+
+    # The global search's "Other" estimate counts Repos by the search, and by Tag.
+    request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
+    assert response.json['others'] == dict(channel_count=0, repo_count=1)
+    body = dict(tag_names=['software'])
+    request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
+    assert response.json['others'] == dict(channel_count=0, repo_count=1)
+    body = dict(tag_names=['software'], search_str='nothing')
+    request, response = await async_client.post('/api/search_other_estimates', content=json.dumps(body))
+    assert response.json['others'] == dict(channel_count=0, repo_count=0)
+
+    # Repos are suggested by name.
+    request, response = await async_client.post('/api/search_suggestions', content=json.dumps(dict(search_str='kiw')))
+    assert response.json['repos'] == [
+        dict(id=kiwix.id, name='kiwix-tools', tag_name='software', location=f'/repos/{kiwix.id}')]
+
+
+@pytest.mark.asyncio
+async def test_readme_headline_is_escaped(test_session, async_client, test_wrolpi_config):
+    """A README comes from an untrusted remote; its headline is rendered as HTML, so only the highlight is markup."""
+    repo = lib.create_repository(test_session, 'https://github.com/a/b')
+    set_readme(test_session, repo, 'zim <img src=x onerror="alert(1)"> &amp; <b>bold</b>')
+
+    repos, _ = lib.search_repos(test_session, 'zim')
+    assert repos[0]['readme_headline'] == \
+           '<b>zim</b> &lt;img src=x onerror=&quot;alert(1)&quot;&gt; &amp;amp; &lt;b&gt;bold&lt;/b&gt;'
+
+
+def test_read_readme_text_refuses_symlinks(tmp_path):
+    secret = tmp_path / 'secret.txt'
+    secret.write_text('secret')
+    readme = tmp_path / 'README.md'
+    readme.symlink_to(secret)
+    assert lib.read_readme_text(readme) is None
+    assert lib.find_readme(tmp_path) is None
+
+
+@pytest.mark.parametrize('text', [
+    pytest.param('|' + ' ' * 100_000 + 'x', id='table-spaces'),
+    pytest.param('-' * 100_000 + 'x', id='dashes'),
+    pytest.param('<!--' * 25_000, id='unclosed-comments'),
+    pytest.param('![' * 50_000, id='image-brackets'),
+    pytest.param('[' * 50_000 + '(' * 50_000, id='link-brackets'),
+    pytest.param('<' * 100_000, id='angles'),
+    pytest.param('\n' + ' \n' * 50_000 + 'x', id='blank-lines'),
+    pytest.param('#' * 100_000, id='hashes'),
+    pytest.param('> ' * 50_000, id='quotes'),
+])
+def test_markdown_to_text_is_linear(text):
+    """A README is untrusted; converting the largest README must never take long.
+
+    Each case takes milliseconds; the patterns this replaced took minutes.  The bound is loose so a busy machine
+    does not fail it."""
+    start = time.monotonic()
+    lib.markdown_to_text(text)
+    assert time.monotonic() - start < 5
+
+
+def test_markdown_to_text():
+    readme = '''# Kiwix Tools <!-- hidden
+comment -->
+
+[![CI](https://x/badge.svg)](https://ci) **Kiwix** is an *offline* reader. See [the docs](https://docs).
+
+| a | b |
+|---|---|
+| 1 | 2 |
+
+```bash
+kiwix-serve --port 80
+```
+> quoted
+[ref]: https://example.com
+---
+#hashtag
+'''
+    assert lib.markdown_to_text(readme) == ('Kiwix Tools\nCI Kiwix is an offline reader. See the docs.'
+                                            '\na b\n1 2\nkiwix-serve --port 80\nquoted\n#hashtag')

wrolpi/cmd.py

@@ -67,6 +67,7 @@ CPUFREQ_SET_BIN = which('cpufreq-set', '/usr/bin/cpufreq-set')
 
 # Files
 WGET_BIN = which('wget', '/usr/bin/wget')
+GIT_BIN = which('git', '/usr/bin/git')
 
 # Map
 BASH_BIN = which('bash', '/bin/bash')
@@ -184,7 +185,7 @@ TESTING_RUN_COMMAND_RESULT = None
 async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | str = None,
                       timeout: int = 600, log_command: bool = True,
                       stdout_callback: callable = None, env: dict = None,
-                      start_new_session: bool = False) -> CommandResult:
+                      start_new_session: bool = False, merge_stderr: bool = False) -> CommandResult:
     """Run a shell command, return the results (stdout/stderr/return code).
 
     When stdout_callback is provided, stdout is piped and each line is passed to the callback
@@ -196,6 +197,8 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
     :param start_new_session: Run the command in its own session/process group; when it finishes
         (or is killed by timeout/cancel), the whole group is killed.  Use for commands that may
         leave children behind (e.g. single-file's browser).
+    :param merge_stderr: Send stderr to stdout (and so to `stdout_callback`); `stderr` will be empty.  Use for
+        commands that write progress to stderr (e.g. git).
     """
     if not isinstance(cmd, (list, tuple)):
         raise RuntimeError('Command must be a list or tuple')
@@ -233,7 +236,7 @@ async def run_command(cmd: tuple[str | pathlib.Path, ...], cwd: pathlib.Path | s
             proc = await asyncio.create_subprocess_exec(
                 *cmd,
                 stdout=stdout_arg,
-                stderr=stderr_fh,
+                stderr=asyncio.subprocess.STDOUT if merge_stderr else stderr_fh,
                 cwd=cwd,
                 env=env,
                 start_new_session=start_new_session,

wrolpi/collections/lib.py

@@ -11,7 +11,7 @@ from sqlalchemy import func
 from sqlalchemy.orm import Session, selectinload, noload
 
 from wrolpi import flags
-from wrolpi.common import logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
+from wrolpi.common import get_media_directory, logger, get_relative_to_media_directory, TRACE_LEVEL, background_task
 from wrolpi.db import get_db_session
 from wrolpi.errors import FileWorkerConflict
 from wrolpi.errors import UnknownCollection, ValidationError
@@ -53,6 +53,9 @@ def check_config_imported(kind: str):
         # Local import to avoid circular import: collections -> videos -> collections
         from modules.videos.channel.lib import check_channels_config_imported
         check_channels_config_imported()
+    elif kind == 'repo':
+        from modules.repos.lib import get_repos_config
+        get_repos_config().check_imported()
 
 
 def _activate_config_save(kind: str):
@@ -70,6 +73,10 @@ def _activate_config_save(kind: str):
     elif kind == 'playlist':
         from .config import save_playlists_config
         save_playlists_config.activate_switch()
+    elif kind == 'repo':
+        # Local import to avoid circular import: collections -> repos -> collections
+        from modules.repos.lib import save_repos_config
+        save_repos_config.activate_switch()
 
 
 def get_collections(session: Session, kind: Optional[str] = None) -> List[dict]:
@@ -571,11 +578,16 @@ def refresh_collection(collection_id: int, send_events: bool = True) -> None:
         Events.send_directory_refresh(f'Refreshing: {relative_dir}')
 
 
-async def _background_move_collection(collection_id: int, target_directory: pathlib.Path):
+async def _background_move_collection(collection_id: int, target_directory: pathlib.Path,
+                                      old_tag_id: Optional[int] = None,
+                                      created_directories: List[pathlib.Path] = None):
     """Helper to run collection move in background with its own database session.
 
     This allows the move to continue even if the original HTTP request is cancelled
     (e.g., user closes browser tab).
+
+    A failed move leaves the Collection as it was: its directory (reverted by `move_collection`), its tag, and no
+    directories made for the move (`created_directories`, which are only removed while empty).
     """
     try:
         with get_db_session(commit=True) as session:
@@ -584,7 +596,16 @@ async def _background_move_collection(collection_id: int, target_directory: path
                 logger.error(f'_background_move_collection: collection {collection_id} not found')
                 Events.send_file_move_failed(f'Collection move failed: collection not found')
                 return
-            await collection.move_collection(target_directory, session, send_events=True)
+            try:
+                await collection.move_collection(target_directory, session, send_events=True)
+            except Exception:
+                collection.tag_id = old_tag_id
+                session.commit()
+                _activate_config_save(collection.kind)
+                for directory in sorted(created_directories or [], key=lambda i: len(i.parts), reverse=True):
+                    if directory.is_dir() and not directory.is_symlink() and not any(directory.iterdir()):
+                        directory.rmdir()
+                raise
     except Exception as e:
         logger.error(f'_background_move_collection: failed for collection_id={collection_id}', exc_info=e)
         Events.send_file_move_failed(f'Collection move failed: {e}')
@@ -616,8 +637,9 @@ async def tag_collection(
     collection = Collection.find_by_id(session, collection_id)
     check_config_imported(collection.kind)
 
-    # Track old directory before any changes for potential file moving
+    # Track old directory and tag before any changes, so a failed move can restore them.
     old_directory = collection.directory
+    old_tag_id = collection.tag_id
 
     # Determine target directory.  Collections can be tagged without a directory for UI
     # search/filtering, so a missing directory is left missing.
@@ -637,6 +659,12 @@ async def tag_collection(
         logger.trace(f'tag_collection: {repr(collection.name)} need_to_move={need_to_move}, '
                      f'{old_directory} -> {target_directory}')
 
+    if need_to_move and collection.kind == 'repo':
+        # A Repo's clone is moved as one directory; never through a link (checked before any directory is made).
+        from modules.repos.lib import repo_directory_error
+        if error := repo_directory_error(target_directory):
+            raise ValidationError(f'Cannot move the Repo: {error}')
+
     if need_to_move and flags.file_worker_busy.is_set():
         raise FileWorkerConflict('Refusing to move collection while FileWorker is busy')
 
@@ -656,10 +684,14 @@ async def tag_collection(
 
     # Move files if directory changed - run in background so closing tab won't cancel it
     if need_to_move:
+        media_directory = get_media_directory()
+        created_directories = [i for i in (target_directory, *target_directory.parents)
+                               if media_directory in i.parents and not i.exists()]
         target_directory.mkdir(parents=True, exist_ok=True)
         # Commit tag changes before starting background move
         session.commit()
-        background_task(_background_move_collection(collection.id, target_directory))
+        background_task(_background_move_collection(collection.id, target_directory, old_tag_id,
+                                                    created_directories))
 
     relative_dir = get_relative_to_media_directory(target_directory) if target_directory else None
     return {

wrolpi/collections/models.py

@@ -1,4 +1,8 @@
+import asyncio
+import errno
+import os
 import pathlib
+import shutil
 from typing import Optional, List
 
 from sqlalchemy import Column, Integer, String, ForeignKey, Text, DateTime, Index, UniqueConstraint, func, BigInteger, \
@@ -13,7 +17,7 @@ from wrolpi.db import serializer
 from wrolpi.downloader import Download, save_downloads_config
 from wrolpi.errors import ValidationError
 from wrolpi.events import Events
-from wrolpi.files.lib import delete_directory
+from wrolpi.files.lib import delete_directory, remove_files_in_ignored_directories
 from wrolpi.files.models import FileGroup
 from wrolpi.files.worker import file_worker
 from wrolpi.media_path import MediaPathType
@@ -37,11 +41,14 @@ def validate_collection_directory(directory: pathlib.Path) -> pathlib.Path:
         Normalized absolute path under media directory
 
     Raises:
-        ValidationError: If absolute path is outside media directory
+        ValidationError: If absolute path is outside media directory, or the path contains ".."
     """
     media_directory = get_media_directory()
     directory = pathlib.Path(directory)
 
+    if '..' in directory.parts:
+        # Would leave the media directory.
+        raise ValidationError(f'Collection directory cannot contain "..": {directory}')
     if not directory.is_absolute():
         # Relative path - make absolute under media directory
         directory = media_directory / directory
@@ -58,6 +65,21 @@ def validate_collection_directory(directory: pathlib.Path) -> pathlib.Path:
     return directory
 
 
+def _move_directory(source: pathlib.Path, destination: pathlib.Path):
+    """Move `source` to `destination`, which must not exist, or be an empty directory."""
+    if destination.is_dir() and not destination.is_symlink():
+        destination.rmdir()  # Raises if it is not empty.
+    elif destination.exists() or destination.is_symlink():
+        raise FileExistsError(f'Cannot move: {destination} already exists')
+    try:
+        os.rename(source, destination)
+    except OSError as e:
+        if e.errno != errno.EXDEV:
+            raise
+        # Another filesystem.
+        shutil.move(source, destination)
+
+
 class Collection(ModelHelper, Base):
     """
     A Collection is a grouping of FileGroups (videos, archives, docs, etc).
@@ -1015,6 +1037,11 @@ class Collection(ModelHelper, Base):
         """
         from wrolpi.common import get_wrolpi_config, escape_file_name
 
+        if self.kind == 'repo':
+            # Local import to avoid circular import: collections -> repos -> collections
+            from modules.repos.lib import format_collection_destination
+            return format_collection_destination(self, tag_name)
+
         config = get_wrolpi_config()
 
         if self.kind == 'channel':
@@ -1076,6 +1103,9 @@ class Collection(ModelHelper, Base):
         elif self.kind == 'channel':
             from modules.videos.lib import save_channels_config
             save_channels_config.activate_switch()
+        elif self.kind == 'repo':
+            from modules.repos.lib import save_repos_config
+            save_repos_config.activate_switch()
 
         return destination
 
@@ -1126,6 +1156,17 @@ class Collection(ModelHelper, Base):
                     file_worker.queue_refresh([old_directory, directory])
                     if send_events:
                         Events.send_file_move_completed(f'Collection {repr(self.name)} was moved (directory missing)')
+                elif self.kind == 'repo':
+                    # A Repo's files are not FileGroups (repos/ is ignored), so its clone is moved as one directory,
+                    # and indexed only as a refresh would index it.
+                    from modules.repos.lib import repo_directory_error
+                    if error := repo_directory_error(directory):
+                        raise ValidationError(f'Cannot move the Repo: {error}')
+                    await asyncio.to_thread(_move_directory, old_directory, directory)
+                    if refresh := remove_files_in_ignored_directories([old_directory, directory]):
+                        file_worker.queue_refresh(refresh)
+                    if send_events:
+                        Events.send_file_move_completed(f'Collection {repr(self.name)} was moved')
                 else:
                     files_to_move = list(old_directory.iterdir())
                     if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
@@ -1146,7 +1187,11 @@ class Collection(ModelHelper, Base):
                 raise
             finally:
                 session.commit()
-                if old_directory.exists() and not next(iter(old_directory.iterdir()), None):
+                # The move keeps each moved directory (e.g. a repo's `.git`), even when emptied; delete the old
+                # directory when only empty directories remain.
+                if old_directory.exists() and not any(not i.is_dir() for i in old_directory.rglob('*')):
+                    for directory in sorted(old_directory.rglob('*'), key=lambda i: len(i.parts), reverse=True):
+                        delete_directory(directory)
                     delete_directory(old_directory)
         else:
             # Files already moved, just send event

wrolpi/collections/test/test_domain_moving.py

@@ -488,3 +488,45 @@ async def test_tag_domain_comprehensive(
 
     # Download was not changed
     assert download.destination == original_directory, f'{download} should not have moved'
+
+
+@pytest.mark.asyncio
+async def test_failed_tag_move_reverts_tag(test_session: Session, test_directory: pathlib.Path, archive_factory,
+                                           async_client, tag_factory, await_background_tasks, monkeypatch):
+    """When the move fails, the Collection keeps its old tag (as it keeps its old directory), and the directory made
+    for the move is removed."""
+    from wrolpi.files import move
+    src_dir = test_directory / 'archive/Old/example.com'
+    src_dir.mkdir(parents=True)
+    old_tag = await tag_factory('Old')
+    await tag_factory('News')
+    collection = Collection(name='example.com', kind='domain', directory=src_dir, tag_id=old_tag.id)
+    test_session.add(collection)
+    test_session.flush([collection])
+    (src_dir / 'page.html').write_text('<html></html>')
+    test_session.commit()
+
+    async def fail(*args, **kwargs):
+        raise RuntimeError('planning failed')
+
+    monkeypatch.setattr(move, 'build_move_plan_bulk', fail)
+    body = {'tag_name': 'News', 'directory': 'archive/News/example.com'}
+    request, response = await async_client.post(f'/api/collections/{collection.id}/tag', json=body)
+    assert response.status_code == HTTPStatus.OK, response.content.decode()
+    await await_background_tasks()
+    test_session.expire_all()
+
+    assert collection.directory == src_dir
+    assert collection.tag_name == 'Old'
+    assert (src_dir / 'page.html').is_file()
+    assert not (test_directory / 'archive/News/example.com').exists()
+
+
+@pytest.mark.parametrize('directory', ['../escape', 'archive/../../escape', '{media}/../escape'])
+def test_validate_collection_directory_refuses_parent_parts(test_directory: pathlib.Path, directory):
+    """A Collection's directory cannot leave the media directory with `..`."""
+    from wrolpi.collections.models import validate_collection_directory
+    from wrolpi.errors import ValidationError
+    with pytest.raises(ValidationError):
+        validate_collection_directory(directory.format(media=test_directory))
+    assert validate_collection_directory('archive/example.com') == test_directory / 'archive/example.com'

wrolpi/collections/types.py

@@ -141,3 +141,9 @@ collection_type_registry.register(
     validate_channel_name,  # Permissive: any non-empty name.
     'Playlist name must be a non-empty string'
 )
+
+collection_type_registry.register(
+    'repo',
+    validate_channel_name,  # Permissive: any non-empty name.
+    'Repo name must be a non-empty string'
+)

wrolpi/common.py

@@ -43,7 +43,7 @@ from wrolpi.dates import now, from_timestamp
 from wrolpi.errors import WROLModeEnabled, NativeOnly, LogLevelError, InvalidConfig, ConfigNotImported, \
     ValidationError
 from wrolpi.log_levels import TRACE_LEVEL
-from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS
+from wrolpi.vars import PYTEST, DOCKERIZED, CONFIG_DIR, MEDIA_DIRECTORY, DEFAULT_HTTP_HEADERS, REPOS_DIRECTORY
 
 
 def add_logging_level(level_name: str, level_int: int, methodName=None):
@@ -445,7 +445,7 @@ def get_media_directory() -> Path:
 
 
 DB_CONFIG_FILE_NAMES = {'tags.yaml', 'channels.yaml', 'domains.yaml', 'download_manager.yaml',
-                        'playlists.yaml'}
+                        'playlists.yaml', 'repos.yaml'}
 
 
 class ConfigFile:
@@ -1117,6 +1117,7 @@ class WROLPiConfigValidator:
     map_destination: str = None
     nav_color: str = None
     playlists_destination: str = None
+    repos_destination: str = None
     require_cookies_unlocked: bool = None
     require_media_mounted: bool = None
     save_ffprobe_json: bool = None
@@ -1185,6 +1186,10 @@ def get_all_configs() -> Dict[str, ConfigFile]:
     if playlists_config := get_playlists_config():
         all_configs[playlists_config.file_name] = playlists_config
 
+    from modules.repos.lib import get_repos_config
+    if repos_config := get_repos_config():
+        all_configs[repos_config.file_name] = repos_config
+
     return all_configs
 
 
@@ -1271,6 +1276,16 @@ async def import_all_db_configs() -> dict[str, bool]:
         logger.warning(f'Failed to import playlists config: {e}')
         results['playlists'] = False
 
+    # Repos (kind='repo' collections; uses tags, links to downloads)
+    try:
+        from modules.repos.lib import import_repos_config
+        await asyncio.to_thread(import_repos_config)
+        results['repos'] = True
+        logger.debug('repos config imported')
+    except Exception as e:
+        logger.warning(f'Failed to import repos config: {e}')
+        results['repos'] = False
+
     # Map pins (YAML-only, no DB)
     try:
         from modules.map.pins import get_map_pins_config
@@ -1329,12 +1344,13 @@ class WROLPiConfig(ConfigFile):
         hotspot_ssid='WROLPi',
         check_for_upgrades=True,
         ignore_outdated_zims=False,
-        ignored_directories=['config', 'tags'],
+        ignored_directories=['config', 'tags', REPOS_DIRECTORY],
         log_level='info',
         map_default_location=None,
         map_destination='map',
         nav_color='violet',
         playlists_destination='playlists',
+        repos_destination='repos/%(repo_tag)s/%(repo_name)s',
         require_cookies_unlocked=True,
         require_media_mounted=True,
         save_ffprobe_json=True,
@@ -1381,12 +1397,24 @@ class WROLPiConfig(ConfigFile):
             self._config['zims_destination'] = self.zims_destination or self.default_config['zims_destination']
             self._config['playlists_destination'] = \
                 self.playlists_destination or self.default_config['playlists_destination']
+            self._config['repos_destination'] = self.repos_destination or self.default_config['repos_destination']
 
             # The playlists directory holds WROLPi-managed hardlinks/stubs; never index it.
             ignored = list(self._config.get('ignored_directories') or [])
             if self._config['playlists_destination'] not in ignored:
                 ignored.append(self._config['playlists_destination'])
                 self._config['ignored_directories'] = ignored
+            # Git repositories are browsed through the Repos module; their files are never indexed.  Every repo is
+            # under the fixed directory at the start of the repos destination.
+            from modules.repos.lib import validate_repos_destination, repos_destination_root
+            if error := validate_repos_destination(self._config['repos_destination'], self):
+                logger.warning(f'Using the default repos destination: {error}')
+                self._config['repos_destination'] = self.default_config['repos_destination']
+            repos_root = repos_destination_root(self._config['repos_destination'])
+            ignored = list(self._config.get('ignored_directories') or [])
+            if repos_root not in ignored:
+                ignored.append(repos_root)
+                self._config['ignored_directories'] = ignored
 
             self.successful_import = True
         except Exception as e:
@@ -1594,6 +1622,14 @@ class WROLPiConfig(ConfigFile):
     def playlists_destination(self, value: str):
         self.update({'playlists_destination': value})
 
+    @property
+    def repos_destination(self) -> str:
+        return self._config['repos_destination']
+
+    @repos_destination.setter
+    def repos_destination(self, value: str):
+        self.update({'repos_destination': value})
+
     @property
     def map_default_location(self) -> dict:
         return self._config.get('map_default_location')
@@ -2833,16 +2869,20 @@ def extract_headlines(entries: List[str], search_str: str) -> List[Tuple[str, fl
     return fts.headline_texts(entries, search_str, tokens=8)
 
 
-async def search_other_estimates(session: Session, tag_names: List[str]) -> dict:
-    """Estimate other things that are Tagged."""
+async def search_other_estimates(session: Session, tag_names: List[str], search_str: str = None) -> dict:
+    """Estimate other things that are Tagged, or match the search (Repos by their name and README)."""
     from sqlalchemy import func
     from wrolpi.collections.models import Collection
     from wrolpi.tags import Tag
     from modules.videos.models import Channel
+    from modules.repos.lib import count_repos
+
+    repo_count = count_repos(session, search_str, tag_names)
 
     if not tag_names:
         return dict(
             channel_count=0,
+            repo_count=repo_count,
         )
 
     # TODO handle multiple tags
@@ -2854,6 +2894,7 @@ async def search_other_estimates(session: Session, tag_names: List[str]) -> dict
 
     others = dict(
         channel_count=channel_count,
+        repo_count=repo_count,
     )
     return others
 

wrolpi/contexts.py

@@ -33,6 +33,7 @@ def attach_shared_contexts(app: Sanic):
     app.shared_ctx.download_manager_config = manager.dict()
     app.shared_ctx.videos_downloader_config = manager.dict()
     app.shared_ctx.domains_config = manager.dict()
+    app.shared_ctx.repos_config = manager.dict()
     app.shared_ctx.archive_downloader_config = manager.dict()
     app.shared_ctx.download_cache_config = manager.dict()
     app.shared_ctx.map_pins_config = manager.dict()
@@ -127,6 +128,7 @@ def reset_shared_contexts(app: Sanic):
     app.shared_ctx.download_manager_config.clear()
     app.shared_ctx.videos_downloader_config.clear()
     app.shared_ctx.domains_config.clear()
+    app.shared_ctx.repos_config.clear()
     app.shared_ctx.archive_downloader_config.clear()
     app.shared_ctx.download_cache_config.clear()
     # Shared dicts.
@@ -295,6 +297,12 @@ def initialize_configs_contexts(app: Sanic):
     except Exception as e:
         logger.error(f'Failed to initialize in-memory domains config: {e}')
 
+    try:
+        from modules.repos.lib import repos_config
+        repos_config.initialize(app.shared_ctx.repos_config)
+    except Exception as e:
+        logger.error(f'Failed to initialize in-memory repos config: {e}')
+
     try:
         from modules.archive.lib import ARCHIVE_DOWNLOADER_CONFIG
         ARCHIVE_DOWNLOADER_CONFIG.initialize(app.shared_ctx.archive_downloader_config)

wrolpi/downloader.py

@@ -499,7 +499,7 @@ class Downloader:
     async def process_runner(self, download: Download, cmd: Tuple[str | pathlib.Path, ...], cwd: pathlib.Path,
                              timeout: int = None, debug: bool = True,
                              stdout_callback: callable = None, env: dict = None,
-                             start_new_session: bool = False,
+                             start_new_session: bool = False, merge_stderr: bool = False,
                              ctx: 'DownloadContext' = None) -> CommandResult:
         """
         Run a subprocess using the provided arguments.  This process can be killed by the Download Manager.
@@ -522,7 +522,7 @@ class Downloader:
             timeout = self.timeout
         coro = ctx.run_command(cmd, cwd=cwd, timeout=timeout, log_command=debug,
                                stdout_callback=stdout_callback, env=env,
-                               start_new_session=start_new_session)
+                               start_new_session=start_new_session, merge_stderr=merge_stderr)
         result = await self.cancel_wrapper(coro, download, ctx=ctx)
 
         if __debug__ and logger.isEnabledFor(TRACE_LEVEL):

wrolpi/files/move.py

@@ -19,11 +19,10 @@ from sqlalchemy import text, or_
 from wrolpi import flags
 from wrolpi.common import get_media_directory, logger, walk, unique_by_predicate, chunks
 from wrolpi.db import get_db_session, get_db_curs
-from wrolpi.errors import NoPrimaryFile
 from wrolpi.events import Events
 from wrolpi.files.lib import (
     split_path_stem_and_suffix, get_unique_files_by_stem, glob_shared_stem,
-    group_files_by_stem, get_primary_file, delete_directory,
+    group_files_by_stem, delete_directory,
     _move_file_group_files, _bulk_update_file_groups_db, MOVE_CHUNK_SIZE,
     get_normalized_ignored_directories, remove_files_in_ignored_directories,
 )
@@ -212,44 +211,35 @@ async def build_move_plan_bulk(
             if missing_db_files:
                 logger.info(f'build_move_plan_bulk: creating FileGroups for {len(missing_db_files)} files not in DB')
                 for paths in group_files_by_stem(missing_db_files):
-                    try:
-                        get_primary_file(paths)
-                        fg = FileGroup.from_paths(session, *paths)
-                        session.flush()
+                    # One FileGroup per stem, as a refresh would group them (even when no file is a primary file,
+                    # e.g. dotfiles); the stem is unique in its directory.
+                    fg = FileGroup.from_paths(session, *paths)
+                    session.flush()
 
-                        # Determine destination path
-                        if any(str(p) in file_source_set for p in paths):
-                            new_path = destination / fg.primary_path.name
+                    # Determine destination path
+                    if any(str(p) in file_source_set for p in paths):
+                        new_path = destination / fg.primary_path.name
+                    else:
+                        # Find which directory source this came from
+                        source_dir = None
+                        for p in paths:
+                            if p in dir_file_mapping:
+                                source_dir = dir_file_mapping[p]
+                                break
+                        if source_dir:
+                            new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir)
                         else:
-                            # Find which directory source this came from
-                            source_dir = None
-                            for p in paths:
-                                if p in dir_file_mapping:
-                                    source_dir = dir_file_mapping[p]
-                                    break
-                            if source_dir:
-                                new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir)
-                            else:
-                                new_path = destination / fg.primary_path.name
+                            new_path = destination / fg.primary_path.name
 
-                        if new_path.exists() and fg.primary_path.exists():
-                            raise FileExistsError(f'Cannot move: {new_path} already exists')
-                        plan[fg.primary_path] = new_path
-
-                    except NoPrimaryFile:
-                        for file in paths:
-                            fg = FileGroup.from_paths(session, file)
-                            session.flush()
-                            if any(str(p) in file_source_set for p in paths):
-                                new_path = destination / fg.primary_path.name
-                            elif file in dir_file_mapping:
-                                source_dir = dir_file_mapping[file]
-                                new_path = destination / source_dir.name / fg.primary_path.relative_to(source_dir)
-                            else:
-                                new_path = destination / fg.primary_path.name
-                            if new_path.exists() and fg.primary_path.exists():
-                                raise FileExistsError(f'Cannot move: {new_path} already exists')
-                            plan[fg.primary_path] = new_path
+                    # Every file of the group is moved next to the primary (see `_move_file_group_files`); none may
+                    # replace a file at the destination.
+                    new_stem, _ = split_path_stem_and_suffix(new_path, full=True)
+                    for path in paths:
+                        _, suffix = split_path_stem_and_suffix(path)
+                        member_path = pathlib.Path(f'{new_stem}{suffix}')
+                        if member_path.exists() and path.exists():
+                            raise FileExistsError(f'Cannot move: {member_path} already exists')
+                    plan[fg.primary_path] = new_path
 
             # Add directories to plan for cleanup
             for source_dir in dir_sources:

wrolpi/files/test/test_worker_move.py

@@ -499,6 +499,50 @@ async def test_file_worker_move_unindexed_files(
     assert len(file_groups) == 2
 
 
+@pytest.mark.asyncio
+async def test_file_worker_move_unindexed_files_without_primary(
+        async_client, test_session, test_directory, make_files_structure
+):
+    """Unindexed files which share a stem, but have no primary file (e.g. dotfiles), are moved as one FileGroup,
+    as a refresh would group them."""
+    make_files_structure(['source/.gitignore', 'source/.npmrc', 'source/notes.txt', 'source/notes.md'])
+    dest = test_directory / 'destination'
+    dest.mkdir()
+
+    task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
+    file_worker.private_queue.put_nowait(task)
+    await file_worker.process_queue()
+
+    for name in ('.gitignore', '.npmrc', 'notes.txt', 'notes.md'):
+        assert (dest / 'source' / name).is_file()
+    test_session.expire_all()
+    groups = {fg.stem: sorted(i.name for i in fg.my_paths()) for fg in test_session.query(FileGroup)}
+    assert groups == {'': ['.gitignore', '.npmrc'], 'notes': ['notes.md', 'notes.txt']}
+
+
+@pytest.mark.asyncio
+async def test_file_worker_move_unindexed_group_never_replaces_a_member(
+        async_client, test_session, test_directory, make_files_structure
+):
+    """Unindexed files moved as one FileGroup never replace a file at the destination, even one which is not the
+    group's primary file."""
+    make_files_structure({'source/.gitignore': 'a', 'source/.npmrc': 'b', 'source/notes.md': 'c',
+                          'source/notes.txt': 'd'})
+    dest = test_directory / 'destination'
+    for name, content in (('.npmrc', 'kept npmrc'), ('notes.txt', 'kept notes')):
+        (dest / 'source').mkdir(parents=True, exist_ok=True)
+        (dest / 'source' / name).write_text(content)
+
+    task = FileTask(FileTaskType.move, [test_directory / 'source'], destination=dest)
+    file_worker.private_queue.put_nowait(task)
+    await file_worker.process_queue()
+
+    assert (dest / 'source/.npmrc').read_text() == 'kept npmrc'
+    assert (dest / 'source/notes.txt').read_text() == 'kept notes'
+    for name, content in (('.gitignore', 'a'), ('.npmrc', 'b'), ('notes.md', 'c'), ('notes.txt', 'd')):
+        assert (test_directory / 'source' / name).read_text() == content
+
+
 @pytest.mark.asyncio
 async def test_file_worker_move_preserves_existing_destination_on_failure(
         async_client, test_session, test_directory, make_files_structure, monkeypatch

wrolpi/fts.py

@@ -97,8 +97,47 @@ FTS_DDL = [
     ''',
 ]
 
+# A Repo's name outranks a mention in its README.
+REPOSITORY_BM25_WEIGHTS = 'bm25(10.0, 1.0)'
+
+# Installed only once the `repository` table exists (see `schema_ddl.install_raw_ddl`); the baseline migration
+# runs before that table is created.
+REPOSITORY_FTS_DDL = [
+    f'''
+    CREATE VIRTUAL TABLE IF NOT EXISTS repository_fts USING fts5(
+        search_name, readme_text,
+        content='repository',
+        content_rowid='id',
+        tokenize='{TOKENIZER}'
+    )
+    ''',
+    f'''
+    INSERT INTO repository_fts(repository_fts, rank) VALUES('rank', '{REPOSITORY_BM25_WEIGHTS}')
+    ''',
+    '''
+    CREATE TRIGGER IF NOT EXISTS repository_fts_ai AFTER INSERT ON repository BEGIN
+        INSERT INTO repository_fts(rowid, search_name, readme_text)
+        VALUES (new.id, new.search_name, new.readme_text);
+    END
+    ''',
+    '''
+    CREATE TRIGGER IF NOT EXISTS repository_fts_ad AFTER DELETE ON repository BEGIN
+        INSERT INTO repository_fts(repository_fts, rowid, search_name, readme_text)
+        VALUES ('delete', old.id, old.search_name, old.readme_text);
+    END
+    ''',
+    '''
+    CREATE TRIGGER IF NOT EXISTS repository_fts_au AFTER UPDATE OF search_name, readme_text ON repository BEGIN
+        INSERT INTO repository_fts(repository_fts, rowid, search_name, readme_text)
+        VALUES ('delete', old.id, old.search_name, old.readme_text);
+        INSERT INTO repository_fts(rowid, search_name, readme_text)
+        VALUES (new.id, new.search_name, new.readme_text);
+    END
+    ''',
+]
+
 # FTS5 shadow tables (and the virtual tables themselves); excluded from alembic autogenerate.
-FTS_TABLE_PREFIXES = ('file_group_fts', 'doc_section_fts')
+FTS_TABLE_PREFIXES = ('file_group_fts', 'doc_section_fts', 'repository_fts')
 
 
 _ITEM_RE = re.compile(r'-?"[^"]*"?|\S+')
@@ -235,6 +274,20 @@ def file_group_search_join(search_str: str, deep: bool = False, headlines: bool
                            params={param_name: match})
 
 
+def repository_search_join(search_str: str, alias: str = 'r', param_name: str = 'repo_match',
+                           start: str = '<b>', stop: str = '</b>', tokens: int = 16) -> Optional[FileGroupSearch]:
+    """Join `repository` (aliased `alias`) to its FTS5 matches.  Provides `fts.ts_rank` and `fts.readme_headline`.
+
+    Returns None when there is nothing usable to search."""
+    match = translate_websearch(search_str)
+    if match is None:
+        return None
+    join = (f"JOIN (SELECT rowid, -rank AS ts_rank, "
+            f"snippet(repository_fts, 1, '{start}', '{stop}', '…', {tokens}) AS readme_headline "
+            f"FROM repository_fts WHERE repository_fts MATCH :{param_name}) fts ON fts.rowid = {alias}.id")
+    return FileGroupSearch(join=join, where='', rank_select='fts.ts_rank AS ts_rank', params={param_name: match})
+
+
 def file_group_headline_selects(start: str = '<b>', stop: str = '</b>', tokens: int = 16) -> str:
     """SELECT fragments producing the b/c/d headline columns from the joined FTS table.
 

wrolpi/root_api.py

@@ -18,6 +18,7 @@ from modules.docs.api import docs_bp
 from modules.flasher.api import flasher_bp
 from modules.inventory import inventory_bp
 from modules.map.api import map_bp
+from modules.repos.api import repos_bp
 from modules.videos.api import videos_bp
 from modules.zim.api import zim_bp
 from wrolpi import flags, schema, dates
@@ -64,6 +65,7 @@ api_app.blueprint(flasher_bp)
 api_app.blueprint(inventory_bp)
 api_app.blueprint(jobs_bp)
 api_app.blueprint(map_bp)
+api_app.blueprint(repos_bp)
 api_app.blueprint(speedtest_bp)
 api_app.blueprint(videos_bp)
 api_app.blueprint(zim_bp)
@@ -192,6 +194,7 @@ def get_settings(_: Request):
         'nav_color': wrolpi_config.nav_color,
         'media_directory': str(get_media_directory()),  # Convert to string to avoid conversion to relative.
         'playlists_destination': wrolpi_config.playlists_destination,
+        'repos_destination': wrolpi_config.repos_destination,
         'tags_directory': wrolpi_config.tags_directory,
         'throttle_on_startup': wrolpi_config.throttle_on_startup,
         'version': __version__,
@@ -277,6 +280,14 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
     else:
         new_config['playlists_destination'] = wrolpi_config.default_config['playlists_destination']
 
+    # The Settings page may not send `repos_destination`; only validate it when it is provided.
+    if body.repos_destination is not None:
+        from modules.repos.lib import validate_repos_destination
+        if not body.repos_destination:
+            new_config['repos_destination'] = wrolpi_config.default_config['repos_destination']
+        elif error := validate_repos_destination(body.repos_destination):
+            raise InvalidConfig(error)
+
     # Validate timezone if provided; empty string clears it.
     if body.timezone is not None:
         if body.timezone == '':
@@ -313,6 +324,12 @@ async def update_settings(_: Request, body: schema.SettingsRequest):
     # Save config settings (hotspot control is now through Controller endpoints)
     wrolpi_config.update(new_config)
 
+    if body.repos_destination is not None:
+        # Repo files are never indexed, wherever repos are saved.
+        from modules.repos.lib import repos_destination_root
+        from wrolpi.files.lib import add_ignore_directory
+        add_ignore_directory(repos_destination_root(wrolpi_config.repos_destination))
+
     # If the Zim directory changed, restart Kiwix so it serves the new directory
     # without waiting for a reboot.  Local import avoids a circular import
     # (modules depend on wrolpi, not the reverse).
@@ -777,6 +794,7 @@ async def post_search_suggestions(request: Request, body: schema.SearchSuggestio
     from modules.videos.channel.lib import search_channels_by_name
     from modules.archive.lib import search_domains_by_name
     from modules.docs.lib import search_authors_by_name, search_subjects_by_name
+    from modules.repos.lib import search_repos_by_name
 
     session = request.ctx.session
     # Run sequentially: a SQLAlchemy Session is not safe for concurrent use, and
@@ -787,12 +805,14 @@ async def post_search_suggestions(request: Request, body: schema.SearchSuggestio
     domains = await search_domains_by_name(session, body.search_str)
     authors = await search_authors_by_name(session, body.search_str)
     subjects = await search_subjects_by_name(session, body.search_str)
+    repos = await search_repos_by_name(session, body.search_str)
 
     ret = dict(
         channels=channels,
         domains=domains,
         authors=authors,
         subjects=subjects,
+        repos=repos,
     )
     return json_response(ret)
 
@@ -831,7 +851,7 @@ async def post_search_file_estimates(request: Request, body: schema.SearchFileEs
 )
 async def post_search_other_estimates(request: Request, body: schema.SearchOtherEstimateRequest):
     """Used by the Global search to suggest FileGroup count to the user."""
-    others = await search_other_estimates(request.ctx.session, body.tag_names)
+    others = await search_other_estimates(request.ctx.session, body.tag_names, body.search_str)
     ret = dict(others=others)
     return json_response(ret)
 

wrolpi/schema.py

@@ -53,6 +53,7 @@ class SettingsResponse:
     archive_destination: str = 'archive/%(domain_tag)s/%(domain)s'
     map_destination: str = 'map'
     playlists_destination: str = 'playlists'
+    repos_destination: str = 'repos/%(repo_tag)s/%(repo_name)s'
     videos_destination: str = 'videos/%(channel_tag)s/%(channel_name)s'
     zims_destination: str = 'zims'
 
@@ -100,6 +101,7 @@ class SettingsRequest:
     nav_color: Optional[str] = None
     media_directory: Optional[str] = None
     playlists_destination: Optional[str] = None
+    repos_destination: Optional[str] = None
     tags_directory: Optional[bool] = None
     throttle_on: Optional[bool] = None
     throttle_on_startup: Optional[bool] = None
@@ -504,3 +506,4 @@ class SearchFileEstimateRequest:
 @dataclass
 class SearchOtherEstimateRequest:
     tag_names: List[str] = field(default_factory=list)
+    search_str: Optional[str] = None

wrolpi/schema_ddl.py

@@ -186,3 +186,12 @@ def install_raw_ddl(conn):
 
     for statement in [*TRIGGER_DDL, *fts.FTS_DDL]:
         conn.execute(statement)
+
+    # DDL for tables created after the baseline migration (which also calls this function).
+    if _table_exists(conn, 'repository'):
+        for statement in fts.REPOSITORY_FTS_DDL:
+            conn.execute(statement)
+
+
+def _table_exists(conn, name: str) -> bool:
+    return conn.execute(f"SELECT 1 FROM sqlite_master WHERE type = 'table' AND name = '{name}'").fetchone() is not None

wrolpi/test/test_bookmarks.py

@@ -175,7 +175,7 @@ def test_bookmarks_config_import(test_directory, test_wrolpi_config):
     ['not a node'],
     'not a list',
 ])
-def test_bookmarks_config_import_rejects_bad_tree(test_directory, test_wrolpi_config, bookmarks):
+async def test_bookmarks_config_import_rejects_bad_tree(async_client, test_directory, test_wrolpi_config, bookmarks):
     """A hand-edited file that the API would refuse is not imported, and is not overwritten."""
     config_file = test_directory / 'config/bookmarks.yaml'
     config_file.parent.mkdir(exist_ok=True)
@@ -195,7 +195,7 @@ def test_bookmarks_config_import_rejects_bad_tree(test_directory, test_wrolpi_co
     assert config_file.read_text() == original
 
 
-def test_bookmarks_config_import_rejects_cycle(test_directory, test_wrolpi_config):
+async def test_bookmarks_config_import_rejects_cycle(async_client, test_directory, test_wrolpi_config):
     """A YAML anchor can make a directory contain itself; walking that would never end."""
     config_file = test_directory / 'config/bookmarks.yaml'
     config_file.parent.mkdir(exist_ok=True)

wrolpi/vars.py

@@ -62,6 +62,8 @@ CIRCLECI = truthy_arg(os.environ.get('CIRCLECI', ''))
 
 # Get the media directory from the environment.
 DEFAULT_MEDIA_DIRECTORY = Path('/media/wrolpi')
+# Git repositories live here; it is always an ignored directory (repo files are never indexed).
+REPOS_DIRECTORY = 'repos'
 MEDIA_DIRECTORY = Path(os.environ.get('MEDIA_DIRECTORY', DEFAULT_MEDIA_DIRECTORY))
 
 CONFIG_DIR: Path = MEDIA_DIRECTORY / 'config'