Creating browser extension to facilitate starting downloads. Chrome/Brave and Firefox supported.

The extension is hosted in a sibling repo and
is distributed off-store. Each WROLPi release ships the prebuilt artifacts
under wrolpi/static/extensions/ and serves them at /api/extensions/<file>:

  - wrolpi-chrome.zip       Load-Unpacked sideload for Chrome / Brave / Edge
  - wrolpi-firefox.xpi      Mozilla-signed (unlisted self-distribution),
                            one-click install in any Firefox
  - versions.json           Per-browser version metadata for the install page

Backend (modules/archive/api.py, wrolpi/root_api.py):

  - POST /api/archive/upload accepts an optional repeated `tagNames`
    multipart form field; the upload switch handler applies tags to the
    new FileGroup via FileGroup.set_tags(). Unknown tag names are auto-
    created inside a per-name SAVEPOINT so concurrent inserts on the same
    novel name don't abort the transaction (Tag.name has unique=True).
  - GET /api/extensions returns metadata (per-file availability + size +
    versions.json contents) for the install page.
  - GET /api/extensions/<filename> serves the binary, whitelisted by
    filename. Firefox .xpi is served as application/x-xpinstall WITHOUT a
    Content-Disposition: attachment header, so Firefox triggers its
    add-on install handler instead of saving the file.

Frontend (app/src/):

  - New /admin/extension page (linked from AdminRoute) detects the user's
    browser via navigator.userAgent and renders an install card per
    Firefox / Chromium / Other. Pulls metadata from /api/extensions to
    show file size and version, and gracefully reports 'Not yet
    available' when binaries are missing.
  - Dashboard's Getters component supports a deep link from the
    extension's 'Configure Download' button:
      /?downloader=<archive|video|...>&download_url=<u1>&download_url=<u2>
    On mount, the matching downloader form auto-opens with URLs pre-
    filled. Query params are cleared on close so a refresh doesn't
    re-open the modal.
  - DownloadMenu and the per-downloader forms accept an `initialUrls`
    seed via the existing `download` prop pattern, joined with newlines
    for the `urls` textarea (and first URL for single-URL forms).

Behavior is fully backwards compatible: pre-existing API callers can
still POST to /api/archive/upload without `tagNames`, and the
DashboardPage modal continues to open via the existing button click
when no deep-link query params are present.
This commit is contained in:
Roland 2026-05-02 14:31:54 -06:00
parent ae460af8d6
commit a9a460c54c
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
21 changed files with 865 additions and 21 deletions

View file

@ -13,7 +13,8 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: medium
# `large` (4 vCPU, 8 GB) so pytest-xdist's `-nauto` has room to scale.
resource_class: large
steps:
- checkout
- run: sudo apt-get update
@ -30,8 +31,11 @@ jobs:
key: deps-3.11-{{ checksum "requirements.txt" }}-5
paths:
- "venv"
# `-nauto` parallelizes across all available cores via pytest-xdist.
# `-svv` was dumping the full test stdout to CircleCI logs on every run,
# which inflated wall time and produced multi-MB log artifacts; dropped.
- run:
command: './venv/bin/pytest -svv --junitxml=test-results.xml'
command: './venv/bin/pytest -nauto --junitxml=test-results.xml'
- store_test_results:
path: test-results.xml
api-alembic-migrations:
@ -71,7 +75,7 @@ jobs:
POSTGRES_USER: postgres
POSTGRES_DB: wrolpi
POSTGRES_PASSWORD: "wrolpi"
resource_class: medium
resource_class: large
steps:
- checkout
- run: sudo apt-get update
@ -89,7 +93,7 @@ jobs:
paths:
- "venv"
- run:
command: './venv/bin/pytest -svv'
command: './venv/bin/pytest -nauto'
sanic-api-start:
docker:
- image: cimg/python:3.11

View file

@ -31,6 +31,19 @@ jobs:
- name: Run Jest tests
run: CI=true npm test -- --watchAll=false --coverage --coverageReporters=text --coverageReporters=lcov
# Webpack build catches errors that Jest+Babel don't, e.g. named imports
# that resolve to `undefined` because the source module doesn't actually
# export that name (Babel turns these into runtime undefined; webpack
# rejects them at build time). GitHub Actions sets CI=true in every
# step's environment by default, which makes react-scripts treat any
# lint warning as a fatal error — so we override it here. The webpack
# build still fails on real errors (the goal of this step); only
# pre-existing lint warnings stop being blocking.
- name: Build app (catches missing imports etc.)
env:
CI: ''
run: npm run build
- name: Upload coverage report
uses: actions/upload-artifact@v4
with:

View file

@ -20,7 +20,7 @@ import {
Segment,
Statistic
} from "./components/Theme";
import {Link, useNavigate} from "react-router";
import {Link, useNavigate, useSearchParams} from "react-router";
import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status";
import {ProgressPlaceholder} from "./components/Placeholder";
import {GridColumn, GridRow, Icon, Input, Message} from "semantic-ui-react";
@ -32,6 +32,7 @@ import {Upload} from "./components/Upload";
import {SearchView, useSearch, useSearchSuggestions} from "./components/Search";
import {OutdatedZimsMessage} from "./components/Zim";
import {useSearchFilter, useSearchRecentFiles, useWROLMode} from "./hooks/customHooks";
import {ExtensionInstallSuggestion} from "./components/admin/ExtensionInstallSuggestion";
import {FileCards, FileSearchFilterButton} from "./components/Files";
import {DateSelectorButton} from "./components/DatesSelector";
import {useCalculators} from "./components/Calculators";
@ -204,11 +205,33 @@ export function Getters() {
const [selectedGetter, setSelectedGetter] = useState(null);
const gettersDisabled = status?.flags?.refresh_complete !== true;
// Deep-link support: the WROLPi browser extension (and anyone else) can open
// /?downloader=<archive|video|...>&download_url=<u1>&download_url=<u2>
// to auto-open the download modal pre-filled with a downloader and URLs.
const [searchParams, setSearchParams] = useSearchParams();
const initialDownloader = searchParams.get('downloader');
const initialUrls = searchParams.getAll('download_url');
React.useEffect(() => {
if (initialDownloader && selectedGetter !== 'downloads') {
setSelectedGetter('downloads');
}
// Run once on mount per change to query params.
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [initialDownloader]);
const handleSetGetter = (e, value) => {
if (e) {
e.preventDefault();
}
setSelectedGetter(value);
// Closing the modal? Clear deep-link params so a refresh doesn't re-open.
if (value === null && (initialDownloader || initialUrls.length)) {
const next = new URLSearchParams(searchParams);
next.delete('downloader');
next.delete('download_url');
setSearchParams(next, {replace: true});
}
}
const getter = <Segment>
@ -240,7 +263,11 @@ export function Getters() {
>
<Modal.Header>Download from the Internet</Modal.Header>
<Modal.Content>
<DownloadMenu disabled={gettersDisabled}/>
<DownloadMenu
disabled={gettersDisabled}
initialDownloader={initialDownloader}
initialUrls={initialUrls}
/>
</Modal.Content>
</Modal>;
} else if (selectedGetter === 'upload') {
@ -476,6 +503,7 @@ export function DashboardPage() {
</Grid>
</Media>
{!searchStr && <FlagsMessages/>}
{!searchStr && <ExtensionInstallSuggestion/>}
{body}
</PageContainer>
}

View file

@ -0,0 +1,78 @@
import React from 'react';
import {render, screen} from '@testing-library/react';
import {MemoryRouter} from 'react-router';
import {Getters} from './DashboardPage';
import {StatusContext, ThemeContext} from './contexts/contexts';
// Replace DownloadMenu with a stub that records its props. The deep-link
// behavior under test is just "Getters reads URL params and forwards them
// to DownloadMenu" — we don't need the real downloader forms to verify that.
jest.mock('./components/Download', () => ({
DownloadMenu: ({initialDownloader, initialUrls, disabled}) => (
<div data-testid="download-menu">
<span data-testid="initial-downloader">{initialDownloader || ''}</span>
<span data-testid="initial-urls">{JSON.stringify(initialUrls || [])}</span>
<span data-testid="disabled">{String(!!disabled)}</span>
</div>
),
}));
jest.mock('./hooks/customHooks', () => ({
...jest.requireActual('./hooks/customHooks'),
useWROLMode: () => false,
}));
function renderAtUrl(url) {
const status = {flags: {refresh_complete: true}};
const theme = {
i: {}, s: {}, t: {}, theme: 'light', inverted: false, setInverted: () => {},
};
return render(
<MemoryRouter initialEntries={[url]}>
<ThemeContext.Provider value={theme}>
<StatusContext.Provider value={{status}}>
<Getters/>
</StatusContext.Provider>
</ThemeContext.Provider>
</MemoryRouter>
);
}
describe('DashboardPage Getters deep-link', () => {
it('does not auto-open the download modal without query params', () => {
renderAtUrl('/');
expect(screen.queryByTestId('download-menu')).toBeNull();
});
it('auto-opens the modal and forwards downloader as initialDownloader', async () => {
renderAtUrl('/?downloader=archive&download_url=https%3A%2F%2Fexample.com%2Fpage');
const menu = await screen.findByTestId('download-menu');
expect(menu).toBeInTheDocument();
expect(screen.getByTestId('initial-downloader').textContent).toBe('archive');
expect(JSON.parse(screen.getByTestId('initial-urls').textContent))
.toEqual(['https://example.com/page']);
});
it('forwards multiple download_url params as an array', async () => {
renderAtUrl(
'/?downloader=video' +
'&download_url=https%3A%2F%2Fa.com' +
'&download_url=https%3A%2F%2Fb.com'
);
await screen.findByTestId('download-menu');
expect(screen.getByTestId('initial-downloader').textContent).toBe('video');
expect(JSON.parse(screen.getByTestId('initial-urls').textContent))
.toEqual(['https://a.com', 'https://b.com']);
});
it('also handles downloader=rss', async () => {
renderAtUrl('/?downloader=rss&download_url=https%3A%2F%2Ffeed.example.com');
await screen.findByTestId('download-menu');
expect(screen.getByTestId('initial-downloader').textContent).toBe('rss');
});
});
// The DownloadMenu's seed-object construction (joining initialUrls with
// newlines into the `download` prop's `urls` field) is exercised in the
// browser via manual verification, not unit-tested here — reaching into the
// per-form trees pulls in heavy fetch-driven components.

View file

@ -1639,8 +1639,12 @@ export function EditScrapeFilesDownloadForm({download, onDelete, onCancel, onSuc
/>
}
export function DownloadMenu({onOpen, disabled}) {
const [downloader, setDownloader] = useState();
export function DownloadMenu({onOpen, disabled, initialDownloader, initialUrls}) {
// initialDownloader and initialUrls support the deep-link flow used by the
// WROLPi browser extension's "Configure Download" button: visiting
// `/?downloader=archive&download_url=...` pre-selects a form and pre-fills
// its URLs textarea via DashboardPage.Getters.
const [downloader, setDownloader] = useState(initialDownloader || undefined);
const localOnOpen = (name) => {
setDownloader(name);
@ -1698,13 +1702,21 @@ export function DownloadMenu({onOpen, disabled}) {
body = null;
}
// Synthesize a partial `download` prop that seeds the form's URL field(s).
// Multi-URL forms read `urls` (newline-separated string); the single-URL
// ChannelDownloadForm reads `url`. Setting both is harmless — each form's
// emptyFormData defines which field actually exists.
const seed = (initialUrls && initialUrls.length)
? {urls: initialUrls.join('\n'), url: initialUrls[0]}
: undefined;
const downloaders = {
archive: <ArchiveDownloadForm onCancel={clearSelected}/>,
video: <VideosDownloadForm singleDownload={false} onCancel={clearSelected}/>,
video_channel: <ChannelDownloadForm onCancel={clearSelected}/>,
rss: <RSSDownloadForm onCancel={clearSelected}/>,
file: <FilesDownloadForm onCancel={clearSelected}/>,
scrape: <ScrapeFilesDownloadForm onCancel={clearSelected}/>
archive: <ArchiveDownloadForm onCancel={clearSelected} download={seed}/>,
video: <VideosDownloadForm singleDownload={false} onCancel={clearSelected} download={seed}/>,
video_channel: <ChannelDownloadForm onCancel={clearSelected} download={seed}/>,
rss: <RSSDownloadForm onCancel={clearSelected} download={seed}/>,
file: <FilesDownloadForm onCancel={clearSelected} download={seed}/>,
scrape: <ScrapeFilesDownloadForm onCancel={clearSelected} download={seed}/>
};
if (downloader in downloaders) {

View file

@ -5,6 +5,7 @@ import {DownloadsPage} from "./Downloads";
import {SettingsPage} from "./Settings";
import {StatusPage} from "./Status";
import {ControllerPage} from "./ControllerPage";
import {ExtensionPage} from "./ExtensionPage";
export default function AdminRoute() {
@ -13,6 +14,7 @@ export default function AdminRoute() {
{text: 'Settings', to: '/admin/settings', key: 'settings'},
{text: 'Status', to: '/admin/status', key: 'status'},
{text: 'Control', to: '/admin/controller', key: 'controller'},
{text: 'Extension', to: '/admin/extension', key: 'extension'},
];
return <PageContainer>
@ -22,6 +24,7 @@ export default function AdminRoute() {
<Route path='settings' exact element={<SettingsPage/>}/>
<Route path='status' exact element={<StatusPage/>}/>
<Route path='controller' exact element={<ControllerPage/>}/>
<Route path='extension' exact element={<ExtensionPage/>}/>
</Routes>
</PageContainer>
}

View file

@ -0,0 +1,38 @@
import React from "react";
import {Link} from "react-router";
import {Button, Icon, Message} from "semantic-ui-react";
import {useLocalStorage} from "../Common";
import {useExtensionInstalled} from "../../hooks/customHooks";
// Top-of-dashboard nudge for users who don't yet have the WROLPi browser
// extension installed (or installed but haven't added this WROLPi as a
// destination). Hidden once the user installs+configures, or dismisses it.
//
// Detection is via a meta tag injected by the extension's content script.
// The extension only injects on configured destination origins, so seeing
// the marker here means "extension is installed AND knows about this WROLPi".
export function ExtensionInstallSuggestion() {
const installed = useExtensionInstalled();
const [dismissed, setDismissed] = useLocalStorage(
'wrolpi-extension-banner-dismissed', false,
);
// installed === null means "still checking" — render nothing rather
// than flashing the banner before the content-script race resolves.
if (installed !== false || dismissed) return null;
return <Message info icon onDismiss={() => setDismissed(true)}>
<Icon name='puzzle piece'/>
<Message.Content>
<Message.Header>Install the WROLPi browser extension</Message.Header>
<p>
Send pages, videos, and feeds to this WROLPi straight from any
tab. Open-source and not distributed via the official browser
stores.
</p>
<Button as={Link} to='/admin/extension' primary size='small'>
Open install page
</Button>
</Message.Content>
</Message>;
}

View file

@ -0,0 +1,48 @@
import React from 'react';
import {render, screen, waitFor} from '../../test-utils';
import {ExtensionInstallSuggestion} from './ExtensionInstallSuggestion';
const setMeta = (version) => {
const existing = document.querySelector('meta[name="wrolpi-extension"]');
if (existing) existing.remove();
if (version === undefined) return;
const meta = document.createElement('meta');
meta.name = 'wrolpi-extension';
meta.content = String(version);
document.head.appendChild(meta);
};
describe('ExtensionInstallSuggestion', () => {
beforeEach(() => {
setMeta(undefined);
window.localStorage.removeItem('wrolpi-extension-banner-dismissed');
});
it('renders nothing when the extension marker meta is present', async () => {
setMeta('0.2.2');
render(<ExtensionInstallSuggestion/>);
// Give the hook a moment in case its initial detect() races; should
// still find the marker on first render.
await waitFor(() => {
expect(screen.queryByText(/Install the WROLPi browser extension/i)).toBeNull();
});
});
it('renders the install message when the marker is absent', async () => {
render(<ExtensionInstallSuggestion/>);
// The hook re-checks once at 500ms before deciding "not installed".
// Wait the recheck out, then assert.
await waitFor(
() => expect(screen.getByText(/Install the WROLPi browser extension/i)).toBeInTheDocument(),
{timeout: 1500},
);
expect(screen.getByText(/Open install page/i)).toBeInTheDocument();
});
it('renders nothing when the user has dismissed the banner', async () => {
window.localStorage.setItem('wrolpi-extension-banner-dismissed', 'true');
render(<ExtensionInstallSuggestion/>);
await new Promise(r => setTimeout(r, 600));
expect(screen.queryByText(/Install the WROLPi browser extension/i)).toBeNull();
});
});

View file

@ -0,0 +1,202 @@
import React from "react";
import {Button, Header, Icon, Segment} from "../Theme";
import {Input, Message} from "semantic-ui-react";
// The WROLPi browser extension is distributed off-store: Chrome users
// sideload the unpacked .zip; Firefox users get a Mozilla-signed .xpi that
// installs in one click. Both binaries are served by the local WROLPi
// backend so this page works on a fully air-gapped install.
const CHROME_ZIP_PATH = '/api/extensions/wrolpi-chrome.zip';
const FIREFOX_XPI_PATH = '/api/extensions/wrolpi-firefox.xpi';
function detectBrowser() {
if (typeof navigator === 'undefined') return 'unknown';
const ua = navigator.userAgent || '';
// Order matters: Edge / Brave / Opera all include 'Chrome' in their UA.
if (ua.includes('Firefox')) return 'firefox';
if (ua.includes('Edg/')) return 'chromium';
if (ua.includes('OPR/') || ua.includes('Opera')) return 'chromium';
if (ua.includes('Chrome')) return 'chromium';
return 'unknown';
}
function formatSize(bytes) {
if (!bytes) return '';
const kb = bytes / 1024;
if (kb < 1024) return `${kb.toFixed(0)} KB`;
return `${(kb / 1024).toFixed(2)} MB`;
}
function DestinationHint() {
// The user is currently viewing this WROLPi at exactly window.location.origin.
// That's the URL they need to paste into the extension's destination field.
const origin = (typeof window !== 'undefined' && window.location && window.location.origin) || '';
const [copied, setCopied] = React.useState(false);
const onCopy = async () => {
try {
await navigator.clipboard.writeText(origin);
setCopied(true);
setTimeout(() => setCopied(false), 1500);
} catch {
// Older / non-secure contexts may block the clipboard API.
// Fall through silently — the user can still copy manually.
}
};
return <Segment style={{marginBottom: '1em'}}>
<Header as='h4'>
<Icon name='globe'/>
This WROLPi's URL
</Header>
<p style={{color: 'grey'}}>
After installing the extension, paste this URL into the
destination field on the extension's options page.
</p>
<Input
fluid
readOnly
value={origin}
action={
<Button
icon={copied ? 'check' : 'copy'}
color={copied ? 'green' : null}
content={copied ? 'Copied' : 'Copy'}
onClick={onCopy}
/>
}
/>
</Segment>;
}
function FirefoxCard({metadata, version}) {
const file = (metadata && metadata.files && metadata.files['wrolpi-firefox.xpi']) || {};
const available = !!file.available;
return <Segment>
<Header as='h3'>
<Icon name='firefox'/>
Firefox{version ? ` — v${version}` : ''}
</Header>
<p>One-click install. Mozilla has signed this build.</p>
<Button
primary
as='a'
href={FIREFOX_XPI_PATH}
disabled={!available}
icon='download'
content={available ? `Install for Firefox (${formatSize(file.size_bytes)})` : 'Not yet available'}
/>
<p style={{marginTop: '1em', color: 'grey'}}>
Firefox will ask you to confirm the install. Accept the prompt.
</p>
<Header as='h4' style={{marginTop: '1.5em'}}>After installing</Header>
<ol style={{lineHeight: 1.7, paddingLeft: '1.2em'}}>
<li>Open the extension's <strong>Options</strong> page (right-click the toolbar icon &rarr; Manage Extension &rarr; Options) and add this WROLPi as a destination. Click <strong>Test</strong> to grant API access.</li>
<li>Click the extension icon in the toolbar &rarr; the permission dropdown &rarr; <strong>Always Allow on this site</strong>. Firefox requires this second click in addition to the Test grant — without it, the extension can talk to your WROLPi but can't mark its tabs as connected.</li>
</ol>
</Segment>;
}
function ChromiumCard({metadata, version}) {
const file = (metadata && metadata.files && metadata.files['wrolpi-chrome.zip']) || {};
const available = !!file.available;
return <Segment>
<Header as='h3'>
<Icon name='chrome'/>
Chrome / Brave / Edge{version ? ` — v${version}` : ''}
</Header>
<p>
Chromium-based browsers don't allow one-click extension installs from
outside the Web Store, so we sideload via the built-in developer flow.
</p>
<Button
primary
as='a'
href={CHROME_ZIP_PATH}
disabled={!available}
icon='download'
content={available ? `Download .zip (${formatSize(file.size_bytes)})` : 'Not yet available'}
download
/>
<Header as='h4' style={{marginTop: '1.5em'}}>Install steps</Header>
<ol style={{lineHeight: 1.7, paddingLeft: '1.2em'}}>
<li>Download the <code>.zip</code> above and extract it to a folder you'll keep around.</li>
<li>Open <code>chrome://extensions</code> (or <code>brave://extensions</code> / <code>edge://extensions</code>).</li>
<li>Toggle <strong>Developer mode</strong> on (top-right).</li>
<li>Click <strong>Load unpacked</strong> and pick the extracted folder.</li>
</ol>
<p style={{color: 'grey'}}>
The extension stays installed across browser restarts. Don't delete the
extracted folder — the browser loads from it directly.
</p>
</Segment>;
}
function UnknownCard() {
return <Segment>
<Header as='h3'>Unsupported browser</Header>
<p>
The WROLPi extension supports Chrome, Brave, Edge, and Firefox. Open
this page in one of those browsers to install.
</p>
</Segment>;
}
export function ExtensionPage() {
const [metadata, setMetadata] = React.useState(null);
const [error, setError] = React.useState(null);
const browser = detectBrowser();
React.useEffect(() => {
let cancelled = false;
fetch('/api/extensions')
.then(res => res.json())
.then(body => { if (!cancelled) setMetadata(body); })
.catch(err => { if (!cancelled) setError(String(err)); });
return () => { cancelled = true; };
}, []);
if (error) {
return <Segment>
<Message negative>
<Message.Header>Couldn't load extension info</Message.Header>
<p>{error}</p>
</Message>
</Segment>;
}
const versions = (metadata && metadata.versions) || {};
const anyAvailable = metadata && Object.values(metadata.files || {}).some(f => f.available);
return <>
<Header as='h1'>Browser Extension</Header>
<p>
Install the WROLPi browser extension to send pages, videos, and feeds to
this WROLPi from any tab. The extension is open-source and not distributed
via official browser stores — it ships with each WROLPi release.
</p>
<DestinationHint/>
{metadata && !anyAvailable && <Message warning>
<Message.Header>Extension binaries not yet installed</Message.Header>
<p>
The extension binaries aren't on this WROLPi yet. They ship with the
next release; pull the latest and they'll appear here.
</p>
</Message>}
{browser === 'firefox' && <FirefoxCard metadata={metadata} version={versions.firefox}/>}
{browser === 'chromium' && <ChromiumCard metadata={metadata} version={versions.chrome}/>}
{browser === 'unknown' && <UnknownCard/>}
<Segment basic style={{marginTop: '2em'}}>
<Header as='h4'>Both browsers, just in case</Header>
<p style={{color: 'grey'}}>
You can install for a different browser too — useful if you sync
between machines.
</p>
{browser !== 'firefox' && <FirefoxCard metadata={metadata} version={versions.firefox}/>}
{browser !== 'chromium' && <ChromiumCard metadata={metadata} version={versions.chrome}/>}
</Segment>
</>;
}

View file

@ -0,0 +1,89 @@
import React from 'react';
import {mockFetch, render, screen, waitFor} from '../../test-utils';
import {ExtensionPage} from './ExtensionPage';
const FIREFOX_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.15; rv:128.0) Gecko/20100101 Firefox/128.0';
const CHROME_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/147.0.0.0 Safari/537.36';
const SAFARI_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.0 Safari/605.1.15';
const setUserAgent = (ua) => {
Object.defineProperty(window.navigator, 'userAgent', {value: ua, configurable: true});
};
describe('ExtensionPage', () => {
let originalFetch;
beforeEach(() => {
originalFetch = global.fetch;
});
afterEach(() => {
global.fetch = originalFetch;
});
it('shows the Firefox install card when the user agent is Firefox', async () => {
setUserAgent(FIREFOX_UA);
global.fetch = mockFetch({
files: {
'wrolpi-firefox.xpi': {available: true, size_bytes: 700000},
'wrolpi-chrome.zip': {available: true, size_bytes: 700000},
},
versions: {firefox: '0.1.0', chrome: '0.1.0'},
});
render(<ExtensionPage/>);
const button = await screen.findByText(/Install for Firefox/i);
expect(button).toBeInTheDocument();
});
it('shows the Chromium install card with install steps when the user agent is Chrome', async () => {
setUserAgent(CHROME_UA);
global.fetch = mockFetch({
files: {
'wrolpi-firefox.xpi': {available: true, size_bytes: 700000},
'wrolpi-chrome.zip': {available: true, size_bytes: 700000},
},
versions: {chrome: '0.1.0', firefox: '0.1.0'},
});
render(<ExtensionPage/>);
expect(await screen.findByText(/Download \.zip/i)).toBeInTheDocument();
expect(screen.getByText(/Developer mode/i)).toBeInTheDocument();
expect(screen.getByText(/Load unpacked/i)).toBeInTheDocument();
});
it('shows the not-yet-installed warning when no binaries are available', async () => {
// This test exists specifically to render the <Message warning> block,
// which catches missing imports from the Theme/Semantic-UI module.
setUserAgent(CHROME_UA);
global.fetch = mockFetch({
files: {
'wrolpi-firefox.xpi': {available: false, size_bytes: null},
'wrolpi-chrome.zip': {available: false, size_bytes: null},
},
versions: {},
});
render(<ExtensionPage/>);
expect(await screen.findByText(/Extension binaries not yet installed/i)).toBeInTheDocument();
});
it('shows the unsupported-browser message for Safari', async () => {
setUserAgent(SAFARI_UA);
global.fetch = mockFetch({
files: {
'wrolpi-firefox.xpi': {available: true, size_bytes: 700000},
'wrolpi-chrome.zip': {available: true, size_bytes: 700000},
},
versions: {},
});
render(<ExtensionPage/>);
expect(await screen.findByText(/Unsupported browser/i)).toBeInTheDocument();
});
});

View file

@ -1850,3 +1850,54 @@ export const useSearchChannels = (defaultTagNames) => {
loading,
}
}
/**
* Detect whether the WROLPi browser extension is installed AND has this
* destination configured. The extension's background worker dynamically
* registers a content script on each configured destination origin; that
* script appends `<meta name="wrolpi-extension" content="<version>">` to the
* head. We read for that marker.
*
* Three-state result so callers can avoid flashing UI during the race
* between React's first render and the content script's document_idle:
* null — checking; render nothing
* false — confirmed absent (after recheck); show "install me" UI
* <string> — installed; the version string the extension reported
*
* Observes <head> for a late-arriving marker (e.g., the user installs the
* extension while the page is open, or grants the destination permission
* after page load) so the banner clears without requiring a manual reload.
*/
export function useExtensionInstalled() {
const detect = () => {
const meta = document.querySelector('meta[name="wrolpi-extension"]');
if (!meta) return null;
return meta.getAttribute('content') || true;
};
const [state, setState] = useState(detect);
useEffect(() => {
// Observe <head> for the marker meta tag appearing at any time.
// Once we see it we clean up — there's no need to keep watching.
const observer = new MutationObserver(() => {
const result = detect();
if (result) setState(result);
});
if (document.head) {
observer.observe(document.head, {childList: true, subtree: true});
}
return () => observer.disconnect();
}, []);
useEffect(() => {
if (state !== null) return;
// Recheck once after the document_idle window closes; only then
// commit to "false". Avoids flashing the banner on every dashboard
// load while we're just waiting for the content script to run.
const t = setTimeout(() => setState(detect() ?? false), 500);
return () => clearTimeout(t);
}, [state]);
return state;
}

View file

@ -5,6 +5,7 @@ from http import HTTPStatus
from sanic import response, Request, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from sqlalchemy.exc import IntegrityError
from wrolpi.api_utils import json_response, api_app
from wrolpi.cmd import get_installed_browsers
@ -136,6 +137,8 @@ async def singlefile_upload_switch_handler(url=None):
"""Used by `post_upload_singlefile` to upload a single file"""
# Local imports to avoid circular import: downloader -> archive -> api -> downloader
from wrolpi.downloader import Download
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
from . import ArchiveDownloader
q: multiprocessing.Queue = api_app.shared_ctx.archive_singlefiles
@ -144,12 +147,19 @@ async def singlefile_upload_switch_handler(url=None):
if trace_enabled:
logger.trace(f'singlefile_upload_switch_handler called for {url}')
try:
singlefile = q.get_nowait()
payload = q.get_nowait()
except queue.Empty:
if trace_enabled:
logger.trace(f'singlefile_upload_switch_handler called on empty queue')
return
# Payload is (singlefile_bytes, tag_names) tuple. Older callers may have put raw bytes;
# tolerate that shape so an in-flight queue across an upgrade does not crash.
if isinstance(payload, tuple):
singlefile, tag_names = payload
else:
singlefile, tag_names = payload, []
try:
q_size = q.qsize()
except NotImplementedError:
@ -174,6 +184,32 @@ async def singlefile_upload_switch_handler(url=None):
Events.send_upload_archive_failed(f'Failed to convert singlefile to archive: {e}')
raise
if tag_names:
# Apply tags to the new FileGroup. Auto-create unknown tag names so any
# API caller (the browser extension does not, but third-party callers
# might) can introduce a new tag at upload time.
#
# Each tag insert sits inside a SAVEPOINT so a concurrent caller
# racing on the same novel name (Tag.name has unique=True) doesn't
# abort the whole transaction — the loser of the race rolls back its
# own savepoint, set_tags() re-resolves the name, and finds the row
# the winner just committed.
file_group_id = archive.file_group.id
with get_db_session(commit=True) as session:
for name in tag_names:
if Tag.get_by_name(session, name):
continue
try:
with session.begin_nested():
session.add(Tag(name=name))
session.flush()
except IntegrityError:
# Another writer created the tag first. Tag.name's unique
# constraint did its job; set_tags() below will find it.
pass
fg = session.query(FileGroup).filter_by(id=file_group_id).one()
fg.set_tags(session, list(tag_names))
name = archive.file_group.title or archive.file_group.url
logger.info(f'Created Archive from upload ({q_size}): {archive}')
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
@ -242,13 +278,16 @@ generate_screenshot_switch_handler: ActivateSwitchMethod
async def post_upload_singlefile(request: Request):
url = request.form['url'][0]
singlefile = request.files['singlefile_contents'][0].body
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
# Optional repeated form field; matches the convention used by /api/files/upload.
tag_names = [str(i) for i in (request.form.getlist('tagNames') or [])]
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url} '
f'with {len(tag_names)} tag(s)')
# Extract the URL to ensure that the singlefile is valid.
lib.get_url_from_singlefile(singlefile)
# Send processing to background task so the extension can continue.
api_app.shared_ctx.archive_singlefiles.put(singlefile)
api_app.shared_ctx.archive_singlefiles.put((singlefile, tag_names))
singlefile_upload_switch_handler.activate_switch(context=dict(url=url))
# Return empty json response because SingleFile extension expects a JSON response.
return json_response(dict(), status=HTTPStatus.OK)

View file

@ -207,6 +207,72 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
events_fixture.assert_has_event('upload_archive')
@pytest.mark.asyncio
async def test_archive_upload_with_tags(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, tag_factory, archive_factory):
"""Uploading with tagNames applies existing tags and auto-creates new ones.
This test mocks `singlefile_to_archive` so it can run on every platform
(the full archive pipeline depends on a readability/screenshot service
that is not available on macOS). The mocked Archive stands in for what
the real pipeline would have produced; the assertion verifies that the
switch handler applies tags to whichever FileGroup the pipeline returns.
"""
existing = await tag_factory('preexisting')
archive = archive_factory('example.com', 'https://example.com/tagged', 'tagged')
test_session.commit()
async def fake_singlefile_to_archive(singlefile, destination=None):
return archive
singlefile_contents = singlefile_contents_factory(title='tagged', url='https://example.com/tagged')
forms = [
dict(name='url', value='https://example.com/tagged'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents'),
dict(name='tagNames', value=existing.name),
dict(name='tagNames', value='brand-new-tag'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
with patch('modules.archive.lib.singlefile_to_archive', side_effect=fake_singlefile_to_archive):
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert sorted(archive.file_group.tag_names) == ['brand-new-tag', 'preexisting']
@pytest.mark.asyncio
async def test_archive_upload_without_tags_unchanged(test_session, async_client, singlefile_contents_factory,
make_multipart_form, await_switches, archive_factory):
"""Uploads from clients that don't send tagNames (e.g. upstream SingleFile) still work."""
archive = archive_factory('example.com', 'https://example.com/no-tags', 'no-tags')
test_session.commit()
async def fake_singlefile_to_archive(singlefile, destination=None):
return archive
singlefile_contents = singlefile_contents_factory(title='no-tags', url='https://example.com/no-tags')
forms = [
dict(name='url', value='https://example.com/no-tags'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
with patch('modules.archive.lib.singlefile_to_archive', side_effect=fake_singlefile_to_archive):
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.file_group.tag_names == []
@pytest.mark.asyncio
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
await_switches, make_multipart_form, image_bytes_factory):

View file

@ -1937,12 +1937,16 @@ async def test_channel_reorganization_preserves_root_level_files(
config._config['yt_dlp_options'] = yt_dlp_options
def test_preview_computes_actual_moves(test_session, test_directory, archive_factory):
async def test_preview_computes_actual_moves(async_client, test_session, test_directory, archive_factory):
"""Preview should compute actual files needing move by building move mappings.
This test verifies that:
- Preview correctly identifies files in wrong location
- After reorganization, preview shows 0 files needing move
Uses the `async_client` fixture so the Sanic app's shared_ctx (including
file_worker_jobs) is initialized before file_worker.process_queue() runs.
Without it, file_worker._jobs raises AttributeError on the missing key.
"""
# Create domain collection with a year-based format (different from default)
domain_dir = test_directory / 'archive' / 'exactcount.com'
@ -1989,8 +1993,7 @@ def test_preview_computes_actual_moves(test_session, test_directory, archive_fac
assert job_id, "Should return job_id"
file_worker.transfer_queue()
import asyncio
asyncio.get_event_loop().run_until_complete(file_worker.process_queue())
await file_worker.process_queue()
# After reorganization, should show 0 files needing move
with get_db_session() as session:

View file

@ -1,4 +1,5 @@
import asyncio
import json
import pathlib
import re
from http import HTTPStatus
@ -767,3 +768,57 @@ async def post_search_other_estimates(_: Request, body: schema.SearchOtherEstima
others = await search_other_estimates(body.tag_names)
ret = dict(others=others)
return json_response(ret)
# Browser extension distribution.
# Files are committed to wrolpi/static/extensions/ and served as binary
# downloads. The whitelist prevents path traversal and limits exposure to
# only the two filenames the install page knows about.
EXTENSION_DIR = pathlib.Path(__file__).parent / 'static' / 'extensions'
EXTENSION_FILES = {
'wrolpi-chrome.zip': 'application/zip',
# Firefox click-to-install requires this exact Content-Type.
'wrolpi-firefox.xpi': 'application/x-xpinstall',
}
@api_bp.get('/extensions')
@openapi.description('Metadata for the WROLPi browser extension binaries.')
async def get_extensions_metadata(_: Request):
versions = {}
versions_path = EXTENSION_DIR / 'versions.json'
if versions_path.is_file():
try:
versions = json.loads(versions_path.read_text())
except json.JSONDecodeError:
logger.warning(f'Could not parse {versions_path}')
files = {}
for name in EXTENSION_FILES:
path = EXTENSION_DIR / name
files[name] = {
'available': path.is_file(),
'size_bytes': path.stat().st_size if path.is_file() else None,
}
return json_response(dict(files=files, versions=versions))
@api_bp.get('/extensions/<filename:str>')
@openapi.description('Download a WROLPi browser extension binary.')
async def get_extension_file(_: Request, filename: str):
if filename not in EXTENSION_FILES:
return response.json({'error': f'Unknown extension file: {filename}'}, status=HTTPStatus.NOT_FOUND)
path = EXTENSION_DIR / filename
if not path.is_file():
return response.json({'error': f'Extension file not installed: {filename}'},
status=HTTPStatus.NOT_FOUND)
mime_type = EXTENSION_FILES[filename]
headers = {}
# Firefox click-to-install requires application/x-xpinstall AND that we do
# NOT send Content-Disposition: attachment — the latter forces the browser
# into "save file" mode, bypassing the add-on install handler. The Chrome
# zip still wants attachment so the browser writes a file instead of
# navigating away from the page.
if mime_type != 'application/x-xpinstall':
headers['Content-Disposition'] = f'attachment; filename="{filename}"'
return await response.file(str(path), mime_type=mime_type, headers=headers)

View file

@ -0,0 +1,17 @@
# Browser extension binaries
This directory holds the prebuilt WROLPi browser extension binaries that the
backend serves at `/api/extensions/`. The frontend's `/admin/extension` page
serves them to end users.
| File | Browser | Source |
|----------------------|---------------------------------------|----------------------------------------------------|
| `wrolpi-chrome.zip` | Chrome / Brave / Edge (Load Unpacked) | `wrolpi-extension` repo, `bash scripts/release.sh` |
| `wrolpi-firefox.xpi` | Firefox (one-click install) | Mozilla-signed via AMO unlisted self-distribution |
| `versions.json` | metadata | hand-edited per release |
See `wrolpi-extension/RELEASING.md` for the full release procedure.
The signed `.xpi` is not committed in the initial release — it must be
produced by uploading the unsigned bundle to AMO once. The install page
gracefully shows "Not yet available" if a binary is missing.

View file

@ -0,0 +1,4 @@
{
"chrome": "0.2.3",
"firefox": "0.2.3"
}

Binary file not shown.

Binary file not shown.

View file

@ -0,0 +1,94 @@
"""Tests for the /api/extensions endpoints that distribute the WROLPi browser extension binaries."""
import json
from http import HTTPStatus
import pytest
from wrolpi import root_api
@pytest.fixture
def extension_dir(tmp_path, monkeypatch):
"""Point root_api.EXTENSION_DIR at a temp directory for the test."""
monkeypatch.setattr(root_api, 'EXTENSION_DIR', tmp_path)
return tmp_path
@pytest.mark.asyncio
async def test_extensions_metadata_no_files(test_session, async_client, extension_dir):
"""Metadata endpoint reports unavailable when no binaries are installed."""
request, response = await async_client.get('/api/extensions')
assert response.status_code == HTTPStatus.OK
body = response.json
assert body['files']['wrolpi-chrome.zip']['available'] is False
assert body['files']['wrolpi-firefox.xpi']['available'] is False
assert body['files']['wrolpi-chrome.zip']['size_bytes'] is None
assert body['versions'] == {}
@pytest.mark.asyncio
async def test_extensions_metadata_with_files(test_session, async_client, extension_dir):
"""Metadata endpoint reports each file's size and parses versions.json."""
(extension_dir / 'wrolpi-chrome.zip').write_bytes(b'PK\x03\x04' + b'fake-chrome' * 100)
(extension_dir / 'wrolpi-firefox.xpi').write_bytes(b'PK\x03\x04' + b'fake-xpi' * 100)
(extension_dir / 'versions.json').write_text(json.dumps({'chrome': '0.1.0', 'firefox': '0.1.0'}))
request, response = await async_client.get('/api/extensions')
assert response.status_code == HTTPStatus.OK
body = response.json
assert body['files']['wrolpi-chrome.zip']['available'] is True
assert body['files']['wrolpi-chrome.zip']['size_bytes'] > 0
assert body['files']['wrolpi-firefox.xpi']['available'] is True
assert body['versions'] == {'chrome': '0.1.0', 'firefox': '0.1.0'}
@pytest.mark.asyncio
async def test_extensions_download_chrome_zip(test_session, async_client, extension_dir):
"""Chrome zip is served with application/zip and an attachment Content-Disposition."""
payload = b'PK\x03\x04' + b'\x00' * 1024
(extension_dir / 'wrolpi-chrome.zip').write_bytes(payload)
request, response = await async_client.get('/api/extensions/wrolpi-chrome.zip')
assert response.status_code == HTTPStatus.OK
assert response.headers.get('content-type') == 'application/zip'
assert 'attachment' in response.headers.get('content-disposition', '')
assert 'wrolpi-chrome.zip' in response.headers.get('content-disposition', '')
assert response.body == payload
@pytest.mark.asyncio
async def test_extensions_download_firefox_xpi_content_type(test_session, async_client, extension_dir):
"""Firefox .xpi must be served as application/x-xpinstall for click-to-install to work."""
payload = b'PK\x03\x04' + b'\x00' * 1024
(extension_dir / 'wrolpi-firefox.xpi').write_bytes(payload)
request, response = await async_client.get('/api/extensions/wrolpi-firefox.xpi')
assert response.status_code == HTTPStatus.OK
assert response.headers.get('content-type') == 'application/x-xpinstall'
# Firefox click-to-install requires the response NOT carry an
# `attachment` Content-Disposition; otherwise the browser saves the file
# instead of invoking the add-on install handler.
assert 'attachment' not in response.headers.get('content-disposition', '').lower()
@pytest.mark.asyncio
async def test_extensions_download_missing_file(test_session, async_client, extension_dir):
"""Whitelisted-but-not-installed file returns 404 with a clear message."""
request, response = await async_client.get('/api/extensions/wrolpi-chrome.zip')
assert response.status_code == HTTPStatus.NOT_FOUND
assert 'not installed' in response.json['error']
@pytest.mark.asyncio
async def test_extensions_download_unknown_file(test_session, async_client, extension_dir):
"""Non-whitelisted filename returns 404 (path-traversal protection)."""
# Even though the file might exist on disk via traversal, the whitelist blocks it.
request, response = await async_client.get('/api/extensions/something-else.zip')
assert response.status_code == HTTPStatus.NOT_FOUND
@pytest.mark.asyncio
async def test_extensions_download_path_traversal(test_session, async_client, extension_dir):
"""Path-traversal attempts are rejected by the whitelist."""
request, response = await async_client.get('/api/extensions/..%2F..%2Fetc%2Fpasswd')
assert response.status_code == HTTPStatus.NOT_FOUND

View file

@ -1 +1 @@
0.21.2-beta
0.21.3-beta