From a9a460c54c2390767f04e59d693b92f9fc800dbe Mon Sep 17 00:00:00 2001 From: Roland Date: Sat, 2 May 2026 14:31:54 -0600 Subject: [PATCH] Creating browser extension to facilitate starting downloads. Chrome/Brave and Firefox supported. The extension is hosted in a sibling repo and is distributed off-store. Each WROLPi release ships the prebuilt artifacts under wrolpi/static/extensions/ and serves them at /api/extensions/: - wrolpi-chrome.zip Load-Unpacked sideload for Chrome / Brave / Edge - wrolpi-firefox.xpi Mozilla-signed (unlisted self-distribution), one-click install in any Firefox - versions.json Per-browser version metadata for the install page Backend (modules/archive/api.py, wrolpi/root_api.py): - POST /api/archive/upload accepts an optional repeated `tagNames` multipart form field; the upload switch handler applies tags to the new FileGroup via FileGroup.set_tags(). Unknown tag names are auto- created inside a per-name SAVEPOINT so concurrent inserts on the same novel name don't abort the transaction (Tag.name has unique=True). - GET /api/extensions returns metadata (per-file availability + size + versions.json contents) for the install page. - GET /api/extensions/ serves the binary, whitelisted by filename. Firefox .xpi is served as application/x-xpinstall WITHOUT a Content-Disposition: attachment header, so Firefox triggers its add-on install handler instead of saving the file. Frontend (app/src/): - New /admin/extension page (linked from AdminRoute) detects the user's browser via navigator.userAgent and renders an install card per Firefox / Chromium / Other. Pulls metadata from /api/extensions to show file size and version, and gracefully reports 'Not yet available' when binaries are missing. - Dashboard's Getters component supports a deep link from the extension's 'Configure Download' button: /?downloader=&download_url=&download_url= On mount, the matching downloader form auto-opens with URLs pre- filled. Query params are cleared on close so a refresh doesn't re-open the modal. - DownloadMenu and the per-downloader forms accept an `initialUrls` seed via the existing `download` prop pattern, joined with newlines for the `urls` textarea (and first URL for single-URL forms). Behavior is fully backwards compatible: pre-existing API callers can still POST to /api/archive/upload without `tagNames`, and the DashboardPage modal continues to open via the existing button click when no deep-link query params are present. --- .circleci/config.yml | 12 +- .github/workflows/jest-tests.yml | 13 ++ app/src/DashboardPage.js | 32 ++- app/src/DashboardPage.test.js | 78 +++++++ app/src/components/Download.js | 28 ++- app/src/components/admin/AdminRoute.js | 3 + .../admin/ExtensionInstallSuggestion.js | 38 ++++ .../admin/ExtensionInstallSuggestion.test.js | 48 +++++ app/src/components/admin/ExtensionPage.js | 202 ++++++++++++++++++ .../components/admin/ExtensionPage.test.js | 89 ++++++++ app/src/hooks/customHooks.js | 51 +++++ modules/archive/api.py | 45 +++- modules/archive/test/test_api.py | 66 ++++++ wrolpi/collections/test/test_reorganize.py | 9 +- wrolpi/root_api.py | 55 +++++ wrolpi/static/extensions/README.md | 17 ++ wrolpi/static/extensions/versions.json | 4 + wrolpi/static/extensions/wrolpi-chrome.zip | Bin 0 -> 302658 bytes wrolpi/static/extensions/wrolpi-firefox.xpi | Bin 0 -> 311867 bytes wrolpi/test/test_extensions.py | 94 ++++++++ wrolpi/version.txt | 2 +- 21 files changed, 865 insertions(+), 21 deletions(-) create mode 100644 app/src/DashboardPage.test.js create mode 100644 app/src/components/admin/ExtensionInstallSuggestion.js create mode 100644 app/src/components/admin/ExtensionInstallSuggestion.test.js create mode 100644 app/src/components/admin/ExtensionPage.js create mode 100644 app/src/components/admin/ExtensionPage.test.js create mode 100644 wrolpi/static/extensions/README.md create mode 100644 wrolpi/static/extensions/versions.json create mode 100644 wrolpi/static/extensions/wrolpi-chrome.zip create mode 100644 wrolpi/static/extensions/wrolpi-firefox.xpi create mode 100644 wrolpi/test/test_extensions.py diff --git a/.circleci/config.yml b/.circleci/config.yml index b7f13ac8..55b8366b 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -13,7 +13,8 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: medium + # `large` (4 vCPU, 8 GB) so pytest-xdist's `-nauto` has room to scale. + resource_class: large steps: - checkout - run: sudo apt-get update @@ -30,8 +31,11 @@ jobs: key: deps-3.11-{{ checksum "requirements.txt" }}-5 paths: - "venv" + # `-nauto` parallelizes across all available cores via pytest-xdist. + # `-svv` was dumping the full test stdout to CircleCI logs on every run, + # which inflated wall time and produced multi-MB log artifacts; dropped. - run: - command: './venv/bin/pytest -svv --junitxml=test-results.xml' + command: './venv/bin/pytest -nauto --junitxml=test-results.xml' - store_test_results: path: test-results.xml api-alembic-migrations: @@ -71,7 +75,7 @@ jobs: POSTGRES_USER: postgres POSTGRES_DB: wrolpi POSTGRES_PASSWORD: "wrolpi" - resource_class: medium + resource_class: large steps: - checkout - run: sudo apt-get update @@ -89,7 +93,7 @@ jobs: paths: - "venv" - run: - command: './venv/bin/pytest -svv' + command: './venv/bin/pytest -nauto' sanic-api-start: docker: - image: cimg/python:3.11 diff --git a/.github/workflows/jest-tests.yml b/.github/workflows/jest-tests.yml index b189c466..03dd7300 100644 --- a/.github/workflows/jest-tests.yml +++ b/.github/workflows/jest-tests.yml @@ -31,6 +31,19 @@ jobs: - name: Run Jest tests run: CI=true npm test -- --watchAll=false --coverage --coverageReporters=text --coverageReporters=lcov + # Webpack build catches errors that Jest+Babel don't, e.g. named imports + # that resolve to `undefined` because the source module doesn't actually + # export that name (Babel turns these into runtime undefined; webpack + # rejects them at build time). GitHub Actions sets CI=true in every + # step's environment by default, which makes react-scripts treat any + # lint warning as a fatal error — so we override it here. The webpack + # build still fails on real errors (the goal of this step); only + # pre-existing lint warnings stop being blocking. + - name: Build app (catches missing imports etc.) + env: + CI: '' + run: npm run build + - name: Upload coverage report uses: actions/upload-artifact@v4 with: diff --git a/app/src/DashboardPage.js b/app/src/DashboardPage.js index b4215fca..681444a1 100644 --- a/app/src/DashboardPage.js +++ b/app/src/DashboardPage.js @@ -20,7 +20,7 @@ import { Segment, Statistic } from "./components/Theme"; -import {Link, useNavigate} from "react-router"; +import {Link, useNavigate, useSearchParams} from "react-router"; import {BandwidthProgressCombined, CPUUsageProgress} from "./components/admin/Status"; import {ProgressPlaceholder} from "./components/Placeholder"; import {GridColumn, GridRow, Icon, Input, Message} from "semantic-ui-react"; @@ -32,6 +32,7 @@ import {Upload} from "./components/Upload"; import {SearchView, useSearch, useSearchSuggestions} from "./components/Search"; import {OutdatedZimsMessage} from "./components/Zim"; import {useSearchFilter, useSearchRecentFiles, useWROLMode} from "./hooks/customHooks"; +import {ExtensionInstallSuggestion} from "./components/admin/ExtensionInstallSuggestion"; import {FileCards, FileSearchFilterButton} from "./components/Files"; import {DateSelectorButton} from "./components/DatesSelector"; import {useCalculators} from "./components/Calculators"; @@ -204,11 +205,33 @@ export function Getters() { const [selectedGetter, setSelectedGetter] = useState(null); const gettersDisabled = status?.flags?.refresh_complete !== true; + // Deep-link support: the WROLPi browser extension (and anyone else) can open + // /?downloader=&download_url=&download_url= + // to auto-open the download modal pre-filled with a downloader and URLs. + const [searchParams, setSearchParams] = useSearchParams(); + const initialDownloader = searchParams.get('downloader'); + const initialUrls = searchParams.getAll('download_url'); + + React.useEffect(() => { + if (initialDownloader && selectedGetter !== 'downloads') { + setSelectedGetter('downloads'); + } + // Run once on mount per change to query params. + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [initialDownloader]); + const handleSetGetter = (e, value) => { if (e) { e.preventDefault(); } setSelectedGetter(value); + // Closing the modal? Clear deep-link params so a refresh doesn't re-open. + if (value === null && (initialDownloader || initialUrls.length)) { + const next = new URLSearchParams(searchParams); + next.delete('downloader'); + next.delete('download_url'); + setSearchParams(next, {replace: true}); + } } const getter = @@ -240,7 +263,11 @@ export function Getters() { > Download from the Internet - + ; } else if (selectedGetter === 'upload') { @@ -476,6 +503,7 @@ export function DashboardPage() { {!searchStr && } + {!searchStr && } {body} } diff --git a/app/src/DashboardPage.test.js b/app/src/DashboardPage.test.js new file mode 100644 index 00000000..d6b21623 --- /dev/null +++ b/app/src/DashboardPage.test.js @@ -0,0 +1,78 @@ +import React from 'react'; +import {render, screen} from '@testing-library/react'; +import {MemoryRouter} from 'react-router'; +import {Getters} from './DashboardPage'; +import {StatusContext, ThemeContext} from './contexts/contexts'; + +// Replace DownloadMenu with a stub that records its props. The deep-link +// behavior under test is just "Getters reads URL params and forwards them +// to DownloadMenu" — we don't need the real downloader forms to verify that. +jest.mock('./components/Download', () => ({ + DownloadMenu: ({initialDownloader, initialUrls, disabled}) => ( +
+ {initialDownloader || ''} + {JSON.stringify(initialUrls || [])} + {String(!!disabled)} +
+ ), +})); + +jest.mock('./hooks/customHooks', () => ({ + ...jest.requireActual('./hooks/customHooks'), + useWROLMode: () => false, +})); + +function renderAtUrl(url) { + const status = {flags: {refresh_complete: true}}; + const theme = { + i: {}, s: {}, t: {}, theme: 'light', inverted: false, setInverted: () => {}, + }; + return render( + + + + + + + + ); +} + +describe('DashboardPage Getters deep-link', () => { + it('does not auto-open the download modal without query params', () => { + renderAtUrl('/'); + expect(screen.queryByTestId('download-menu')).toBeNull(); + }); + + it('auto-opens the modal and forwards downloader as initialDownloader', async () => { + renderAtUrl('/?downloader=archive&download_url=https%3A%2F%2Fexample.com%2Fpage'); + const menu = await screen.findByTestId('download-menu'); + expect(menu).toBeInTheDocument(); + expect(screen.getByTestId('initial-downloader').textContent).toBe('archive'); + expect(JSON.parse(screen.getByTestId('initial-urls').textContent)) + .toEqual(['https://example.com/page']); + }); + + it('forwards multiple download_url params as an array', async () => { + renderAtUrl( + '/?downloader=video' + + '&download_url=https%3A%2F%2Fa.com' + + '&download_url=https%3A%2F%2Fb.com' + ); + await screen.findByTestId('download-menu'); + expect(screen.getByTestId('initial-downloader').textContent).toBe('video'); + expect(JSON.parse(screen.getByTestId('initial-urls').textContent)) + .toEqual(['https://a.com', 'https://b.com']); + }); + + it('also handles downloader=rss', async () => { + renderAtUrl('/?downloader=rss&download_url=https%3A%2F%2Ffeed.example.com'); + await screen.findByTestId('download-menu'); + expect(screen.getByTestId('initial-downloader').textContent).toBe('rss'); + }); +}); + +// The DownloadMenu's seed-object construction (joining initialUrls with +// newlines into the `download` prop's `urls` field) is exercised in the +// browser via manual verification, not unit-tested here — reaching into the +// per-form trees pulls in heavy fetch-driven components. diff --git a/app/src/components/Download.js b/app/src/components/Download.js index 63175306..e9497320 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -1639,8 +1639,12 @@ export function EditScrapeFilesDownloadForm({download, onDelete, onCancel, onSuc /> } -export function DownloadMenu({onOpen, disabled}) { - const [downloader, setDownloader] = useState(); +export function DownloadMenu({onOpen, disabled, initialDownloader, initialUrls}) { + // initialDownloader and initialUrls support the deep-link flow used by the + // WROLPi browser extension's "Configure Download" button: visiting + // `/?downloader=archive&download_url=...` pre-selects a form and pre-fills + // its URLs textarea via DashboardPage.Getters. + const [downloader, setDownloader] = useState(initialDownloader || undefined); const localOnOpen = (name) => { setDownloader(name); @@ -1698,13 +1702,21 @@ export function DownloadMenu({onOpen, disabled}) { body = null; } + // Synthesize a partial `download` prop that seeds the form's URL field(s). + // Multi-URL forms read `urls` (newline-separated string); the single-URL + // ChannelDownloadForm reads `url`. Setting both is harmless — each form's + // emptyFormData defines which field actually exists. + const seed = (initialUrls && initialUrls.length) + ? {urls: initialUrls.join('\n'), url: initialUrls[0]} + : undefined; + const downloaders = { - archive: , - video: , - video_channel: , - rss: , - file: , - scrape: + archive: , + video: , + video_channel: , + rss: , + file: , + scrape: }; if (downloader in downloaders) { diff --git a/app/src/components/admin/AdminRoute.js b/app/src/components/admin/AdminRoute.js index 7996cfc9..3d5b4295 100644 --- a/app/src/components/admin/AdminRoute.js +++ b/app/src/components/admin/AdminRoute.js @@ -5,6 +5,7 @@ import {DownloadsPage} from "./Downloads"; import {SettingsPage} from "./Settings"; import {StatusPage} from "./Status"; import {ControllerPage} from "./ControllerPage"; +import {ExtensionPage} from "./ExtensionPage"; export default function AdminRoute() { @@ -13,6 +14,7 @@ export default function AdminRoute() { {text: 'Settings', to: '/admin/settings', key: 'settings'}, {text: 'Status', to: '/admin/status', key: 'status'}, {text: 'Control', to: '/admin/controller', key: 'controller'}, + {text: 'Extension', to: '/admin/extension', key: 'extension'}, ]; return @@ -22,6 +24,7 @@ export default function AdminRoute() { }/> }/> }/> + }/> } diff --git a/app/src/components/admin/ExtensionInstallSuggestion.js b/app/src/components/admin/ExtensionInstallSuggestion.js new file mode 100644 index 00000000..bd83df8c --- /dev/null +++ b/app/src/components/admin/ExtensionInstallSuggestion.js @@ -0,0 +1,38 @@ +import React from "react"; +import {Link} from "react-router"; +import {Button, Icon, Message} from "semantic-ui-react"; +import {useLocalStorage} from "../Common"; +import {useExtensionInstalled} from "../../hooks/customHooks"; + +// Top-of-dashboard nudge for users who don't yet have the WROLPi browser +// extension installed (or installed but haven't added this WROLPi as a +// destination). Hidden once the user installs+configures, or dismisses it. +// +// Detection is via a meta tag injected by the extension's content script. +// The extension only injects on configured destination origins, so seeing +// the marker here means "extension is installed AND knows about this WROLPi". +export function ExtensionInstallSuggestion() { + const installed = useExtensionInstalled(); + const [dismissed, setDismissed] = useLocalStorage( + 'wrolpi-extension-banner-dismissed', false, + ); + + // installed === null means "still checking" — render nothing rather + // than flashing the banner before the content-script race resolves. + if (installed !== false || dismissed) return null; + + return setDismissed(true)}> + + + Install the WROLPi browser extension +

+ Send pages, videos, and feeds to this WROLPi straight from any + tab. Open-source and not distributed via the official browser + stores. +

+ + + ; +} diff --git a/app/src/components/admin/ExtensionInstallSuggestion.test.js b/app/src/components/admin/ExtensionInstallSuggestion.test.js new file mode 100644 index 00000000..13d44ead --- /dev/null +++ b/app/src/components/admin/ExtensionInstallSuggestion.test.js @@ -0,0 +1,48 @@ +import React from 'react'; +import {render, screen, waitFor} from '../../test-utils'; +import {ExtensionInstallSuggestion} from './ExtensionInstallSuggestion'; + +const setMeta = (version) => { + const existing = document.querySelector('meta[name="wrolpi-extension"]'); + if (existing) existing.remove(); + if (version === undefined) return; + const meta = document.createElement('meta'); + meta.name = 'wrolpi-extension'; + meta.content = String(version); + document.head.appendChild(meta); +}; + +describe('ExtensionInstallSuggestion', () => { + beforeEach(() => { + setMeta(undefined); + window.localStorage.removeItem('wrolpi-extension-banner-dismissed'); + }); + + it('renders nothing when the extension marker meta is present', async () => { + setMeta('0.2.2'); + render(); + // Give the hook a moment in case its initial detect() races; should + // still find the marker on first render. + await waitFor(() => { + expect(screen.queryByText(/Install the WROLPi browser extension/i)).toBeNull(); + }); + }); + + it('renders the install message when the marker is absent', async () => { + render(); + // The hook re-checks once at 500ms before deciding "not installed". + // Wait the recheck out, then assert. + await waitFor( + () => expect(screen.getByText(/Install the WROLPi browser extension/i)).toBeInTheDocument(), + {timeout: 1500}, + ); + expect(screen.getByText(/Open install page/i)).toBeInTheDocument(); + }); + + it('renders nothing when the user has dismissed the banner', async () => { + window.localStorage.setItem('wrolpi-extension-banner-dismissed', 'true'); + render(); + await new Promise(r => setTimeout(r, 600)); + expect(screen.queryByText(/Install the WROLPi browser extension/i)).toBeNull(); + }); +}); diff --git a/app/src/components/admin/ExtensionPage.js b/app/src/components/admin/ExtensionPage.js new file mode 100644 index 00000000..52851811 --- /dev/null +++ b/app/src/components/admin/ExtensionPage.js @@ -0,0 +1,202 @@ +import React from "react"; +import {Button, Header, Icon, Segment} from "../Theme"; +import {Input, Message} from "semantic-ui-react"; + +// The WROLPi browser extension is distributed off-store: Chrome users +// sideload the unpacked .zip; Firefox users get a Mozilla-signed .xpi that +// installs in one click. Both binaries are served by the local WROLPi +// backend so this page works on a fully air-gapped install. + +const CHROME_ZIP_PATH = '/api/extensions/wrolpi-chrome.zip'; +const FIREFOX_XPI_PATH = '/api/extensions/wrolpi-firefox.xpi'; + +function detectBrowser() { + if (typeof navigator === 'undefined') return 'unknown'; + const ua = navigator.userAgent || ''; + // Order matters: Edge / Brave / Opera all include 'Chrome' in their UA. + if (ua.includes('Firefox')) return 'firefox'; + if (ua.includes('Edg/')) return 'chromium'; + if (ua.includes('OPR/') || ua.includes('Opera')) return 'chromium'; + if (ua.includes('Chrome')) return 'chromium'; + return 'unknown'; +} + +function formatSize(bytes) { + if (!bytes) return ''; + const kb = bytes / 1024; + if (kb < 1024) return `${kb.toFixed(0)} KB`; + return `${(kb / 1024).toFixed(2)} MB`; +} + +function DestinationHint() { + // The user is currently viewing this WROLPi at exactly window.location.origin. + // That's the URL they need to paste into the extension's destination field. + const origin = (typeof window !== 'undefined' && window.location && window.location.origin) || ''; + const [copied, setCopied] = React.useState(false); + const onCopy = async () => { + try { + await navigator.clipboard.writeText(origin); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + } catch { + // Older / non-secure contexts may block the clipboard API. + // Fall through silently — the user can still copy manually. + } + }; + return +
+ + This WROLPi's URL +
+

+ After installing the extension, paste this URL into the + destination field on the extension's options page. +

+ + } + /> + ; +} + +function FirefoxCard({metadata, version}) { + const file = (metadata && metadata.files && metadata.files['wrolpi-firefox.xpi']) || {}; + const available = !!file.available; + return +
+ + Firefox{version ? ` — v${version}` : ''} +
+

One-click install. Mozilla has signed this build.

+