rt/wrolpi
1
0
Fork 0
mirror of https://github.com/lrnselfreliance/wrolpi.git synced 2026-10-26 02:48:38 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions 5

Creating browser extension to facilitate starting downloads. Chrome/Brave and Firefox supported.

Browse source 
The extension is hosted in a sibling repo and
is distributed off-store. Each WROLPi release ships the prebuilt artifacts
under wrolpi/static/extensions/ and serves them at /api/extensions/<file>:

  - wrolpi-chrome.zip       Load-Unpacked sideload for Chrome / Brave / Edge
  - wrolpi-firefox.xpi      Mozilla-signed (unlisted self-distribution),
                            one-click install in any Firefox
  - versions.json           Per-browser version metadata for the install page

Backend (modules/archive/api.py, wrolpi/root_api.py):

  - POST /api/archive/upload accepts an optional repeated `tagNames`
    multipart form field; the upload switch handler applies tags to the
    new FileGroup via FileGroup.set_tags(). Unknown tag names are auto-
    created inside a per-name SAVEPOINT so concurrent inserts on the same
    novel name don't abort the transaction (Tag.name has unique=True).
  - GET /api/extensions returns metadata (per-file availability + size +
    versions.json contents) for the install page.
  - GET /api/extensions/<filename> serves the binary, whitelisted by
    filename. Firefox .xpi is served as application/x-xpinstall WITHOUT a
    Content-Disposition: attachment header, so Firefox triggers its
    add-on install handler instead of saving the file.

Frontend (app/src/):

  - New /admin/extension page (linked from AdminRoute) detects the user's
    browser via navigator.userAgent and renders an install card per
    Firefox / Chromium / Other. Pulls metadata from /api/extensions to
    show file size and version, and gracefully reports 'Not yet
    available' when binaries are missing.
  - Dashboard's Getters component supports a deep link from the
    extension's 'Configure Download' button:
      /?downloader=<archive|video|...>&download_url=<u1>&download_url=<u2>
    On mount, the matching downloader form auto-opens with URLs pre-
    filled. Query params are cleared on close so a refresh doesn't
    re-open the modal.
  - DownloadMenu and the per-downloader forms accept an `initialUrls`
    seed via the existing `download` prop pattern, joined with newlines
    for the `urls` textarea (and first URL for single-URL forms).

Behavior is fully backwards compatible: pre-existing API callers can
still POST to /api/archive/upload without `tagNames`, and the
DashboardPage modal continues to open via the existing button click
when no deep-link query params are present.
This commit is contained in:
Roland 2026-05-02 14:31:54 -06:00
parent ae460af8d6
commit a9a460c54c
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
21 changed files with 865 additions and 21 deletions
Download patch file Download diff file Expand all files Collapse all files

45
modules/archive/api.py
View file

@ -5,6 +5,7 @@ from http import HTTPStatus
from sanic import response, Request, Blueprint
from sanic_ext import validate
from sanic_ext.extensions.openapi import openapi
from sqlalchemy.exc import IntegrityError
from wrolpi.api_utils import json_response, api_app
from wrolpi.cmd import get_installed_browsers
@ -136,6 +137,8 @@ async def singlefile_upload_switch_handler(url=None):
"""Used by `post_upload_singlefile` to upload a single file"""
# Local imports to avoid circular import: downloader -> archive -> api -> downloader
from wrolpi.downloader import Download
from wrolpi.files.models import FileGroup
from wrolpi.tags import Tag
from . import ArchiveDownloader
q: multiprocessing.Queue = api_app.shared_ctx.archive_singlefiles
@ -144,12 +147,19 @@ async def singlefile_upload_switch_handler(url=None):
if trace_enabled:
logger.trace(f'singlefile_upload_switch_handler called for {url}')
try:
singlefile = q.get_nowait()
payload = q.get_nowait()
except queue.Empty:
if trace_enabled:
logger.trace(f'singlefile_upload_switch_handler called on empty queue')
return
# Payload is (singlefile_bytes, tag_names) tuple. Older callers may have put raw bytes;
# tolerate that shape so an in-flight queue across an upgrade does not crash.
if isinstance(payload, tuple):
singlefile, tag_names = payload
else:
singlefile, tag_names = payload, []
try:
q_size = q.qsize()
except NotImplementedError:
@ -174,6 +184,32 @@ async def singlefile_upload_switch_handler(url=None):
Events.send_upload_archive_failed(f'Failed to convert singlefile to archive: {e}')
raise
if tag_names:
# Apply tags to the new FileGroup. Auto-create unknown tag names so any
# API caller (the browser extension does not, but third-party callers
# might) can introduce a new tag at upload time.
#
# Each tag insert sits inside a SAVEPOINT so a concurrent caller
# racing on the same novel name (Tag.name has unique=True) doesn't
# abort the whole transaction — the loser of the race rolls back its
# own savepoint, set_tags() re-resolves the name, and finds the row
# the winner just committed.
file_group_id = archive.file_group.id
with get_db_session(commit=True) as session:
for name in tag_names:
if Tag.get_by_name(session, name):
continue
try:
with session.begin_nested():
session.add(Tag(name=name))
session.flush()
except IntegrityError:
# Another writer created the tag first. Tag.name's unique
# constraint did its job; set_tags() below will find it.
pass
fg = session.query(FileGroup).filter_by(id=file_group_id).one()
fg.set_tags(session, list(tag_names))
name = archive.file_group.title or archive.file_group.url
logger.info(f'Created Archive from upload ({q_size}): {archive}')
Events.send_archive_uploaded(f'Created Archive from upload: {name}', url=archive.location)
@ -242,13 +278,16 @@ generate_screenshot_switch_handler: ActivateSwitchMethod
async def post_upload_singlefile(request: Request):
url = request.form['url'][0]
singlefile = request.files['singlefile_contents'][0].body
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url}')
# Optional repeated form field; matches the convention used by /api/files/upload.
tag_names = [str(i) for i in (request.form.getlist('tagNames') or [])]
logger.info(f'Got Archive upload of {len(singlefile)} bytes for URL: {url} '
f'with {len(tag_names)} tag(s)')
# Extract the URL to ensure that the singlefile is valid.
lib.get_url_from_singlefile(singlefile)
# Send processing to background task so the extension can continue.
api_app.shared_ctx.archive_singlefiles.put(singlefile)
api_app.shared_ctx.archive_singlefiles.put((singlefile, tag_names))
singlefile_upload_switch_handler.activate_switch(context=dict(url=url))
# Return empty json response because SingleFile extension expects a JSON response.
return json_response(dict(), status=HTTPStatus.OK)

66
modules/archive/test/test_api.py
View file

@ -207,6 +207,72 @@ async def test_archive_upload(test_session, async_client, singlefile_contents_fa
events_fixture.assert_has_event('upload_archive')
@pytest.mark.asyncio
async def test_archive_upload_with_tags(test_session, async_client, singlefile_contents_factory, make_multipart_form,
await_switches, tag_factory, archive_factory):
"""Uploading with tagNames applies existing tags and auto-creates new ones.
This test mocks `singlefile_to_archive` so it can run on every platform
(the full archive pipeline depends on a readability/screenshot service
that is not available on macOS). The mocked Archive stands in for what
the real pipeline would have produced; the assertion verifies that the
switch handler applies tags to whichever FileGroup the pipeline returns.
"""
existing = await tag_factory('preexisting')
archive = archive_factory('example.com', 'https://example.com/tagged', 'tagged')
test_session.commit()
async def fake_singlefile_to_archive(singlefile, destination=None):
return archive
singlefile_contents = singlefile_contents_factory(title='tagged', url='https://example.com/tagged')
forms = [
dict(name='url', value='https://example.com/tagged'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents'),
dict(name='tagNames', value=existing.name),
dict(name='tagNames', value='brand-new-tag'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
with patch('modules.archive.lib.singlefile_to_archive', side_effect=fake_singlefile_to_archive):
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert sorted(archive.file_group.tag_names) == ['brand-new-tag', 'preexisting']
@pytest.mark.asyncio
async def test_archive_upload_without_tags_unchanged(test_session, async_client, singlefile_contents_factory,
make_multipart_form, await_switches, archive_factory):
"""Uploads from clients that don't send tagNames (e.g. upstream SingleFile) still work."""
archive = archive_factory('example.com', 'https://example.com/no-tags', 'no-tags')
test_session.commit()
async def fake_singlefile_to_archive(singlefile, destination=None):
return archive
singlefile_contents = singlefile_contents_factory(title='no-tags', url='https://example.com/no-tags')
forms = [
dict(name='url', value='https://example.com/no-tags'),
dict(name='singlefile_contents', value=singlefile_contents, filename='singlefile_contents'),
]
body = make_multipart_form(forms)
headers = {'Content-Type': 'multipart/form-data; boundary=-----------------------------sanic'}
with patch('modules.archive.lib.singlefile_to_archive', side_effect=fake_singlefile_to_archive):
request, response = await async_client.post('/api/archive/upload', content=body, headers=headers)
assert response.status_code == HTTPStatus.OK
await await_switches()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.file_group.tag_names == []
@pytest.mark.asyncio
async def test_archive_upload_file_tracking(test_session, async_client, archive_directory, archive_factory,
await_switches, make_multipart_form, image_bytes_factory):