Creating Generate Screenshot button for Archives which are missing a screenshot.

This commit is contained in:
Roland 2025-10-19 11:40:02 -06:00 • committed by lrnselfreliance
parent 41e1ed6be6
commit 1ed0d63bac
11 changed files with 508 additions and 4 deletions

View file

@ -105,6 +105,19 @@ function handleEvents(events) {
eventToast('Archive Upload Failed!', message, 'error', 5000);
}
if (event === 'screenshot_generated') {
eventToast(
'Screenshot Generated',
message,
'success',
5000,
() => window.open(url, '_self'));
}
if (event === 'screenshot_generation_failed') {
eventToast('Screenshot Generation Failed!', message, 'error', 5000);
}
if (subject) {
newestEvents[subject] = dt;
}

View file

@ -750,6 +750,28 @@ export async function getArchive(archiveId) {
}
}
export async function generateArchiveScreenshot(archiveId) {
const response = await apiPost(`${ARCHIVES_API}/${archiveId}/generate_screenshot`);
if (response.ok) {
toast({
type: 'success',
title: 'Screenshot Generation Queued',
description: 'Screenshot generation has been queued. This may take a moment.',
time: 3000,
});
return true;
} else {
const message = await getErrorMessage(response, 'Failed to queue screenshot generation.');
toast({
type: 'error',
title: 'Screenshot Generation Error',
description: message,
time: 5000,
});
return false;
}
}
export async function postDownload(downloadData) {
if (!downloadData.downloader) {
toast({

View file

@ -35,7 +35,7 @@ import {
textEllipsis,
useTitle
} from "./Common";
import {deleteArchives, postDownload, tagFileGroup, untagFileGroup} from "../api";
import {deleteArchives, generateArchiveScreenshot, postDownload, tagFileGroup, untagFileGroup} from "../api";
import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
import {useArchive, useDomains, useSearchArchives, useSearchOrder} from "../hooks/customHooks";
@ -122,6 +122,14 @@ function ArchivePage() {
}
}
const localGenerateScreenshot = async () => {
const success = await generateArchiveScreenshot(data.id);
if (success) {
// Refresh the archive data after a short delay to show the new screenshot
setTimeout(() => fetchArchive(), 2000);
}
}
const updateButton = <APIButton
text='Update'
color='green'
@ -143,6 +151,17 @@ function ArchivePage() {
>
Delete
</APIButton>;
const generateScreenshotButton = !screenshotUrl ? <APIButton
text='Generate Screenshot'
color='yellow'
confirmContent='Generate a screenshot for this archive?'
confirmButton='Generate'
onClick={localGenerateScreenshot}
obeyWROLMode={true}
style={{marginTop: '0.5em'}}
>
Generate Screenshot
</APIButton> : null;
let historyList = <Loader active/>;
if (history && history.length === 0) {
@ -254,6 +273,7 @@ function ArchivePage() {
{readButton}
{updateButton}
{deleteButton}
{generateScreenshotButton}
</Segment>
<Segment>

View file

@ -10,6 +10,7 @@ import logging
import os.path
import pathlib
import subprocess
import sys
import tempfile
import traceback
from json import JSONDecodeError
@ -33,6 +34,12 @@ if not SINGLEFILE_PATH.is_file():
if not SINGLEFILE_PATH.is_file():
raise FileNotFoundError("Can't find single-file executable!")
BROWSER_EXEC = pathlib.Path('/usr/bin/google-chrome')
if not BROWSER_EXEC.is_file():
print('Unable to find browser!', file=sys.stderr)
sys.exit(1)
# Increase response timeout, archiving can take several minutes.
RESPONSE_TIMEOUT = 10 * 60
config = {
@ -117,7 +124,7 @@ async def extract_readability(path: str, url: str) -> dict:
async def take_screenshot(url: str) -> bytes:
cmd = '/usr/bin/google-chrome' \
cmd = f'{BROWSER_EXEC}' \
' --headless' \
' --disable-gpu' \
' --no-sandbox' \
@ -155,6 +162,61 @@ def prepare_bytes(b: bytes) -> str:
return b
@app.post('/screenshot')
async def post_screenshot(request: Request):
"""Generate a screenshot for the provided singlefile."""
url = request.json['url']
singlefile = request.json.get('singlefile')
try:
logger.info(f'Generating screenshot for {url}')
# Decode and decompress the singlefile
if singlefile:
singlefile = base64.b64decode(singlefile)
singlefile = gzip.decompress(singlefile)
if not singlefile:
raise ValueError(f'No singlefile provided for {url}')
# Write singlefile to temp file and screenshot it
# Use html suffix so chrome screenshot recognizes it as an HTML file
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(singlefile)
fh.flush()
screenshot = None
try:
# Screenshot the local singlefile
screenshot = await take_screenshot(f'file://{fh.name}')
except Exception as e:
logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e)
# Fall back to URL if local screenshot failed
if not screenshot:
logger.warning(f'Failed to screenshot local singlefile, attempting to screenshot URL: {url}')
try:
screenshot = await take_screenshot(url)
except Exception as e:
logger.error(f'Failed to take screenshot of {url}', exc_info=e)
if not screenshot:
raise ValueError(f'Failed to generate screenshot for {url}')
# Compress for smaller response
screenshot = prepare_bytes(screenshot)
ret = dict(
url=url,
screenshot=screenshot,
)
return response.json(ret)
except Exception as e:
logger.error(f'Failed to generate screenshot for {url}', exc_info=e)
error = str(traceback.format_exc())
return response.json({'error': f'Failed to generate screenshot for {url} traceback is below... \n\n {error}'})
@app.post('/json')
async def post_archive(request: Request):
url = request.json['url']

View file

@ -9,8 +9,10 @@ from sqlalchemy.orm import Session
from wrolpi.api_utils import json_response, api_app
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
from wrolpi.db import get_db_session
from wrolpi.errors import ValidationError
from wrolpi.events import Events
from wrolpi.files.lib import upsert_file
from wrolpi.schema import JSONErrorResponse
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
from . import lib, schema
@ -136,6 +138,49 @@ async def singlefile_upload_switch_handler(url=None):
singlefile_upload_switch_handler: ActivateSwitchMethod
@register_switch_handler('generate_screenshot_switch_handler')
async def generate_screenshot_switch_handler(archive_id=None):
"""Used by `post_generate_screenshot` to generate screenshots in the background"""
q: multiprocessing.Queue = api_app.shared_ctx.archive_screenshots
trace_enabled = logger.isEnabledFor(TRACE_LEVEL)
if trace_enabled:
logger.trace(f'generate_screenshot_switch_handler called for archive_id={archive_id}')
try:
archive_id = q.get_nowait()
except queue.Empty:
if trace_enabled:
logger.trace(f'generate_screenshot_switch_handler called on empty queue')
return
try:
q_size = q.qsize()
except NotImplementedError:
# qsize() is not implemented on macOS
q_size = '?'
logger.info(f'generate_screenshot_switch_handler queue size: {q_size}')
try:
await lib.generate_archive_screenshot(archive_id)
# Always send success event since exceptions are raised on failure
from modules.archive import Archive
archive = lib.get_archive(archive_id=archive_id)
location = archive.location
name = archive.file_group.title or archive.file_group.url
logger.info(f'Generated screenshot for Archive ({q_size}): {archive_id}')
Events.send_screenshot_generated(f'Generated screenshot for: {name}', url=location)
except Exception as e:
logger.error(f'generate_screenshot_switch_handler failed for Archive {archive_id}', exc_info=e)
Events.send_screenshot_generation_failed(f'Failed to generate screenshot: {e}')
raise
# Call this function again so any new screenshot requests can be processed.
generate_screenshot_switch_handler.activate_switch()
generate_screenshot_switch_handler: ActivateSwitchMethod
@archive_bp.post('/upload')
@openapi.definition(
summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.'
@ -153,3 +198,28 @@ async def post_upload_singlefile(request: Request):
singlefile_upload_switch_handler.activate_switch(context=dict(url=url))
# Return empty json response because SingleFile extension expects a JSON response.
return json_response(dict(), status=HTTPStatus.OK)
@archive_bp.post('/<archive_id:int>/generate_screenshot')
@openapi.description('Generate a screenshot for an Archive that does not have one')
@openapi.response(HTTPStatus.OK, description='Screenshot generation queued')
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
@wrol_mode_check
async def post_generate_screenshot(_: Request, archive_id: int):
"""Queue a screenshot generation request for an Archive."""
# Verify archive exists
try:
archive = lib.get_archive(archive_id=archive_id)
except Exception:
return json_response({'error': f'Archive {archive_id} not found'}, status=HTTPStatus.NOT_FOUND)
if not archive.singlefile_path:
return json_response({'error': 'Archive has no singlefile'}, status=HTTPStatus.BAD_REQUEST)
# Queue the screenshot generation request
logger.info(f'Queueing screenshot generation for Archive {archive_id}')
api_app.shared_ctx.archive_screenshots.put(archive_id)
generate_screenshot_switch_handler.activate_switch(context=dict(archive_id=archive_id))
return json_response({'message': 'Screenshot generation queued'}, status=HTTPStatus.OK)

View file

@ -156,6 +156,42 @@ async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Option
return singlefile, readability, screenshot
async def request_screenshot(url: str, singlefile_path: pathlib.Path) -> Optional[bytes]:
"""Send a request to the archive service to generate a screenshot from the singlefile."""
logger.info(f'Sending screenshot request to archive service: {url}')
# Read, compress, and encode the singlefile
singlefile_contents = singlefile_path.read_bytes()
singlefile_compressed = gzip.compress(singlefile_contents)
singlefile_b64 = base64.b64encode(singlefile_compressed).decode()
data = dict(url=url, singlefile=singlefile_b64)
try:
async with aiohttp_post(f'{ARCHIVE_SERVICE}/screenshot', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
status = response.status
contents = await response.json()
if contents and (error := contents.get('error')):
# Report the error from the archive service.
raise Exception(f'Received error from archive service: {error}')
# Compressed base64
screenshot = contents.get('screenshot')
if not screenshot:
logger.warning(f'Failed to get screenshot for {url=}')
return None
logger.debug(f'screenshot request status code {status}')
except Exception as e:
logger.error('Error when requesting screenshot', exc_info=e)
raise
# Decode and decompress.
screenshot = base64.b64decode(screenshot)
screenshot = gzip.decompress(screenshot)
return screenshot
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes) -> Archive:
"""
Convert results from ArchiveDownloader into real files. Create Archive record.
@ -625,3 +661,74 @@ async def singlefile_to_archive(singlefile: bytes) -> Archive:
logger.trace(f'singlefile_to_archive modeling: {url}')
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
return archive
async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
"""
Generate a screenshot for an existing Archive that doesn't have one.
If the Archive already has a screenshot, verify it exists and ensure it's tracked in the FileGroup.
Returns the path to the generated screenshot.
Raises:
ValueError: If Archive has no singlefile
RuntimeError: If screenshot generation fails
"""
from wrolpi.db import get_db_session
with get_db_session() as session:
archive = Archive.find_by_id(archive_id, session=session)
if not archive.singlefile_path:
raise ValueError(f'Cannot generate screenshot for Archive {archive_id}: no singlefile')
# Check if screenshot already exists
if archive.screenshot_path:
# Verify the screenshot file actually exists on disk
if archive.screenshot_path.is_file():
logger.info(f'Archive {archive_id} already has a screenshot, ensuring it is tracked')
# Ensure the screenshot is tracked in FileGroup.files and FileGroup.data
with get_db_session(commit=True) as tracking_session:
archive = Archive.find_by_id(archive_id, session=tracking_session)
file_group = archive.file_group
# append_files uses unique_by_predicate, so this is safe even if already tracked
file_group.append_files(archive.screenshot_path)
# Also update FileGroup.data (same pattern as set_screenshot)
data = dict(file_group.data) if file_group.data else {}
data['screenshot_path'] = str(archive.screenshot_path)
file_group.data = data
archive.validate()
tracking_session.flush()
return archive.screenshot_path
else:
logger.warning(f'Archive {archive_id} has screenshot_path but file does not exist, regenerating')
singlefile_path = archive.singlefile_path
url = archive.file_group.url
# Request screenshot from Archive docker service or generate locally
if DOCKERIZED:
logger.debug(f'Requesting screenshot from archive service for Archive {archive_id}')
screenshot_bytes = await request_screenshot(url, singlefile_path)
else:
logger.debug(f'Generating screenshot locally for Archive {archive_id}')
singlefile_contents = singlefile_path.read_bytes()
screenshot_bytes = html_screenshot(singlefile_contents)
if not screenshot_bytes:
raise RuntimeError(f'Failed to generate screenshot for Archive {archive_id}')
# Save screenshot next to singlefile with same naming pattern
screenshot_path = singlefile_path.with_suffix('.png')
screenshot_path.write_bytes(screenshot_bytes)
logger.info(f'Generated screenshot for Archive {archive_id}: {screenshot_path}')
# Update the Archive to include the new screenshot file
with get_db_session(commit=True) as session:
archive = Archive.find_by_id(archive_id, session=session)
archive.set_screenshot(screenshot_path)
session.flush()
return screenshot_path

View file

@ -187,7 +187,7 @@ class Archive(Base, ModelHelper):
return readability_json_file['path']
@property
def screenshot_file(self) -> Optional[dict]:
def screenshot_file(self) -> dict | None:
files = self.file_group.my_files('image/')
for file in files:
return file
@ -197,6 +197,35 @@ class Archive(Base, ModelHelper):
if screenshot_file := self.screenshot_file:
return screenshot_file['path']
def set_screenshot(self, screenshot_path: pathlib.Path) -> None:
"""Set the screenshot for this Archive.
Args:
screenshot_path: Path to the screenshot file
Raises:
ValueError: If Archive already has a screenshot or path doesn't exist
"""
# Check if screenshot already exists
if self.screenshot_path:
raise ValueError(f'Archive {self.id} already has a screenshot at {self.screenshot_path}')
# Validate path exists
if not screenshot_path.is_file():
raise ValueError(f'Screenshot path does not exist: {screenshot_path}')
# Add to FileGroup.files
self.file_group.append_files(screenshot_path)
# Update FileGroup.data for quick lookup (similar to ebook cover_path pattern)
# Create new dict and reassign (like append_files does) to ensure SQLAlchemy detects the change
data = dict(self.file_group.data) if self.file_group.data else {}
data['screenshot_path'] = str(screenshot_path)
self.file_group.data = data
# Validate the archive
self.validate()
@property
def info_json_file(self) -> Optional[dict]:
files = self.file_group.my_files()

View file

@ -1,5 +1,6 @@
import json
from http import HTTPStatus
from unittest.mock import patch
import pytest
@ -364,3 +365,97 @@ async def test_archive_upload_file_tracking(test_session, async_client, archive_
# Assert FileGroup has correct number of files
assert len(archive.file_group.files) == file_count_before_image + 1, \
f'FileGroup should have {file_count_before_image + 1} files after adding image'
@pytest.mark.asyncio
async def test_archive_generate_screenshot(test_session, async_client, archive_factory, await_switches,
image_bytes_factory, wrol_mode_fixture):
"""Test generating a screenshot for an Archive that doesn't have one."""
# Mock html_screenshot to avoid Selenium dependency
mock_screenshot_bytes = image_bytes_factory()
with patch('modules.archive.lib.html_screenshot', return_value=mock_screenshot_bytes):
# Test success case: Archive without screenshot
archive = archive_factory('example.com', 'https://example.com/test', 'Test Archive', screenshot=False)
test_session.commit()
# Verify archive has no screenshot
assert archive.screenshot_path is None, 'Archive should not have a screenshot yet'
assert archive.singlefile_path is not None, 'Archive should have a singlefile'
# Request screenshot generation
request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot')
assert response.status_code == HTTPStatus.OK
assert response.json['message'] == 'Screenshot generation queued'
# Wait for background processing
await await_switches()
# Verify screenshot was generated
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.screenshot_path is not None, 'Screenshot should have been generated'
assert archive.screenshot_path.is_file(), 'Screenshot file should exist'
assert archive.screenshot_file is not None, 'Screenshot file should be tracked'
# Test error case: Archive not found
request, response = await async_client.post('/api/archive/99999/generate_screenshot')
assert response.status_code == HTTPStatus.NOT_FOUND
assert 'not found' in response.json['error'].lower()
# Test success case: Archive already has screenshot - should return OK and ensure tracking
original_screenshot_path = archive.screenshot_path
assert original_screenshot_path is not None, 'Archive should have a screenshot'
assert original_screenshot_path.is_file(), 'Screenshot file should exist'
# Simulate the bug: archive has screenshot file but data['screenshot_path'] is not set
# (This could happen if the archive was created before set_screenshot was implemented)
if archive.file_group.data and 'screenshot_path' in archive.file_group.data:
data = dict(archive.file_group.data)
del data['screenshot_path']
archive.file_group.data = data
test_session.commit()
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Verify data was cleared but file still exists
assert 'screenshot_path' not in (archive.file_group.data or {}), 'screenshot_path should not be in data yet'
assert archive.screenshot_path is not None, 'But screenshot file should still exist'
request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot')
assert response.status_code == HTTPStatus.OK
assert response.json['message'] == 'Screenshot generation queued'
# Wait for background processing
await await_switches()
# Verify screenshot is still there and properly tracked
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
assert archive.screenshot_path == original_screenshot_path, 'Screenshot path should be unchanged'
assert archive.screenshot_path.is_file(), 'Screenshot file should still exist'
assert archive.screenshot_file is not None, 'Screenshot file should still be tracked'
# IMPORTANT: Also verify FileGroup.data was updated (this catches the bug where only files was updated)
assert 'screenshot_path' in archive.file_group.data, \
'screenshot_path should be in FileGroup.data after ensuring tracking'
assert str(archive.file_group.data['screenshot_path']) == str(original_screenshot_path), \
'screenshot_path in data should match the file path'
# Test error case: Archive has no singlefile
archive_no_singlefile = archive_factory('example.com', 'https://example.com/no-singlefile', screenshot=False)
test_session.commit()
# Delete the singlefile to simulate missing file
if archive_no_singlefile.singlefile_path and archive_no_singlefile.singlefile_path.is_file():
archive_no_singlefile.singlefile_path.unlink()
request, response = await async_client.post(f'/api/archive/{archive_no_singlefile.id}/generate_screenshot')
assert response.status_code == HTTPStatus.BAD_REQUEST
assert 'no singlefile' in response.json['error'].lower()
# Test WROL mode case
await wrol_mode_fixture(True)
archive_wrol = archive_factory('example.com', 'https://example.com/wrol-test', screenshot=False)
test_session.commit()
request, response = await async_client.post(f'/api/archive/{archive_wrol.id}/generate_screenshot')
assert response.status_code == HTTPStatus.FORBIDDEN
await wrol_mode_fixture(False)

View file

@ -1,8 +1,9 @@
from datetime import datetime
import pathlib
import pytest
from modules.archive import Domain
from modules.archive import Domain, Archive
from wrolpi.common import get_wrolpi_config
@ -25,3 +26,73 @@ async def test_archive_download_destination(async_client, test_session, test_dir
# More download date is supported.
wrolpi_config.archive_destination = 'archive/%(domain)s/%(year)s/%(month)s/%(day)s'
assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2')
@pytest.mark.asyncio
async def test_set_screenshot(test_session, archive_factory, image_bytes_factory):
"""Test Archive.set_screenshot() method with various scenarios."""
# ========== Test 1: Successfully set screenshot on archive without one ==========
archive = archive_factory('example.com', 'https://example.com/test', 'Test Archive', screenshot=False)
test_session.commit()
# Verify no screenshot initially
assert archive.screenshot_path is None
# Create a screenshot file
screenshot_path = archive.singlefile_path.parent / 'test_screenshot.png'
screenshot_path.write_bytes(image_bytes_factory())
# Set the screenshot
archive.set_screenshot(screenshot_path)
# Verify data was set before commit
assert archive.file_group.data['screenshot_path'] == str(screenshot_path), \
f"Data not set correctly before commit: {archive.file_group.data.get('screenshot_path')}"
test_session.commit()
# Refresh the archive to get latest data from DB
test_session.expire_all()
archive = test_session.query(Archive).filter_by(id=archive.id).one()
# Verify screenshot was set
assert archive.screenshot_path == screenshot_path
assert archive.screenshot_file is not None
# Verify FileGroup.data was persisted to DB (FancyJSON converts strings back to Path objects)
assert 'screenshot_path' in archive.file_group.data
assert str(archive.file_group.data['screenshot_path']) == str(screenshot_path)
# Verify file is tracked in FileGroup.files
screenshot_files = archive.file_group.my_files('image/')
assert len(screenshot_files) > 0
assert screenshot_files[0]['path'] == screenshot_path
# ========== Test 2: Error when trying to set screenshot on archive that already has one ==========
original_screenshot = archive.screenshot_path
assert original_screenshot is not None
# Try to set another screenshot
new_screenshot_path = archive.singlefile_path.parent / 'another_screenshot.png'
new_screenshot_path.write_bytes(b'fake image data')
# Should raise ValueError
with pytest.raises(ValueError, match='already has a screenshot'):
archive.set_screenshot(new_screenshot_path)
# Verify original screenshot is unchanged
assert archive.screenshot_path == original_screenshot
# ========== Test 3: Error when trying to set non-existent screenshot file ==========
archive_no_screenshot = archive_factory('example.com', 'https://example.com/test2', 'Test Archive 2', screenshot=False)
test_session.commit()
# Try to set a screenshot that doesn't exist
nonexistent_path = pathlib.Path('/tmp/does_not_exist.png')
# Should raise ValueError
with pytest.raises(ValueError, match='does not exist'):
archive_no_screenshot.set_screenshot(nonexistent_path)
# Verify no screenshot was set
assert archive_no_screenshot.screenshot_path is None

View file

@ -56,6 +56,7 @@ def attach_shared_contexts(app: Sanic):
app.shared_ctx.switches_lock = multiprocessing.Lock()
app.shared_ctx.switches_changed = multiprocessing.Event()
app.shared_ctx.archive_singlefiles = multiprocessing.Queue()
app.shared_ctx.archive_screenshots = multiprocessing.Queue()
# Warnings
app.shared_ctx.warn_once = manager.dict()
@ -118,6 +119,12 @@ def reset_shared_contexts(app: Sanic):
app.shared_ctx.archive_singlefiles.get_nowait()
except queue.Empty:
break
while True:
# Clear out any pending screenshot generation switches.
try:
app.shared_ctx.archive_screenshots.get_nowait()
except queue.Empty:
break
# Events.
app.shared_ctx.single_tasks_started.clear()

View file

@ -100,6 +100,14 @@ class Events:
def send_upload_archive_failed(cls, message: str = None):
send_event('upload_archive_failed', message, subject='upload')
@classmethod
def send_screenshot_generated(cls, message: str = None, url: str = None):
send_event('screenshot_generated', message, subject='screenshot', url=url)
@classmethod
def send_screenshot_generation_failed(cls, message: str = None):
send_event('screenshot_generation_failed', message, subject='screenshot')
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
log = f'{event=}'