mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
Creating Generate Screenshot button for Archives which are missing a screenshot.
This commit is contained in:
parent
41e1ed6be6
commit
1ed0d63bac
11 changed files with 508 additions and 4 deletions
|
|
@ -105,6 +105,19 @@ function handleEvents(events) {
|
|||
eventToast('Archive Upload Failed!', message, 'error', 5000);
|
||||
}
|
||||
|
||||
if (event === 'screenshot_generated') {
|
||||
eventToast(
|
||||
'Screenshot Generated',
|
||||
message,
|
||||
'success',
|
||||
5000,
|
||||
() => window.open(url, '_self'));
|
||||
}
|
||||
|
||||
if (event === 'screenshot_generation_failed') {
|
||||
eventToast('Screenshot Generation Failed!', message, 'error', 5000);
|
||||
}
|
||||
|
||||
if (subject) {
|
||||
newestEvents[subject] = dt;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -750,6 +750,28 @@ export async function getArchive(archiveId) {
|
|||
}
|
||||
}
|
||||
|
||||
export async function generateArchiveScreenshot(archiveId) {
|
||||
const response = await apiPost(`${ARCHIVES_API}/${archiveId}/generate_screenshot`);
|
||||
if (response.ok) {
|
||||
toast({
|
||||
type: 'success',
|
||||
title: 'Screenshot Generation Queued',
|
||||
description: 'Screenshot generation has been queued. This may take a moment.',
|
||||
time: 3000,
|
||||
});
|
||||
return true;
|
||||
} else {
|
||||
const message = await getErrorMessage(response, 'Failed to queue screenshot generation.');
|
||||
toast({
|
||||
type: 'error',
|
||||
title: 'Screenshot Generation Error',
|
||||
description: message,
|
||||
time: 5000,
|
||||
});
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export async function postDownload(downloadData) {
|
||||
if (!downloadData.downloader) {
|
||||
toast({
|
||||
|
|
|
|||
|
|
@ -35,7 +35,7 @@ import {
|
|||
textEllipsis,
|
||||
useTitle
|
||||
} from "./Common";
|
||||
import {deleteArchives, postDownload, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {deleteArchives, generateArchiveScreenshot, postDownload, tagFileGroup, untagFileGroup} from "../api";
|
||||
import {Link, Route, Routes, useNavigate, useParams} from "react-router-dom";
|
||||
import Message from "semantic-ui-react/dist/commonjs/collections/Message";
|
||||
import {useArchive, useDomains, useSearchArchives, useSearchOrder} from "../hooks/customHooks";
|
||||
|
|
@ -122,6 +122,14 @@ function ArchivePage() {
|
|||
}
|
||||
}
|
||||
|
||||
const localGenerateScreenshot = async () => {
|
||||
const success = await generateArchiveScreenshot(data.id);
|
||||
if (success) {
|
||||
// Refresh the archive data after a short delay to show the new screenshot
|
||||
setTimeout(() => fetchArchive(), 2000);
|
||||
}
|
||||
}
|
||||
|
||||
const updateButton = <APIButton
|
||||
text='Update'
|
||||
color='green'
|
||||
|
|
@ -143,6 +151,17 @@ function ArchivePage() {
|
|||
>
|
||||
Delete
|
||||
</APIButton>;
|
||||
const generateScreenshotButton = !screenshotUrl ? <APIButton
|
||||
text='Generate Screenshot'
|
||||
color='yellow'
|
||||
confirmContent='Generate a screenshot for this archive?'
|
||||
confirmButton='Generate'
|
||||
onClick={localGenerateScreenshot}
|
||||
obeyWROLMode={true}
|
||||
style={{marginTop: '0.5em'}}
|
||||
>
|
||||
Generate Screenshot
|
||||
</APIButton> : null;
|
||||
|
||||
let historyList = <Loader active/>;
|
||||
if (history && history.length === 0) {
|
||||
|
|
@ -254,6 +273,7 @@ function ArchivePage() {
|
|||
{readButton}
|
||||
{updateButton}
|
||||
{deleteButton}
|
||||
{generateScreenshotButton}
|
||||
</Segment>
|
||||
|
||||
<Segment>
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import logging
|
|||
import os.path
|
||||
import pathlib
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
from json import JSONDecodeError
|
||||
|
|
@ -33,6 +34,12 @@ if not SINGLEFILE_PATH.is_file():
|
|||
if not SINGLEFILE_PATH.is_file():
|
||||
raise FileNotFoundError("Can't find single-file executable!")
|
||||
|
||||
BROWSER_EXEC = pathlib.Path('/usr/bin/google-chrome')
|
||||
if not BROWSER_EXEC.is_file():
|
||||
print('Unable to find browser!', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
# Increase response timeout, archiving can take several minutes.
|
||||
RESPONSE_TIMEOUT = 10 * 60
|
||||
config = {
|
||||
|
|
@ -117,7 +124,7 @@ async def extract_readability(path: str, url: str) -> dict:
|
|||
|
||||
|
||||
async def take_screenshot(url: str) -> bytes:
|
||||
cmd = '/usr/bin/google-chrome' \
|
||||
cmd = f'{BROWSER_EXEC}' \
|
||||
' --headless' \
|
||||
' --disable-gpu' \
|
||||
' --no-sandbox' \
|
||||
|
|
@ -155,6 +162,61 @@ def prepare_bytes(b: bytes) -> str:
|
|||
return b
|
||||
|
||||
|
||||
@app.post('/screenshot')
|
||||
async def post_screenshot(request: Request):
|
||||
"""Generate a screenshot for the provided singlefile."""
|
||||
url = request.json['url']
|
||||
singlefile = request.json.get('singlefile')
|
||||
|
||||
try:
|
||||
logger.info(f'Generating screenshot for {url}')
|
||||
|
||||
# Decode and decompress the singlefile
|
||||
if singlefile:
|
||||
singlefile = base64.b64decode(singlefile)
|
||||
singlefile = gzip.decompress(singlefile)
|
||||
|
||||
if not singlefile:
|
||||
raise ValueError(f'No singlefile provided for {url}')
|
||||
|
||||
# Write singlefile to temp file and screenshot it
|
||||
# Use html suffix so chrome screenshot recognizes it as an HTML file
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
|
||||
fh.write(singlefile)
|
||||
fh.flush()
|
||||
|
||||
screenshot = None
|
||||
try:
|
||||
# Screenshot the local singlefile
|
||||
screenshot = await take_screenshot(f'file://{fh.name}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e)
|
||||
|
||||
# Fall back to URL if local screenshot failed
|
||||
if not screenshot:
|
||||
logger.warning(f'Failed to screenshot local singlefile, attempting to screenshot URL: {url}')
|
||||
try:
|
||||
screenshot = await take_screenshot(url)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to take screenshot of {url}', exc_info=e)
|
||||
|
||||
if not screenshot:
|
||||
raise ValueError(f'Failed to generate screenshot for {url}')
|
||||
|
||||
# Compress for smaller response
|
||||
screenshot = prepare_bytes(screenshot)
|
||||
|
||||
ret = dict(
|
||||
url=url,
|
||||
screenshot=screenshot,
|
||||
)
|
||||
return response.json(ret)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to generate screenshot for {url}', exc_info=e)
|
||||
error = str(traceback.format_exc())
|
||||
return response.json({'error': f'Failed to generate screenshot for {url} traceback is below... \n\n {error}'})
|
||||
|
||||
|
||||
@app.post('/json')
|
||||
async def post_archive(request: Request):
|
||||
url = request.json['url']
|
||||
|
|
|
|||
|
|
@ -9,8 +9,10 @@ from sqlalchemy.orm import Session
|
|||
|
||||
from wrolpi.api_utils import json_response, api_app
|
||||
from wrolpi.common import logger, wrol_mode_check, api_param_limiter, TRACE_LEVEL
|
||||
from wrolpi.db import get_db_session
|
||||
from wrolpi.errors import ValidationError
|
||||
from wrolpi.events import Events
|
||||
from wrolpi.files.lib import upsert_file
|
||||
from wrolpi.schema import JSONErrorResponse
|
||||
from wrolpi.switches import register_switch_handler, ActivateSwitchMethod
|
||||
from . import lib, schema
|
||||
|
|
@ -136,6 +138,49 @@ async def singlefile_upload_switch_handler(url=None):
|
|||
singlefile_upload_switch_handler: ActivateSwitchMethod
|
||||
|
||||
|
||||
@register_switch_handler('generate_screenshot_switch_handler')
|
||||
async def generate_screenshot_switch_handler(archive_id=None):
|
||||
"""Used by `post_generate_screenshot` to generate screenshots in the background"""
|
||||
q: multiprocessing.Queue = api_app.shared_ctx.archive_screenshots
|
||||
|
||||
trace_enabled = logger.isEnabledFor(TRACE_LEVEL)
|
||||
if trace_enabled:
|
||||
logger.trace(f'generate_screenshot_switch_handler called for archive_id={archive_id}')
|
||||
try:
|
||||
archive_id = q.get_nowait()
|
||||
except queue.Empty:
|
||||
if trace_enabled:
|
||||
logger.trace(f'generate_screenshot_switch_handler called on empty queue')
|
||||
return
|
||||
|
||||
try:
|
||||
q_size = q.qsize()
|
||||
except NotImplementedError:
|
||||
# qsize() is not implemented on macOS
|
||||
q_size = '?'
|
||||
logger.info(f'generate_screenshot_switch_handler queue size: {q_size}')
|
||||
|
||||
try:
|
||||
await lib.generate_archive_screenshot(archive_id)
|
||||
# Always send success event since exceptions are raised on failure
|
||||
from modules.archive import Archive
|
||||
archive = lib.get_archive(archive_id=archive_id)
|
||||
location = archive.location
|
||||
name = archive.file_group.title or archive.file_group.url
|
||||
logger.info(f'Generated screenshot for Archive ({q_size}): {archive_id}')
|
||||
Events.send_screenshot_generated(f'Generated screenshot for: {name}', url=location)
|
||||
except Exception as e:
|
||||
logger.error(f'generate_screenshot_switch_handler failed for Archive {archive_id}', exc_info=e)
|
||||
Events.send_screenshot_generation_failed(f'Failed to generate screenshot: {e}')
|
||||
raise
|
||||
|
||||
# Call this function again so any new screenshot requests can be processed.
|
||||
generate_screenshot_switch_handler.activate_switch()
|
||||
|
||||
|
||||
generate_screenshot_switch_handler: ActivateSwitchMethod
|
||||
|
||||
|
||||
@archive_bp.post('/upload')
|
||||
@openapi.definition(
|
||||
summary='Upload SingleFile from SingleFile browser extension and convert it to an Archive.'
|
||||
|
|
@ -153,3 +198,28 @@ async def post_upload_singlefile(request: Request):
|
|||
singlefile_upload_switch_handler.activate_switch(context=dict(url=url))
|
||||
# Return empty json response because SingleFile extension expects a JSON response.
|
||||
return json_response(dict(), status=HTTPStatus.OK)
|
||||
|
||||
|
||||
@archive_bp.post('/<archive_id:int>/generate_screenshot')
|
||||
@openapi.description('Generate a screenshot for an Archive that does not have one')
|
||||
@openapi.response(HTTPStatus.OK, description='Screenshot generation queued')
|
||||
@openapi.response(HTTPStatus.NOT_FOUND, JSONErrorResponse)
|
||||
@openapi.response(HTTPStatus.BAD_REQUEST, JSONErrorResponse)
|
||||
@wrol_mode_check
|
||||
async def post_generate_screenshot(_: Request, archive_id: int):
|
||||
"""Queue a screenshot generation request for an Archive."""
|
||||
# Verify archive exists
|
||||
try:
|
||||
archive = lib.get_archive(archive_id=archive_id)
|
||||
except Exception:
|
||||
return json_response({'error': f'Archive {archive_id} not found'}, status=HTTPStatus.NOT_FOUND)
|
||||
|
||||
if not archive.singlefile_path:
|
||||
return json_response({'error': 'Archive has no singlefile'}, status=HTTPStatus.BAD_REQUEST)
|
||||
|
||||
# Queue the screenshot generation request
|
||||
logger.info(f'Queueing screenshot generation for Archive {archive_id}')
|
||||
api_app.shared_ctx.archive_screenshots.put(archive_id)
|
||||
generate_screenshot_switch_handler.activate_switch(context=dict(archive_id=archive_id))
|
||||
|
||||
return json_response({'message': 'Screenshot generation queued'}, status=HTTPStatus.OK)
|
||||
|
|
|
|||
|
|
@ -156,6 +156,42 @@ async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Option
|
|||
return singlefile, readability, screenshot
|
||||
|
||||
|
||||
async def request_screenshot(url: str, singlefile_path: pathlib.Path) -> Optional[bytes]:
|
||||
"""Send a request to the archive service to generate a screenshot from the singlefile."""
|
||||
logger.info(f'Sending screenshot request to archive service: {url}')
|
||||
|
||||
# Read, compress, and encode the singlefile
|
||||
singlefile_contents = singlefile_path.read_bytes()
|
||||
singlefile_compressed = gzip.compress(singlefile_contents)
|
||||
singlefile_b64 = base64.b64encode(singlefile_compressed).decode()
|
||||
|
||||
data = dict(url=url, singlefile=singlefile_b64)
|
||||
try:
|
||||
async with aiohttp_post(f'{ARCHIVE_SERVICE}/screenshot', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
|
||||
status = response.status
|
||||
contents = await response.json()
|
||||
if contents and (error := contents.get('error')):
|
||||
# Report the error from the archive service.
|
||||
raise Exception(f'Received error from archive service: {error}')
|
||||
|
||||
# Compressed base64
|
||||
screenshot = contents.get('screenshot')
|
||||
if not screenshot:
|
||||
logger.warning(f'Failed to get screenshot for {url=}')
|
||||
return None
|
||||
|
||||
logger.debug(f'screenshot request status code {status}')
|
||||
except Exception as e:
|
||||
logger.error('Error when requesting screenshot', exc_info=e)
|
||||
raise
|
||||
|
||||
# Decode and decompress.
|
||||
screenshot = base64.b64decode(screenshot)
|
||||
screenshot = gzip.decompress(screenshot)
|
||||
|
||||
return screenshot
|
||||
|
||||
|
||||
async def model_archive_result(url: str, singlefile: str, readability: dict, screenshot: bytes) -> Archive:
|
||||
"""
|
||||
Convert results from ArchiveDownloader into real files. Create Archive record.
|
||||
|
|
@ -625,3 +661,74 @@ async def singlefile_to_archive(singlefile: bytes) -> Archive:
|
|||
logger.trace(f'singlefile_to_archive modeling: {url}')
|
||||
archive: Archive = await model_archive_result(url, singlefile, readability, screenshot)
|
||||
return archive
|
||||
|
||||
|
||||
async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
|
||||
"""
|
||||
Generate a screenshot for an existing Archive that doesn't have one.
|
||||
If the Archive already has a screenshot, verify it exists and ensure it's tracked in the FileGroup.
|
||||
|
||||
Returns the path to the generated screenshot.
|
||||
|
||||
Raises:
|
||||
ValueError: If Archive has no singlefile
|
||||
RuntimeError: If screenshot generation fails
|
||||
"""
|
||||
from wrolpi.db import get_db_session
|
||||
|
||||
with get_db_session() as session:
|
||||
archive = Archive.find_by_id(archive_id, session=session)
|
||||
|
||||
if not archive.singlefile_path:
|
||||
raise ValueError(f'Cannot generate screenshot for Archive {archive_id}: no singlefile')
|
||||
|
||||
# Check if screenshot already exists
|
||||
if archive.screenshot_path:
|
||||
# Verify the screenshot file actually exists on disk
|
||||
if archive.screenshot_path.is_file():
|
||||
logger.info(f'Archive {archive_id} already has a screenshot, ensuring it is tracked')
|
||||
# Ensure the screenshot is tracked in FileGroup.files and FileGroup.data
|
||||
with get_db_session(commit=True) as tracking_session:
|
||||
archive = Archive.find_by_id(archive_id, session=tracking_session)
|
||||
file_group = archive.file_group
|
||||
# append_files uses unique_by_predicate, so this is safe even if already tracked
|
||||
file_group.append_files(archive.screenshot_path)
|
||||
|
||||
# Also update FileGroup.data (same pattern as set_screenshot)
|
||||
data = dict(file_group.data) if file_group.data else {}
|
||||
data['screenshot_path'] = str(archive.screenshot_path)
|
||||
file_group.data = data
|
||||
|
||||
archive.validate()
|
||||
tracking_session.flush()
|
||||
return archive.screenshot_path
|
||||
else:
|
||||
logger.warning(f'Archive {archive_id} has screenshot_path but file does not exist, regenerating')
|
||||
|
||||
singlefile_path = archive.singlefile_path
|
||||
url = archive.file_group.url
|
||||
|
||||
# Request screenshot from Archive docker service or generate locally
|
||||
if DOCKERIZED:
|
||||
logger.debug(f'Requesting screenshot from archive service for Archive {archive_id}')
|
||||
screenshot_bytes = await request_screenshot(url, singlefile_path)
|
||||
else:
|
||||
logger.debug(f'Generating screenshot locally for Archive {archive_id}')
|
||||
singlefile_contents = singlefile_path.read_bytes()
|
||||
screenshot_bytes = html_screenshot(singlefile_contents)
|
||||
|
||||
if not screenshot_bytes:
|
||||
raise RuntimeError(f'Failed to generate screenshot for Archive {archive_id}')
|
||||
|
||||
# Save screenshot next to singlefile with same naming pattern
|
||||
screenshot_path = singlefile_path.with_suffix('.png')
|
||||
screenshot_path.write_bytes(screenshot_bytes)
|
||||
logger.info(f'Generated screenshot for Archive {archive_id}: {screenshot_path}')
|
||||
|
||||
# Update the Archive to include the new screenshot file
|
||||
with get_db_session(commit=True) as session:
|
||||
archive = Archive.find_by_id(archive_id, session=session)
|
||||
archive.set_screenshot(screenshot_path)
|
||||
session.flush()
|
||||
|
||||
return screenshot_path
|
||||
|
|
|
|||
|
|
@ -187,7 +187,7 @@ class Archive(Base, ModelHelper):
|
|||
return readability_json_file['path']
|
||||
|
||||
@property
|
||||
def screenshot_file(self) -> Optional[dict]:
|
||||
def screenshot_file(self) -> dict | None:
|
||||
files = self.file_group.my_files('image/')
|
||||
for file in files:
|
||||
return file
|
||||
|
|
@ -197,6 +197,35 @@ class Archive(Base, ModelHelper):
|
|||
if screenshot_file := self.screenshot_file:
|
||||
return screenshot_file['path']
|
||||
|
||||
def set_screenshot(self, screenshot_path: pathlib.Path) -> None:
|
||||
"""Set the screenshot for this Archive.
|
||||
|
||||
Args:
|
||||
screenshot_path: Path to the screenshot file
|
||||
|
||||
Raises:
|
||||
ValueError: If Archive already has a screenshot or path doesn't exist
|
||||
"""
|
||||
# Check if screenshot already exists
|
||||
if self.screenshot_path:
|
||||
raise ValueError(f'Archive {self.id} already has a screenshot at {self.screenshot_path}')
|
||||
|
||||
# Validate path exists
|
||||
if not screenshot_path.is_file():
|
||||
raise ValueError(f'Screenshot path does not exist: {screenshot_path}')
|
||||
|
||||
# Add to FileGroup.files
|
||||
self.file_group.append_files(screenshot_path)
|
||||
|
||||
# Update FileGroup.data for quick lookup (similar to ebook cover_path pattern)
|
||||
# Create new dict and reassign (like append_files does) to ensure SQLAlchemy detects the change
|
||||
data = dict(self.file_group.data) if self.file_group.data else {}
|
||||
data['screenshot_path'] = str(screenshot_path)
|
||||
self.file_group.data = data
|
||||
|
||||
# Validate the archive
|
||||
self.validate()
|
||||
|
||||
@property
|
||||
def info_json_file(self) -> Optional[dict]:
|
||||
files = self.file_group.my_files()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,6 @@
|
|||
import json
|
||||
from http import HTTPStatus
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
|
|
@ -364,3 +365,97 @@ async def test_archive_upload_file_tracking(test_session, async_client, archive_
|
|||
# Assert FileGroup has correct number of files
|
||||
assert len(archive.file_group.files) == file_count_before_image + 1, \
|
||||
f'FileGroup should have {file_count_before_image + 1} files after adding image'
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_archive_generate_screenshot(test_session, async_client, archive_factory, await_switches,
|
||||
image_bytes_factory, wrol_mode_fixture):
|
||||
"""Test generating a screenshot for an Archive that doesn't have one."""
|
||||
# Mock html_screenshot to avoid Selenium dependency
|
||||
mock_screenshot_bytes = image_bytes_factory()
|
||||
|
||||
with patch('modules.archive.lib.html_screenshot', return_value=mock_screenshot_bytes):
|
||||
# Test success case: Archive without screenshot
|
||||
archive = archive_factory('example.com', 'https://example.com/test', 'Test Archive', screenshot=False)
|
||||
test_session.commit()
|
||||
|
||||
# Verify archive has no screenshot
|
||||
assert archive.screenshot_path is None, 'Archive should not have a screenshot yet'
|
||||
assert archive.singlefile_path is not None, 'Archive should have a singlefile'
|
||||
|
||||
# Request screenshot generation
|
||||
request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['message'] == 'Screenshot generation queued'
|
||||
|
||||
# Wait for background processing
|
||||
await await_switches()
|
||||
|
||||
# Verify screenshot was generated
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
assert archive.screenshot_path is not None, 'Screenshot should have been generated'
|
||||
assert archive.screenshot_path.is_file(), 'Screenshot file should exist'
|
||||
assert archive.screenshot_file is not None, 'Screenshot file should be tracked'
|
||||
|
||||
# Test error case: Archive not found
|
||||
request, response = await async_client.post('/api/archive/99999/generate_screenshot')
|
||||
assert response.status_code == HTTPStatus.NOT_FOUND
|
||||
assert 'not found' in response.json['error'].lower()
|
||||
|
||||
# Test success case: Archive already has screenshot - should return OK and ensure tracking
|
||||
original_screenshot_path = archive.screenshot_path
|
||||
assert original_screenshot_path is not None, 'Archive should have a screenshot'
|
||||
assert original_screenshot_path.is_file(), 'Screenshot file should exist'
|
||||
|
||||
# Simulate the bug: archive has screenshot file but data['screenshot_path'] is not set
|
||||
# (This could happen if the archive was created before set_screenshot was implemented)
|
||||
if archive.file_group.data and 'screenshot_path' in archive.file_group.data:
|
||||
data = dict(archive.file_group.data)
|
||||
del data['screenshot_path']
|
||||
archive.file_group.data = data
|
||||
test_session.commit()
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
# Verify data was cleared but file still exists
|
||||
assert 'screenshot_path' not in (archive.file_group.data or {}), 'screenshot_path should not be in data yet'
|
||||
assert archive.screenshot_path is not None, 'But screenshot file should still exist'
|
||||
|
||||
request, response = await async_client.post(f'/api/archive/{archive.id}/generate_screenshot')
|
||||
assert response.status_code == HTTPStatus.OK
|
||||
assert response.json['message'] == 'Screenshot generation queued'
|
||||
|
||||
# Wait for background processing
|
||||
await await_switches()
|
||||
|
||||
# Verify screenshot is still there and properly tracked
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
assert archive.screenshot_path == original_screenshot_path, 'Screenshot path should be unchanged'
|
||||
assert archive.screenshot_path.is_file(), 'Screenshot file should still exist'
|
||||
assert archive.screenshot_file is not None, 'Screenshot file should still be tracked'
|
||||
# IMPORTANT: Also verify FileGroup.data was updated (this catches the bug where only files was updated)
|
||||
assert 'screenshot_path' in archive.file_group.data, \
|
||||
'screenshot_path should be in FileGroup.data after ensuring tracking'
|
||||
assert str(archive.file_group.data['screenshot_path']) == str(original_screenshot_path), \
|
||||
'screenshot_path in data should match the file path'
|
||||
|
||||
# Test error case: Archive has no singlefile
|
||||
archive_no_singlefile = archive_factory('example.com', 'https://example.com/no-singlefile', screenshot=False)
|
||||
test_session.commit()
|
||||
# Delete the singlefile to simulate missing file
|
||||
if archive_no_singlefile.singlefile_path and archive_no_singlefile.singlefile_path.is_file():
|
||||
archive_no_singlefile.singlefile_path.unlink()
|
||||
|
||||
request, response = await async_client.post(f'/api/archive/{archive_no_singlefile.id}/generate_screenshot')
|
||||
assert response.status_code == HTTPStatus.BAD_REQUEST
|
||||
assert 'no singlefile' in response.json['error'].lower()
|
||||
|
||||
# Test WROL mode case
|
||||
await wrol_mode_fixture(True)
|
||||
archive_wrol = archive_factory('example.com', 'https://example.com/wrol-test', screenshot=False)
|
||||
test_session.commit()
|
||||
|
||||
request, response = await async_client.post(f'/api/archive/{archive_wrol.id}/generate_screenshot')
|
||||
assert response.status_code == HTTPStatus.FORBIDDEN
|
||||
await wrol_mode_fixture(False)
|
||||
|
|
|
|||
|
|
@ -1,8 +1,9 @@
|
|||
from datetime import datetime
|
||||
import pathlib
|
||||
|
||||
import pytest
|
||||
|
||||
from modules.archive import Domain
|
||||
from modules.archive import Domain, Archive
|
||||
from wrolpi.common import get_wrolpi_config
|
||||
|
||||
|
||||
|
|
@ -25,3 +26,73 @@ async def test_archive_download_destination(async_client, test_session, test_dir
|
|||
# More download date is supported.
|
||||
wrolpi_config.archive_destination = 'archive/%(domain)s/%(year)s/%(month)s/%(day)s'
|
||||
assert str(domain.download_directory) == str(test_directory / 'archive/wrolpi.org/2000/1/2')
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_set_screenshot(test_session, archive_factory, image_bytes_factory):
|
||||
"""Test Archive.set_screenshot() method with various scenarios."""
|
||||
|
||||
# ========== Test 1: Successfully set screenshot on archive without one ==========
|
||||
archive = archive_factory('example.com', 'https://example.com/test', 'Test Archive', screenshot=False)
|
||||
test_session.commit()
|
||||
|
||||
# Verify no screenshot initially
|
||||
assert archive.screenshot_path is None
|
||||
|
||||
# Create a screenshot file
|
||||
screenshot_path = archive.singlefile_path.parent / 'test_screenshot.png'
|
||||
screenshot_path.write_bytes(image_bytes_factory())
|
||||
|
||||
# Set the screenshot
|
||||
archive.set_screenshot(screenshot_path)
|
||||
|
||||
# Verify data was set before commit
|
||||
assert archive.file_group.data['screenshot_path'] == str(screenshot_path), \
|
||||
f"Data not set correctly before commit: {archive.file_group.data.get('screenshot_path')}"
|
||||
|
||||
test_session.commit()
|
||||
|
||||
# Refresh the archive to get latest data from DB
|
||||
test_session.expire_all()
|
||||
archive = test_session.query(Archive).filter_by(id=archive.id).one()
|
||||
|
||||
# Verify screenshot was set
|
||||
assert archive.screenshot_path == screenshot_path
|
||||
assert archive.screenshot_file is not None
|
||||
# Verify FileGroup.data was persisted to DB (FancyJSON converts strings back to Path objects)
|
||||
assert 'screenshot_path' in archive.file_group.data
|
||||
assert str(archive.file_group.data['screenshot_path']) == str(screenshot_path)
|
||||
|
||||
# Verify file is tracked in FileGroup.files
|
||||
screenshot_files = archive.file_group.my_files('image/')
|
||||
assert len(screenshot_files) > 0
|
||||
assert screenshot_files[0]['path'] == screenshot_path
|
||||
|
||||
# ========== Test 2: Error when trying to set screenshot on archive that already has one ==========
|
||||
original_screenshot = archive.screenshot_path
|
||||
assert original_screenshot is not None
|
||||
|
||||
# Try to set another screenshot
|
||||
new_screenshot_path = archive.singlefile_path.parent / 'another_screenshot.png'
|
||||
new_screenshot_path.write_bytes(b'fake image data')
|
||||
|
||||
# Should raise ValueError
|
||||
with pytest.raises(ValueError, match='already has a screenshot'):
|
||||
archive.set_screenshot(new_screenshot_path)
|
||||
|
||||
# Verify original screenshot is unchanged
|
||||
assert archive.screenshot_path == original_screenshot
|
||||
|
||||
# ========== Test 3: Error when trying to set non-existent screenshot file ==========
|
||||
archive_no_screenshot = archive_factory('example.com', 'https://example.com/test2', 'Test Archive 2', screenshot=False)
|
||||
test_session.commit()
|
||||
|
||||
# Try to set a screenshot that doesn't exist
|
||||
nonexistent_path = pathlib.Path('/tmp/does_not_exist.png')
|
||||
|
||||
# Should raise ValueError
|
||||
with pytest.raises(ValueError, match='does not exist'):
|
||||
archive_no_screenshot.set_screenshot(nonexistent_path)
|
||||
|
||||
# Verify no screenshot was set
|
||||
assert archive_no_screenshot.screenshot_path is None
|
||||
|
|
|
|||
|
|
@ -56,6 +56,7 @@ def attach_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.switches_lock = multiprocessing.Lock()
|
||||
app.shared_ctx.switches_changed = multiprocessing.Event()
|
||||
app.shared_ctx.archive_singlefiles = multiprocessing.Queue()
|
||||
app.shared_ctx.archive_screenshots = multiprocessing.Queue()
|
||||
|
||||
# Warnings
|
||||
app.shared_ctx.warn_once = manager.dict()
|
||||
|
|
@ -118,6 +119,12 @@ def reset_shared_contexts(app: Sanic):
|
|||
app.shared_ctx.archive_singlefiles.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
while True:
|
||||
# Clear out any pending screenshot generation switches.
|
||||
try:
|
||||
app.shared_ctx.archive_screenshots.get_nowait()
|
||||
except queue.Empty:
|
||||
break
|
||||
|
||||
# Events.
|
||||
app.shared_ctx.single_tasks_started.clear()
|
||||
|
|
|
|||
|
|
@ -100,6 +100,14 @@ class Events:
|
|||
def send_upload_archive_failed(cls, message: str = None):
|
||||
send_event('upload_archive_failed', message, subject='upload')
|
||||
|
||||
@classmethod
|
||||
def send_screenshot_generated(cls, message: str = None, url: str = None):
|
||||
send_event('screenshot_generated', message, subject='screenshot', url=url)
|
||||
|
||||
@classmethod
|
||||
def send_screenshot_generation_failed(cls, message: str = None):
|
||||
send_event('screenshot_generation_failed', message, subject='screenshot')
|
||||
|
||||
|
||||
def log_event(event: str, message: str = None, action: str = None, subject: str = None):
|
||||
log = f'{event=}'
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue