mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
Archives can be compressed (SingleFileZ self-extracting HTML) (#477)
* Archives can be compressed (SingleFileZ self-extracting HTML). A new compress_singlefile download setting runs single-file with --compress-content, creating a self-extracting HTML file which is also a valid ZIP. Only the singlefile .html is compressed; readability files are always written uncompressed. Readability, screenshots, titles, and URLs are extracted from the index.html inside the ZIP. A Compress checkbox in the Archive and RSS download forms remembers the user's choice in local storage. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
parent
bfbfa0710b
commit
0a776d030f
11 changed files with 409 additions and 41 deletions
|
|
@ -504,6 +504,22 @@ function AdvancedVideoSettings({form, isVideoLevel = false, isConfigLoaded = tru
|
|||
</Accordion>
|
||||
}
|
||||
|
||||
export function CompressSinglefileToggle({form}) {
|
||||
return <>
|
||||
<InfoHeader
|
||||
headerSize='h4'
|
||||
headerContent='Compress'
|
||||
popupContent='Create a compressed, self-extracting (SingleFileZ) archive. The HTML file is smaller, but not
|
||||
human-readable. Readability files are never compressed.'
|
||||
/>
|
||||
<ToggleForm
|
||||
form={form}
|
||||
name='compress_singlefile'
|
||||
path='settings.compress_singlefile'
|
||||
/>
|
||||
</>
|
||||
}
|
||||
|
||||
function AdvancedDownloadSettings({form}) {
|
||||
const [active, setActive] = React.useState(false);
|
||||
|
||||
|
|
@ -1096,6 +1112,9 @@ export function ArchiveDownloadForm({
|
|||
}) {
|
||||
const [showMessage, setShowMessage] = React.useState(false);
|
||||
|
||||
// Remember the user's compression choice between downloads.
|
||||
const [defaultCompressSinglefile, setDefaultCompressSinglefile] = useLocalStorage('compress_singlefile', false);
|
||||
|
||||
const submitter = propSubmitter || (async (formData) => {
|
||||
const downloadData = {
|
||||
downloader: formData.downloader,
|
||||
|
|
@ -1112,6 +1131,7 @@ export function ArchiveDownloadForm({
|
|||
tag_names: [],
|
||||
settings: {
|
||||
skip_already_downloaded: false,
|
||||
compress_singlefile: defaultCompressSinglefile,
|
||||
},
|
||||
};
|
||||
|
||||
|
|
@ -1130,6 +1150,14 @@ export function ArchiveDownloadForm({
|
|||
onSuccess,
|
||||
});
|
||||
|
||||
// Persist the user's compression choice in local storage.
|
||||
React.useEffect(() => {
|
||||
const {compress_singlefile} = form.formData.settings;
|
||||
if (compress_singlefile !== undefined && compress_singlefile !== defaultCompressSinglefile) {
|
||||
setDefaultCompressSinglefile(compress_singlefile);
|
||||
}
|
||||
}, [form.formData, defaultCompressSinglefile]);
|
||||
|
||||
const localOnCancel = (e) => {
|
||||
if (e) e.preventDefault();
|
||||
if (onCancel) {
|
||||
|
|
@ -1156,6 +1184,11 @@ export function ArchiveDownloadForm({
|
|||
<DownloadTagsSelector form={form}/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<CompressSinglefileToggle form={form}/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<AdvancedDownloadSettings form={form}/>
|
||||
|
|
@ -1179,6 +1212,8 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
|
||||
|
||||
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
|
||||
// Remember the user's compression choice between downloads.
|
||||
const [defaultCompressSinglefile, setDefaultCompressSinglefile] = useLocalStorage('compress_singlefile', false);
|
||||
|
||||
submitter = submitter || (async (formData) => {
|
||||
const downloadData = {
|
||||
|
|
@ -1202,6 +1237,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
frequency: weeklyOption.value,
|
||||
sub_downloader: null,
|
||||
settings: {
|
||||
compress_singlefile: defaultCompressSinglefile,
|
||||
excluded_urls: null,
|
||||
title_exclude: null,
|
||||
title_include: null,
|
||||
|
|
@ -1272,6 +1308,14 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
}
|
||||
}, [form.formData]);
|
||||
|
||||
// Persist the user's compression choice in local storage.
|
||||
React.useEffect(() => {
|
||||
const {compress_singlefile} = form.formData.settings;
|
||||
if (compress_singlefile !== undefined && compress_singlefile !== defaultCompressSinglefile) {
|
||||
setDefaultCompressSinglefile(compress_singlefile);
|
||||
}
|
||||
}, [form.formData, defaultCompressSinglefile]);
|
||||
|
||||
// Default to "new" download buttons.
|
||||
actions = actions || DownloadFormButtons;
|
||||
const actionsElm = actions({onDelete, onCancel, form});
|
||||
|
|
@ -1333,11 +1377,18 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
|
|||
</Grid.Row>
|
||||
</>;
|
||||
} else if (form.formData.sub_downloader === Downloaders.Archive) {
|
||||
downloaderRows = <Grid.Row>
|
||||
<Grid.Column>
|
||||
<ExcludedUrls form={form}/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>;
|
||||
downloaderRows = <>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<ExcludedUrls form={form}/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
<Grid.Row>
|
||||
<Grid.Column>
|
||||
<CompressSinglefileToggle form={form}/>
|
||||
</Grid.Column>
|
||||
</Grid.Row>
|
||||
</>;
|
||||
}
|
||||
|
||||
return <Form>
|
||||
|
|
|
|||
|
|
@ -5,6 +5,7 @@ This file is a simple Python/Sanic wrapper around the single-file CLI command.
|
|||
import asyncio
|
||||
import base64
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import logging
|
||||
import os.path
|
||||
|
|
@ -13,6 +14,7 @@ import subprocess
|
|||
import sys
|
||||
import tempfile
|
||||
import traceback
|
||||
import zipfile
|
||||
from json import JSONDecodeError
|
||||
|
||||
from sanic import Sanic, response
|
||||
|
|
@ -83,15 +85,17 @@ async def check_output(cmd, always_log_stderr: bool = False, cwd=None, timeout:
|
|||
raise TimeoutError(f"Command '{cmd}' timed out after {timeout} seconds")
|
||||
|
||||
|
||||
async def call_single_file(url) -> bytes:
|
||||
async def call_single_file(url, compress: bool = False) -> bytes:
|
||||
"""
|
||||
Call the CLI command for SingleFile.
|
||||
|
||||
See https://github.com/gildas-lormeau/SingleFile
|
||||
"""
|
||||
logger.info(f'archiving {url}')
|
||||
# --compress-content creates a compressed (SingleFileZ) self-extracting HTML file.
|
||||
cmd = f'{SINGLEFILE_PATH}' \
|
||||
r' --dump-content ' \
|
||||
f'{" --compress-content " if compress else ""}' \
|
||||
f' "{url}"'
|
||||
logger.debug(f'archive cmd: {cmd}')
|
||||
stdout, stderr, return_code = await check_output(cmd, always_log_stderr=True, timeout=3 * 60)
|
||||
|
|
@ -152,6 +156,27 @@ async def take_screenshot(url: str) -> bytes:
|
|||
return png
|
||||
|
||||
|
||||
def extract_compressed_singlefile(singlefile: bytes, tmp_dir: str) -> str:
|
||||
"""A compressed (SingleFileZ) singlefile is also a valid ZIP containing the uncompressed page as
|
||||
index.html (plus its resources). Extract it so readability/screenshot can use the real HTML.
|
||||
|
||||
Returns the path to the extracted index.html, or an empty string if the singlefile is not compressed.
|
||||
"""
|
||||
buf = io.BytesIO(singlefile)
|
||||
if not zipfile.is_zipfile(buf):
|
||||
return ''
|
||||
try:
|
||||
with zipfile.ZipFile(buf) as zip_:
|
||||
if 'index.html' not in zip_.namelist():
|
||||
return ''
|
||||
extracted = pathlib.Path(tmp_dir) / 'extracted'
|
||||
extracted.mkdir(exist_ok=True)
|
||||
zip_.extractall(extracted)
|
||||
return str(extracted / 'index.html')
|
||||
except zipfile.BadZipFile:
|
||||
return ''
|
||||
|
||||
|
||||
def prepare_bytes(b: bytes) -> str:
|
||||
"""
|
||||
Compress and encode bytes for smaller response.
|
||||
|
|
@ -181,16 +206,19 @@ async def post_screenshot(request: Request):
|
|||
|
||||
# Write singlefile to temp file and screenshot it
|
||||
# Use html suffix so chrome screenshot recognizes it as an HTML file
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
|
||||
fh.write(singlefile)
|
||||
fh.flush()
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
singlefile_path = pathlib.Path(tmp_dir) / 'singlefile.html'
|
||||
singlefile_path.write_bytes(singlefile)
|
||||
|
||||
# A compressed (SingleFileZ) singlefile must be extracted before it can be screenshot.
|
||||
extractable = extract_compressed_singlefile(singlefile, tmp_dir) or str(singlefile_path)
|
||||
|
||||
screenshot = None
|
||||
try:
|
||||
# Screenshot the local singlefile
|
||||
screenshot = await take_screenshot(f'file://{fh.name}')
|
||||
screenshot = await take_screenshot(f'file://{extractable}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e)
|
||||
logger.error(f'Failed to take screenshot of {extractable}', exc_info=e)
|
||||
|
||||
# Fall back to URL if local screenshot failed
|
||||
if not screenshot:
|
||||
|
|
@ -225,17 +253,23 @@ async def post_archive(request: Request):
|
|||
# May have been passed the singlefile contents, do the extractions and return.
|
||||
singlefile = request.json.get('singlefile')
|
||||
singlefile = base64.b64decode(singlefile) if singlefile else None
|
||||
compress = bool(request.json.get('compress'))
|
||||
|
||||
# Use provided singlefile, or fetch and create from the internet.
|
||||
singlefile = singlefile or await call_single_file(url)
|
||||
singlefile = singlefile or await call_single_file(url, compress=compress)
|
||||
|
||||
# Use html suffix so chrome screenshot recognizes it as an HTML file.
|
||||
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
|
||||
fh.write(singlefile)
|
||||
fh.flush()
|
||||
with tempfile.TemporaryDirectory() as tmp_dir:
|
||||
singlefile_path = pathlib.Path(tmp_dir) / 'singlefile.html'
|
||||
singlefile_path.write_bytes(singlefile)
|
||||
|
||||
# A compressed (SingleFileZ) singlefile contains the uncompressed page; readability and
|
||||
# screenshot must use the extracted index.html.
|
||||
extractable = extract_compressed_singlefile(singlefile, tmp_dir) or str(singlefile_path)
|
||||
|
||||
readability = None
|
||||
try:
|
||||
readability = await extract_readability(fh.name, url)
|
||||
readability = await extract_readability(extractable, url)
|
||||
except Exception as e:
|
||||
# Readability had error, but its is not required.
|
||||
logger.error(f'Failed to get readability', exc_info=e)
|
||||
|
|
@ -243,9 +277,9 @@ async def post_archive(request: Request):
|
|||
screenshot = None
|
||||
try:
|
||||
# Screenshot the local singlefile, if that fails try the URL.
|
||||
screenshot = await take_screenshot(f'file://{fh.name}')
|
||||
screenshot = await take_screenshot(f'file://{extractable}')
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e)
|
||||
logger.error(f'Failed to take screenshot of {extractable}', exc_info=e)
|
||||
|
||||
if not screenshot:
|
||||
logger.warning(f'Failed to screenshot local singlefile attempting to screenshot: {url}')
|
||||
|
|
|
|||
|
|
@ -80,8 +80,9 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
download: Download = None) -> ExecutedArchive:
|
||||
"""Run the archive container or local singlefile + readability/screenshot extraction,
|
||||
then write the artifacts to disk. No DB."""
|
||||
compress = bool(prepared.settings.get('compress_singlefile'))
|
||||
if DOCKERIZED or PYTEST:
|
||||
singlefile, readability, screenshot = await request_archive(prepared.url)
|
||||
singlefile, readability, screenshot = await request_archive(prepared.url, compress=compress)
|
||||
artifacts = SinglefileArtifacts(url=prepared.url, singlefile=singlefile,
|
||||
readability=readability, screenshot=screenshot)
|
||||
else:
|
||||
|
|
@ -143,6 +144,8 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
'--user-agent', user_agent,
|
||||
'--dump-content',
|
||||
'--load-deferred-images-dispatch-scroll-event',
|
||||
# Create a compressed (SingleFileZ) self-extracting HTML file.
|
||||
*(('--compress-content',) if prepared.settings.get('compress_singlefile') else ()),
|
||||
prepared.url,
|
||||
)
|
||||
cwd = pathlib.Path('/home/wrolpi')
|
||||
|
|
@ -153,7 +156,8 @@ class ArchiveDownloader(Downloader, ABC):
|
|||
cmd, cwd, ctx=ctx)
|
||||
|
||||
stderr = result.stderr.decode()
|
||||
log_output = stderr or result.stdout.decode() or 'No stderr or stdout!'
|
||||
# stdout may be a binary ZIP when --compress-content is used.
|
||||
log_output = stderr or result.stdout.decode(errors='replace') or 'No stderr or stdout!'
|
||||
|
||||
if result.return_code != 0:
|
||||
e = ChildProcessError(log_output[:1000])
|
||||
|
|
@ -244,7 +248,7 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
|
|||
if readability_json_path:
|
||||
title = get_title(readability_json_path)
|
||||
if not title:
|
||||
title = get_title_from_html(singlefile_path.read_text())
|
||||
title = get_title_from_html(lib.read_singlefile_html(singlefile_path))
|
||||
|
||||
contents = None
|
||||
if readability_txt_path:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,8 @@
|
|||
import datetime
|
||||
import io
|
||||
import json
|
||||
import pathlib
|
||||
import zipfile
|
||||
from typing import List
|
||||
from uuid import uuid4
|
||||
|
||||
|
|
@ -101,3 +103,28 @@ def archive_factory(test_session, archive_directory, make_files_structure, image
|
|||
return archive
|
||||
|
||||
return _
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def compressed_singlefile_factory():
|
||||
"""Mimic the file created by `single-file --compress-content`: an HTML prelude containing the
|
||||
SingleFile comment header, followed by a ZIP containing the uncompressed page as index.html."""
|
||||
|
||||
def _(url: str = 'https://example.com', title: str = 'compressed title') -> bytes:
|
||||
prelude = '<!DOCTYPE html> <html data-sfz><!--\n' \
|
||||
' Page saved with SingleFile \n' \
|
||||
f' url: {url} \n' \
|
||||
' saved date: Thu Jun 11 2026 22:49:39 GMT+0000 (Coordinated Universal Time)\n' \
|
||||
'--><meta charset=windows-1252><title></title>'
|
||||
index_html = '<html><!--\n Page saved with SingleFile \n' \
|
||||
f' url: {url} \n' \
|
||||
f'-->\n<head><title>{title}</title></head><body>test compressed singlefile</body></html>'
|
||||
buf = io.BytesIO()
|
||||
buf.write(prelude.encode())
|
||||
# Deflate like the real single-file CLI; the page must not be readable as plain text.
|
||||
with zipfile.ZipFile(buf, 'a', compression=zipfile.ZIP_DEFLATED) as zip_:
|
||||
zip_.writestr('index.html', index_html)
|
||||
zip_.writestr('manifest.json', '{}')
|
||||
return buf.getvalue()
|
||||
|
||||
return _
|
||||
|
|
|
|||
|
|
@ -1,11 +1,13 @@
|
|||
import base64
|
||||
import gzip
|
||||
import io
|
||||
import json
|
||||
import os
|
||||
import pathlib
|
||||
import re
|
||||
import shlex
|
||||
import tempfile
|
||||
import zipfile
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime
|
||||
from json import JSONDecodeError
|
||||
|
|
@ -630,11 +632,12 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P
|
|||
ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response.
|
||||
|
||||
|
||||
async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]:
|
||||
async def request_archive(url: str, singlefile: str = None, compress: bool = False) \
|
||||
-> Tuple[str | bytes, Optional[dict], Optional[str]]:
|
||||
"""Send a request to the archive service to archive the URL."""
|
||||
logger.info(f'Sending archive request to archive service: {url}')
|
||||
|
||||
data = dict(url=url, singlefile=singlefile)
|
||||
data = dict(url=url, singlefile=singlefile, compress=compress)
|
||||
try:
|
||||
async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
|
||||
status = response.status
|
||||
|
|
@ -662,7 +665,9 @@ async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Option
|
|||
# Decode and decompress.
|
||||
singlefile = base64.b64decode(singlefile)
|
||||
singlefile = gzip.decompress(singlefile)
|
||||
singlefile = singlefile.decode()
|
||||
# A compressed (SingleFileZ) singlefile is binary and must stay bytes; only decode regular singlefiles.
|
||||
if not is_compressed_singlefile(singlefile):
|
||||
singlefile = singlefile.decode()
|
||||
|
||||
if not screenshot:
|
||||
logger.info(f'Failed to get screenshot for {url=}')
|
||||
|
|
@ -717,20 +722,23 @@ class WrittenArchiveFiles:
|
|||
paths: List[pathlib.Path]
|
||||
|
||||
|
||||
def write_archive_files(url: str, singlefile: str, readability: Optional[dict],
|
||||
def write_archive_files(url: str, singlefile: str | bytes, readability: Optional[dict],
|
||||
screenshot: Optional[bytes],
|
||||
destination: Optional[pathlib.Path] = None) -> WrittenArchiveFiles:
|
||||
"""Pure I/O: write the singlefile/readability/screenshot/json files to disk.
|
||||
|
||||
Returns the URL and the list of paths to register, leaving DB work to register_archive.
|
||||
"""
|
||||
compressed = isinstance(singlefile, bytes) and is_compressed_singlefile(singlefile)
|
||||
|
||||
readability = readability.copy() if readability else readability
|
||||
# First try to get the title from Readability.
|
||||
title = readability.get('title') if readability else None
|
||||
|
||||
if not title and singlefile:
|
||||
# Try to get the title ourselves from the HTML.
|
||||
title = get_title_from_html(singlefile, url=url)
|
||||
# Try to get the title ourselves from the HTML. A compressed singlefile contains the
|
||||
# uncompressed page as index.html.
|
||||
title = get_title_from_html(get_compressed_singlefile_index(singlefile) if compressed else singlefile, url=url)
|
||||
if readability:
|
||||
# Readability could not find title, lets use ours.
|
||||
readability['title'] = title
|
||||
|
|
@ -751,8 +759,12 @@ def write_archive_files(url: str, singlefile: str, readability: Optional[dict],
|
|||
archive_files.readability_txt = archive_files.readability = None
|
||||
|
||||
# Store the single-file HTML in its own file.
|
||||
singlefile = format_html_string(singlefile)
|
||||
archive_files.singlefile.write_text(singlefile)
|
||||
if compressed:
|
||||
# A compressed (SingleFileZ) singlefile is a binary ZIP; prettifying would corrupt it.
|
||||
archive_files.singlefile.write_bytes(singlefile)
|
||||
else:
|
||||
singlefile = format_html_string(singlefile)
|
||||
archive_files.singlefile.write_text(singlefile)
|
||||
|
||||
if screenshot:
|
||||
archive_files.screenshot.write_bytes(screenshot)
|
||||
|
|
@ -992,10 +1004,41 @@ def link_domain_and_downloads(session: Session):
|
|||
|
||||
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
|
||||
|
||||
COMPRESSED_SINGLEFILE_INDEX = 'index.html'
|
||||
|
||||
|
||||
def is_compressed_singlefile(singlefile: bytes) -> bool:
|
||||
"""A compressed (SingleFileZ) singlefile is a self-extracting HTML file which is also a valid ZIP
|
||||
containing the uncompressed page as index.html."""
|
||||
buf = io.BytesIO(singlefile)
|
||||
if not zipfile.is_zipfile(buf):
|
||||
return False
|
||||
try:
|
||||
with zipfile.ZipFile(buf) as zip_:
|
||||
return COMPRESSED_SINGLEFILE_INDEX in zip_.namelist()
|
||||
except zipfile.BadZipFile:
|
||||
return False
|
||||
|
||||
|
||||
def get_compressed_singlefile_index(singlefile: bytes) -> bytes:
|
||||
"""Extract the uncompressed page (index.html) from a compressed (SingleFileZ) singlefile."""
|
||||
with zipfile.ZipFile(io.BytesIO(singlefile)) as zip_:
|
||||
return zip_.read(COMPRESSED_SINGLEFILE_INDEX)
|
||||
|
||||
|
||||
def read_singlefile_html(path: pathlib.Path) -> bytes:
|
||||
"""Read the page HTML of the singlefile at `path`; extract the uncompressed page when the
|
||||
singlefile is compressed (SingleFileZ)."""
|
||||
contents = path.read_bytes()
|
||||
if is_compressed_singlefile(contents):
|
||||
return get_compressed_singlefile_index(contents)
|
||||
return contents
|
||||
|
||||
|
||||
def get_url_from_singlefile(html: bytes | str) -> str:
|
||||
"""Extract URL from SingleFile contents."""
|
||||
html = html.decode() if isinstance(html, bytes) else html
|
||||
# Ignore decode errors; a compressed singlefile has a plain-text prelude but a binary ZIP tail.
|
||||
html = html.decode(errors='ignore') if isinstance(html, bytes) else html
|
||||
if SINGLEFILE_HEADER not in html:
|
||||
raise RuntimeError(f'Not a singlefile!')
|
||||
|
||||
|
|
@ -1185,9 +1228,10 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
|
|||
if ARCHIVE_MATCHER.match(path.name):
|
||||
# A file name matching exactly the Archive format from WROLPi should always be an Archive.
|
||||
return True
|
||||
with path.open('rt') as fh:
|
||||
with path.open('rb') as fh:
|
||||
# Lastly, try to read the header of this HTML file. Read a large part of the head because URLs can be long.
|
||||
header = fh.read(1000)
|
||||
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
|
||||
header = fh.read(1000).decode(errors='ignore')
|
||||
if SINGLEFILE_HEADER in header:
|
||||
return True
|
||||
return False
|
||||
|
|
@ -1463,24 +1507,30 @@ async def extract_singlefile_artifacts(singlefile: bytes) -> SinglefileArtifacts
|
|||
url = get_url_from_singlefile(singlefile)
|
||||
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
|
||||
|
||||
# A compressed (SingleFileZ) singlefile contains the uncompressed page as index.html; use that
|
||||
# for readability and screenshot, but keep the compressed bytes as the singlefile.
|
||||
compressed = is_compressed_singlefile(singlefile)
|
||||
extractable = get_compressed_singlefile_index(singlefile) if compressed else singlefile
|
||||
|
||||
if DOCKERIZED:
|
||||
# Perform the archive in the Archive docker container. (Typically in the development environment).
|
||||
encoded = base64.b64encode(singlefile).decode()
|
||||
logger.debug(f'extract_singlefile_artifacts sending to archive service: {url}')
|
||||
singlefile, readability, screenshot = await request_archive(url, singlefile=encoded)
|
||||
singlefile, readability, screenshot = await request_archive(url, singlefile=encoded, compress=compressed)
|
||||
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
|
||||
else:
|
||||
# JSON from readability-extractor
|
||||
readability = dict()
|
||||
try:
|
||||
logger.debug(f'extract_singlefile_artifacts extracting readability: {url}')
|
||||
readability = await html_to_readability(singlefile, url)
|
||||
readability = await html_to_readability(extractable, url)
|
||||
except RuntimeError as e:
|
||||
logger.error(f'Failed to extract readability from: {url}', exc_info=e)
|
||||
|
||||
screenshot = None
|
||||
try:
|
||||
logger.debug(f'extract_singlefile_artifacts creating screenshot: {url}')
|
||||
screenshot = html_screenshot(singlefile)
|
||||
screenshot = html_screenshot(extractable)
|
||||
except Exception as e:
|
||||
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
|
||||
|
||||
|
|
@ -1620,6 +1670,8 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
|
|||
else:
|
||||
logger.debug(f'Generating screenshot locally for Archive {archive_id}')
|
||||
singlefile_contents = singlefile_path.read_bytes()
|
||||
if is_compressed_singlefile(singlefile_contents):
|
||||
singlefile_contents = get_compressed_singlefile_index(singlefile_contents)
|
||||
screenshot_bytes = html_screenshot(singlefile_contents)
|
||||
|
||||
if not screenshot_bytes:
|
||||
|
|
|
|||
|
|
@ -303,8 +303,9 @@ class Archive(Base, ModelHelper):
|
|||
# This data has already been read.
|
||||
return
|
||||
|
||||
with path.open('rt') as fh:
|
||||
head = fh.read(1000)
|
||||
with path.open('rb') as fh:
|
||||
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
|
||||
head = fh.read(1000).decode(errors='ignore')
|
||||
if 'Page saved with SingleFile' not in head:
|
||||
logger.error(f'Could not find SingleFile header in {self}')
|
||||
return
|
||||
|
|
@ -342,14 +343,16 @@ class Archive(Base, ModelHelper):
|
|||
|
||||
def apply_singlefile_title(self):
|
||||
"""Get the title from the Singlefile, if it's missing."""
|
||||
# Local import to avoid circular import within archive module
|
||||
from modules.archive import lib
|
||||
if self.singlefile_path and not self.file_group.title:
|
||||
self.file_group.title = get_title_from_html(self.singlefile_path.read_text())
|
||||
self.file_group.title = get_title_from_html(lib.read_singlefile_html(self.singlefile_path))
|
||||
|
||||
def apply_metadata(self):
|
||||
"""Read and apply <meta> (and more) data from the Singlefile HTML."""
|
||||
# Local import to avoid circular import within archive module
|
||||
from modules.archive import lib
|
||||
contents = self.singlefile_path.read_bytes()
|
||||
contents = lib.read_singlefile_html(self.singlefile_path)
|
||||
|
||||
metadata = lib.parse_article_html_metadata(contents)
|
||||
if metadata.author:
|
||||
|
|
|
|||
187
modules/archive/test/test_compressed_singlefile.py
Normal file
187
modules/archive/test/test_compressed_singlefile.py
Normal file
|
|
@ -0,0 +1,187 @@
|
|||
"""Tests for compressed (SingleFileZ) singlefile Archives.
|
||||
|
||||
A compressed singlefile is a self-extracting HTML file which is also a valid ZIP containing the
|
||||
uncompressed page as index.html. Only the singlefile .html is compressed; readability files are
|
||||
always written uncompressed.
|
||||
"""
|
||||
import pathlib
|
||||
import zipfile
|
||||
|
||||
import pytest
|
||||
|
||||
import modules.archive
|
||||
from modules.archive import archive_downloader, PreparedArchive, model_archive
|
||||
from modules.archive.conftest import make_test_ctx
|
||||
from modules.archive.lib import (
|
||||
is_compressed_singlefile,
|
||||
get_compressed_singlefile_index,
|
||||
get_url_from_singlefile,
|
||||
is_singlefile_file,
|
||||
write_archive_files,
|
||||
)
|
||||
from modules.archive.test.test_lib import make_fake_archive_result
|
||||
from wrolpi.downloader import Download
|
||||
from wrolpi.files.lib import get_mimetype, _mimetype_suffix_map
|
||||
from wrolpi.files.models import FileGroup
|
||||
from wrolpi.schema import DownloadRequest
|
||||
|
||||
|
||||
def test_is_compressed_singlefile(compressed_singlefile_factory, singlefile_contents_factory):
|
||||
compressed = compressed_singlefile_factory()
|
||||
assert is_compressed_singlefile(compressed) is True
|
||||
|
||||
# A regular singlefile is not compressed.
|
||||
assert is_compressed_singlefile(singlefile_contents_factory().encode()) is False
|
||||
# Garbage is not compressed.
|
||||
assert is_compressed_singlefile(b'not a zip') is False
|
||||
|
||||
# The uncompressed page can be extracted.
|
||||
index = get_compressed_singlefile_index(compressed)
|
||||
assert b'test compressed singlefile' in index
|
||||
assert b'<title>compressed title</title>' in index
|
||||
|
||||
|
||||
def test_get_url_from_compressed_singlefile(compressed_singlefile_factory):
|
||||
"""The URL can be extracted from the plain-text prelude despite the binary ZIP tail."""
|
||||
compressed = compressed_singlefile_factory(url='https://wrolpi.org/some-article')
|
||||
assert get_url_from_singlefile(compressed) == 'https://wrolpi.org/some-article'
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
'name,expected', [
|
||||
('2000-01-01-00-00-00_Some NA.html', True), # Matches the WROLPi Archive file name.
|
||||
('foo.html', True), # Detected by the SingleFile header in the prelude.
|
||||
('foo.txt', False),
|
||||
]
|
||||
)
|
||||
def test_is_singlefile_file_compressed(name, expected, make_files_structure, compressed_singlefile_factory):
|
||||
"""A compressed singlefile is recognized even though it has a binary ZIP tail."""
|
||||
path, = make_files_structure([name])
|
||||
path.write_bytes(compressed_singlefile_factory())
|
||||
assert is_singlefile_file(path) == expected
|
||||
|
||||
|
||||
def test_compressed_singlefile_mimetype(make_files_structure, compressed_singlefile_factory):
|
||||
"""A compressed singlefile must be `text/html` so the archive modeler finds it."""
|
||||
# `magic` reports application/octet-stream for the real (binary) compressed singlefile.
|
||||
assert _mimetype_suffix_map(pathlib.Path('foo.html'), 'application/octet-stream') == 'text/html'
|
||||
|
||||
path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
|
||||
path.write_bytes(compressed_singlefile_factory())
|
||||
assert get_mimetype(path) == 'text/html'
|
||||
|
||||
|
||||
def test_write_archive_files_compressed(test_directory, test_session, image_bytes_factory,
|
||||
compressed_singlefile_factory):
|
||||
"""The compressed singlefile is written verbatim (prettifying would corrupt the ZIP);
|
||||
readability files are written uncompressed."""
|
||||
compressed = compressed_singlefile_factory()
|
||||
readability = dict(content='<html>readability content</html>', textContent='readability text', title='a title')
|
||||
destination = test_directory / 'archive/example.com'
|
||||
destination.mkdir(parents=True)
|
||||
|
||||
written = write_archive_files('https://example.com', compressed, readability, image_bytes_factory(),
|
||||
destination=destination)
|
||||
|
||||
singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)
|
||||
# The file on disk is unchanged, and still a valid ZIP.
|
||||
assert singlefile_path.read_bytes() == compressed
|
||||
assert zipfile.is_zipfile(singlefile_path)
|
||||
|
||||
# Readability files are plain text, not compressed.
|
||||
readability_path = next(i for i in written.paths if i.name.endswith('.readability.html'))
|
||||
assert 'readability content' in readability_path.read_text()
|
||||
readability_txt_path = next(i for i in written.paths if i.name.endswith('.readability.txt'))
|
||||
assert 'readability text' in readability_txt_path.read_text()
|
||||
|
||||
|
||||
def test_write_archive_files_compressed_title_fallback(test_directory, test_session, compressed_singlefile_factory):
|
||||
"""Without a readability title, the title comes from the ZIP's index.html."""
|
||||
compressed = compressed_singlefile_factory(title='the fallback title')
|
||||
destination = test_directory / 'archive/example.com'
|
||||
destination.mkdir(parents=True)
|
||||
|
||||
written = write_archive_files('https://example.com', compressed, None, None, destination=destination)
|
||||
|
||||
singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)
|
||||
assert 'the fallback title' in singlefile_path.name
|
||||
assert singlefile_path.read_bytes() == compressed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_execute_download_compressed(test_session, test_directory, monkeypatch, compressed_singlefile_factory):
|
||||
"""The compress_singlefile setting is forwarded to the archive service, and the compressed
|
||||
bytes returned are written to disk unchanged."""
|
||||
compressed = compressed_singlefile_factory()
|
||||
_, readability, screenshot = make_fake_archive_result()
|
||||
request_archive_kwargs = {}
|
||||
|
||||
async def mock_request_archive(url, compress=False):
|
||||
request_archive_kwargs['compress'] = compress
|
||||
return compressed, readability, screenshot
|
||||
|
||||
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
|
||||
|
||||
destination = test_directory / 'archive/example.com'
|
||||
destination.mkdir(parents=True)
|
||||
prepared = PreparedArchive(
|
||||
url='https://example.com',
|
||||
destination=destination,
|
||||
settings={'compress_singlefile': True},
|
||||
)
|
||||
|
||||
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
|
||||
|
||||
assert request_archive_kwargs['compress'] is True
|
||||
singlefile_path = next(i for i in executed.written.paths if i.suffix == '.html' and '.readability' not in i.name)
|
||||
assert singlefile_path.read_bytes() == compressed
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_archive_compressed(async_client, test_session, test_directory, make_files_structure,
|
||||
compressed_singlefile_factory):
|
||||
"""A compressed singlefile on disk (without readability files) can be modeled as an Archive;
|
||||
the title and URL come from the compressed file itself.
|
||||
|
||||
async_client is required because get_or_create_domain_collection activates a Sanic switch."""
|
||||
path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
|
||||
path.write_bytes(compressed_singlefile_factory(url='https://example.com/article', title='compressed title'))
|
||||
|
||||
file_group = FileGroup.from_paths(test_session, path)
|
||||
test_session.flush()
|
||||
|
||||
archive = model_archive(test_session, file_group)
|
||||
test_session.commit()
|
||||
|
||||
assert archive.file_group.url == 'https://example.com/article'
|
||||
assert archive.file_group.title == 'compressed title'
|
||||
assert archive.collection.name == 'example.com'
|
||||
|
||||
|
||||
def test_download_request_compress_singlefile():
|
||||
"""compress_singlefile=True survives DownloadRequest settings validation."""
|
||||
request = DownloadRequest(
|
||||
urls=['https://example.com'],
|
||||
downloader='archive',
|
||||
settings=dict(compress_singlefile=True),
|
||||
)
|
||||
assert request.settings['compress_singlefile'] is True
|
||||
|
||||
|
||||
def test_rss_forwards_compress_singlefile(test_session):
|
||||
"""RSSDownloader passes compress_singlefile on to the child archive downloads."""
|
||||
from wrolpi.downloader import rss_downloader, ExecutedRSS
|
||||
|
||||
download = Download(
|
||||
url='https://example.com/feed',
|
||||
downloader='rss',
|
||||
sub_downloader='archive',
|
||||
settings={'compress_singlefile': True},
|
||||
)
|
||||
executed = ExecutedRSS(yt_channel_id=None, candidate_urls=['https://example.com/article'])
|
||||
|
||||
result = rss_downloader.finalize_download(test_session, download, executed)
|
||||
|
||||
assert result.success is True
|
||||
assert result.downloads == ['https://example.com/article']
|
||||
assert result.settings['compress_singlefile'] is True
|
||||
|
|
@ -1161,7 +1161,7 @@ async def test_archive_download_uses_domain_collection_directory(async_client, t
|
|||
# Mock request_archive to return fake archive data
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
|
||||
async def mock_request_archive(url):
|
||||
async def mock_request_archive(url, compress=False):
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
# Patch in both lib and archive module (imported at module level)
|
||||
|
|
@ -1223,7 +1223,7 @@ async def test_archive_download_explicit_destination_overrides_domain_collection
|
|||
# Mock request_archive to return fake archive data
|
||||
singlefile, readability, screenshot = make_fake_archive_result()
|
||||
|
||||
async def mock_request_archive(url):
|
||||
async def mock_request_archive(url, compress=False):
|
||||
return singlefile, readability, screenshot
|
||||
|
||||
# Patch in both lib and archive module (imported at module level)
|
||||
|
|
|
|||
|
|
@ -2230,6 +2230,9 @@ class RSSDownloader(Downloader):
|
|||
for key in CHANNEL_INHERITABLE_SETTINGS:
|
||||
if key in settings:
|
||||
next_download_settings[key] = settings[key]
|
||||
# Pass compression on to child archive downloads.
|
||||
if 'compress_singlefile' in settings:
|
||||
next_download_settings['compress_singlefile'] = settings['compress_singlefile']
|
||||
# Use download.destination column (settings['destination'] is legacy)
|
||||
if download.destination:
|
||||
next_download_settings['destination'] = str(download.destination)
|
||||
|
|
|
|||
|
|
@ -384,6 +384,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
|
|||
return 'model/3mf'
|
||||
if suffix.endswith('.mp4'):
|
||||
return 'video/mp4'
|
||||
if suffix.endswith('.html') or suffix.endswith('.htm'):
|
||||
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
|
||||
return 'text/html'
|
||||
if suffix.endswith('.hgt'):
|
||||
return 'application/octet-stream'
|
||||
if mimetype == 'text/plain':
|
||||
|
|
@ -414,6 +417,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
|
|||
if mimetype == 'application/x-subrip':
|
||||
# Fallback to old mimetype.
|
||||
return 'text/srt'
|
||||
if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'):
|
||||
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
|
||||
return 'text/html'
|
||||
if mimetype == 'application/zip' and suffix == '.docx':
|
||||
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
|
||||
if mimetype == 'application/zip' and suffix == '.cbz':
|
||||
|
|
|
|||
|
|
@ -260,6 +260,7 @@ class DownloadSettings:
|
|||
video_resolutions: List[str] = field(default_factory=list)
|
||||
parent_download_url: Optional[str] = None
|
||||
skip_already_downloaded: Optional[bool] = None
|
||||
compress_singlefile: Optional[bool] = None
|
||||
# Inheritable video settings (Optional so they're only set when user overrides)
|
||||
writesubtitles: Optional[bool] = None
|
||||
writeautomaticsub: Optional[bool] = None
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue