Archives can be compressed (SingleFileZ self-extracting HTML) (#477)

* Archives can be compressed (SingleFileZ self-extracting HTML).

A new compress_singlefile download setting runs single-file with
--compress-content, creating a self-extracting HTML file which is also a
valid ZIP.  Only the singlefile .html is compressed; readability files are
always written uncompressed.  Readability, screenshots, titles, and URLs
are extracted from the index.html inside the ZIP.  A Compress checkbox in
the Archive and RSS download forms remembers the user's choice in local
storage.

Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
lrnselfreliance 2026-06-11 17:54:22 -06:00 • committed by Roland
parent bfbfa0710b
commit 0a776d030f
No known key found for this signature in database
GPG key ID: 1BA620CE6C9A4680
11 changed files with 409 additions and 41 deletions

View file

@ -504,6 +504,22 @@ function AdvancedVideoSettings({form, isVideoLevel = false, isConfigLoaded = tru
</Accordion>
}
export function CompressSinglefileToggle({form}) {
return <>
<InfoHeader
headerSize='h4'
headerContent='Compress'
popupContent='Create a compressed, self-extracting (SingleFileZ) archive. The HTML file is smaller, but not
human-readable. Readability files are never compressed.'
/>
<ToggleForm
form={form}
name='compress_singlefile'
path='settings.compress_singlefile'
/>
</>
}
function AdvancedDownloadSettings({form}) {
const [active, setActive] = React.useState(false);
@ -1096,6 +1112,9 @@ export function ArchiveDownloadForm({
}) {
const [showMessage, setShowMessage] = React.useState(false);
// Remember the user's compression choice between downloads.
const [defaultCompressSinglefile, setDefaultCompressSinglefile] = useLocalStorage('compress_singlefile', false);
const submitter = propSubmitter || (async (formData) => {
const downloadData = {
downloader: formData.downloader,
@ -1112,6 +1131,7 @@ export function ArchiveDownloadForm({
tag_names: [],
settings: {
skip_already_downloaded: false,
compress_singlefile: defaultCompressSinglefile,
},
};
@ -1130,6 +1150,14 @@ export function ArchiveDownloadForm({
onSuccess,
});
// Persist the user's compression choice in local storage.
React.useEffect(() => {
const {compress_singlefile} = form.formData.settings;
if (compress_singlefile !== undefined && compress_singlefile !== defaultCompressSinglefile) {
setDefaultCompressSinglefile(compress_singlefile);
}
}, [form.formData, defaultCompressSinglefile]);
const localOnCancel = (e) => {
if (e) e.preventDefault();
if (onCancel) {
@ -1156,6 +1184,11 @@ export function ArchiveDownloadForm({
<DownloadTagsSelector form={form}/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<CompressSinglefileToggle form={form}/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<AdvancedDownloadSettings form={form}/>
@ -1179,6 +1212,8 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
const [userChangedResolutions, setUserChangedResolutions] = React.useState(false);
const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption);
// Remember the user's compression choice between downloads.
const [defaultCompressSinglefile, setDefaultCompressSinglefile] = useLocalStorage('compress_singlefile', false);
submitter = submitter || (async (formData) => {
const downloadData = {
@ -1202,6 +1237,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
frequency: weeklyOption.value,
sub_downloader: null,
settings: {
compress_singlefile: defaultCompressSinglefile,
excluded_urls: null,
title_exclude: null,
title_include: null,
@ -1272,6 +1308,14 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
}
}, [form.formData]);
// Persist the user's compression choice in local storage.
React.useEffect(() => {
const {compress_singlefile} = form.formData.settings;
if (compress_singlefile !== undefined && compress_singlefile !== defaultCompressSinglefile) {
setDefaultCompressSinglefile(compress_singlefile);
}
}, [form.formData, defaultCompressSinglefile]);
// Default to "new" download buttons.
actions = actions || DownloadFormButtons;
const actionsElm = actions({onDelete, onCancel, form});
@ -1333,11 +1377,18 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action
</Grid.Row>
</>;
} else if (form.formData.sub_downloader === Downloaders.Archive) {
downloaderRows = <Grid.Row>
<Grid.Column>
<ExcludedUrls form={form}/>
</Grid.Column>
</Grid.Row>;
downloaderRows = <>
<Grid.Row>
<Grid.Column>
<ExcludedUrls form={form}/>
</Grid.Column>
</Grid.Row>
<Grid.Row>
<Grid.Column>
<CompressSinglefileToggle form={form}/>
</Grid.Column>
</Grid.Row>
</>;
}
return <Form>

View file

@ -5,6 +5,7 @@ This file is a simple Python/Sanic wrapper around the single-file CLI command.
import asyncio
import base64
import gzip
import io
import json
import logging
import os.path
@ -13,6 +14,7 @@ import subprocess
import sys
import tempfile
import traceback
import zipfile
from json import JSONDecodeError
from sanic import Sanic, response
@ -83,15 +85,17 @@ async def check_output(cmd, always_log_stderr: bool = False, cwd=None, timeout:
raise TimeoutError(f"Command '{cmd}' timed out after {timeout} seconds")
async def call_single_file(url) -> bytes:
async def call_single_file(url, compress: bool = False) -> bytes:
"""
Call the CLI command for SingleFile.
See https://github.com/gildas-lormeau/SingleFile
"""
logger.info(f'archiving {url}')
# --compress-content creates a compressed (SingleFileZ) self-extracting HTML file.
cmd = f'{SINGLEFILE_PATH}' \
r' --dump-content ' \
f'{" --compress-content " if compress else ""}' \
f' "{url}"'
logger.debug(f'archive cmd: {cmd}')
stdout, stderr, return_code = await check_output(cmd, always_log_stderr=True, timeout=3 * 60)
@ -152,6 +156,27 @@ async def take_screenshot(url: str) -> bytes:
return png
def extract_compressed_singlefile(singlefile: bytes, tmp_dir: str) -> str:
"""A compressed (SingleFileZ) singlefile is also a valid ZIP containing the uncompressed page as
index.html (plus its resources). Extract it so readability/screenshot can use the real HTML.
Returns the path to the extracted index.html, or an empty string if the singlefile is not compressed.
"""
buf = io.BytesIO(singlefile)
if not zipfile.is_zipfile(buf):
return ''
try:
with zipfile.ZipFile(buf) as zip_:
if 'index.html' not in zip_.namelist():
return ''
extracted = pathlib.Path(tmp_dir) / 'extracted'
extracted.mkdir(exist_ok=True)
zip_.extractall(extracted)
return str(extracted / 'index.html')
except zipfile.BadZipFile:
return ''
def prepare_bytes(b: bytes) -> str:
"""
Compress and encode bytes for smaller response.
@ -181,16 +206,19 @@ async def post_screenshot(request: Request):
# Write singlefile to temp file and screenshot it
# Use html suffix so chrome screenshot recognizes it as an HTML file
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(singlefile)
fh.flush()
with tempfile.TemporaryDirectory() as tmp_dir:
singlefile_path = pathlib.Path(tmp_dir) / 'singlefile.html'
singlefile_path.write_bytes(singlefile)
# A compressed (SingleFileZ) singlefile must be extracted before it can be screenshot.
extractable = extract_compressed_singlefile(singlefile, tmp_dir) or str(singlefile_path)
screenshot = None
try:
# Screenshot the local singlefile
screenshot = await take_screenshot(f'file://{fh.name}')
screenshot = await take_screenshot(f'file://{extractable}')
except Exception as e:
logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e)
logger.error(f'Failed to take screenshot of {extractable}', exc_info=e)
# Fall back to URL if local screenshot failed
if not screenshot:
@ -225,17 +253,23 @@ async def post_archive(request: Request):
# May have been passed the singlefile contents, do the extractions and return.
singlefile = request.json.get('singlefile')
singlefile = base64.b64decode(singlefile) if singlefile else None
compress = bool(request.json.get('compress'))
# Use provided singlefile, or fetch and create from the internet.
singlefile = singlefile or await call_single_file(url)
singlefile = singlefile or await call_single_file(url, compress=compress)
# Use html suffix so chrome screenshot recognizes it as an HTML file.
with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh:
fh.write(singlefile)
fh.flush()
with tempfile.TemporaryDirectory() as tmp_dir:
singlefile_path = pathlib.Path(tmp_dir) / 'singlefile.html'
singlefile_path.write_bytes(singlefile)
# A compressed (SingleFileZ) singlefile contains the uncompressed page; readability and
# screenshot must use the extracted index.html.
extractable = extract_compressed_singlefile(singlefile, tmp_dir) or str(singlefile_path)
readability = None
try:
readability = await extract_readability(fh.name, url)
readability = await extract_readability(extractable, url)
except Exception as e:
# Readability had error, but its is not required.
logger.error(f'Failed to get readability', exc_info=e)
@ -243,9 +277,9 @@ async def post_archive(request: Request):
screenshot = None
try:
# Screenshot the local singlefile, if that fails try the URL.
screenshot = await take_screenshot(f'file://{fh.name}')
screenshot = await take_screenshot(f'file://{extractable}')
except Exception as e:
logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e)
logger.error(f'Failed to take screenshot of {extractable}', exc_info=e)
if not screenshot:
logger.warning(f'Failed to screenshot local singlefile attempting to screenshot: {url}')

View file

@ -80,8 +80,9 @@ class ArchiveDownloader(Downloader, ABC):
download: Download = None) -> ExecutedArchive:
"""Run the archive container or local singlefile + readability/screenshot extraction,
then write the artifacts to disk. No DB."""
compress = bool(prepared.settings.get('compress_singlefile'))
if DOCKERIZED or PYTEST:
singlefile, readability, screenshot = await request_archive(prepared.url)
singlefile, readability, screenshot = await request_archive(prepared.url, compress=compress)
artifacts = SinglefileArtifacts(url=prepared.url, singlefile=singlefile,
readability=readability, screenshot=screenshot)
else:
@ -143,6 +144,8 @@ class ArchiveDownloader(Downloader, ABC):
'--user-agent', user_agent,
'--dump-content',
'--load-deferred-images-dispatch-scroll-event',
# Create a compressed (SingleFileZ) self-extracting HTML file.
*(('--compress-content',) if prepared.settings.get('compress_singlefile') else ()),
prepared.url,
)
cwd = pathlib.Path('/home/wrolpi')
@ -153,7 +156,8 @@ class ArchiveDownloader(Downloader, ABC):
cmd, cwd, ctx=ctx)
stderr = result.stderr.decode()
log_output = stderr or result.stdout.decode() or 'No stderr or stdout!'
# stdout may be a binary ZIP when --compress-content is used.
log_output = stderr or result.stdout.decode(errors='replace') or 'No stderr or stdout!'
if result.return_code != 0:
e = ChildProcessError(log_output[:1000])
@ -244,7 +248,7 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive:
if readability_json_path:
title = get_title(readability_json_path)
if not title:
title = get_title_from_html(singlefile_path.read_text())
title = get_title_from_html(lib.read_singlefile_html(singlefile_path))
contents = None
if readability_txt_path:

View file

@ -1,6 +1,8 @@
import datetime
import io
import json
import pathlib
import zipfile
from typing import List
from uuid import uuid4
@ -101,3 +103,28 @@ def archive_factory(test_session, archive_directory, make_files_structure, image
return archive
return _
@pytest.fixture
def compressed_singlefile_factory():
"""Mimic the file created by `single-file --compress-content`: an HTML prelude containing the
SingleFile comment header, followed by a ZIP containing the uncompressed page as index.html."""
def _(url: str = 'https://example.com', title: str = 'compressed title') -> bytes:
prelude = '<!DOCTYPE html> <html data-sfz><!--\n' \
' Page saved with SingleFile \n' \
f' url: {url} \n' \
' saved date: Thu Jun 11 2026 22:49:39 GMT+0000 (Coordinated Universal Time)\n' \
'--><meta charset=windows-1252><title></title>'
index_html = '<html><!--\n Page saved with SingleFile \n' \
f' url: {url} \n' \
f'-->\n<head><title>{title}</title></head><body>test compressed singlefile</body></html>'
buf = io.BytesIO()
buf.write(prelude.encode())
# Deflate like the real single-file CLI; the page must not be readable as plain text.
with zipfile.ZipFile(buf, 'a', compression=zipfile.ZIP_DEFLATED) as zip_:
zip_.writestr('index.html', index_html)
zip_.writestr('manifest.json', '{}')
return buf.getvalue()
return _

View file

@ -1,11 +1,13 @@
import base64
import gzip
import io
import json
import os
import pathlib
import re
import shlex
import tempfile
import zipfile
from dataclasses import dataclass, field
from datetime import datetime
from json import JSONDecodeError
@ -630,11 +632,12 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P
ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response.
async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]:
async def request_archive(url: str, singlefile: str = None, compress: bool = False) \
-> Tuple[str | bytes, Optional[dict], Optional[str]]:
"""Send a request to the archive service to archive the URL."""
logger.info(f'Sending archive request to archive service: {url}')
data = dict(url=url, singlefile=singlefile)
data = dict(url=url, singlefile=singlefile, compress=compress)
try:
async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response:
status = response.status
@ -662,7 +665,9 @@ async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Option
# Decode and decompress.
singlefile = base64.b64decode(singlefile)
singlefile = gzip.decompress(singlefile)
singlefile = singlefile.decode()
# A compressed (SingleFileZ) singlefile is binary and must stay bytes; only decode regular singlefiles.
if not is_compressed_singlefile(singlefile):
singlefile = singlefile.decode()
if not screenshot:
logger.info(f'Failed to get screenshot for {url=}')
@ -717,20 +722,23 @@ class WrittenArchiveFiles:
paths: List[pathlib.Path]
def write_archive_files(url: str, singlefile: str, readability: Optional[dict],
def write_archive_files(url: str, singlefile: str | bytes, readability: Optional[dict],
screenshot: Optional[bytes],
destination: Optional[pathlib.Path] = None) -> WrittenArchiveFiles:
"""Pure I/O: write the singlefile/readability/screenshot/json files to disk.
Returns the URL and the list of paths to register, leaving DB work to register_archive.
"""
compressed = isinstance(singlefile, bytes) and is_compressed_singlefile(singlefile)
readability = readability.copy() if readability else readability
# First try to get the title from Readability.
title = readability.get('title') if readability else None
if not title and singlefile:
# Try to get the title ourselves from the HTML.
title = get_title_from_html(singlefile, url=url)
# Try to get the title ourselves from the HTML. A compressed singlefile contains the
# uncompressed page as index.html.
title = get_title_from_html(get_compressed_singlefile_index(singlefile) if compressed else singlefile, url=url)
if readability:
# Readability could not find title, lets use ours.
readability['title'] = title
@ -751,8 +759,12 @@ def write_archive_files(url: str, singlefile: str, readability: Optional[dict],
archive_files.readability_txt = archive_files.readability = None
# Store the single-file HTML in its own file.
singlefile = format_html_string(singlefile)
archive_files.singlefile.write_text(singlefile)
if compressed:
# A compressed (SingleFileZ) singlefile is a binary ZIP; prettifying would corrupt it.
archive_files.singlefile.write_bytes(singlefile)
else:
singlefile = format_html_string(singlefile)
archive_files.singlefile.write_text(singlefile)
if screenshot:
archive_files.screenshot.write_bytes(screenshot)
@ -992,10 +1004,41 @@ def link_domain_and_downloads(session: Session):
SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n')
COMPRESSED_SINGLEFILE_INDEX = 'index.html'
def is_compressed_singlefile(singlefile: bytes) -> bool:
"""A compressed (SingleFileZ) singlefile is a self-extracting HTML file which is also a valid ZIP
containing the uncompressed page as index.html."""
buf = io.BytesIO(singlefile)
if not zipfile.is_zipfile(buf):
return False
try:
with zipfile.ZipFile(buf) as zip_:
return COMPRESSED_SINGLEFILE_INDEX in zip_.namelist()
except zipfile.BadZipFile:
return False
def get_compressed_singlefile_index(singlefile: bytes) -> bytes:
"""Extract the uncompressed page (index.html) from a compressed (SingleFileZ) singlefile."""
with zipfile.ZipFile(io.BytesIO(singlefile)) as zip_:
return zip_.read(COMPRESSED_SINGLEFILE_INDEX)
def read_singlefile_html(path: pathlib.Path) -> bytes:
"""Read the page HTML of the singlefile at `path`; extract the uncompressed page when the
singlefile is compressed (SingleFileZ)."""
contents = path.read_bytes()
if is_compressed_singlefile(contents):
return get_compressed_singlefile_index(contents)
return contents
def get_url_from_singlefile(html: bytes | str) -> str:
"""Extract URL from SingleFile contents."""
html = html.decode() if isinstance(html, bytes) else html
# Ignore decode errors; a compressed singlefile has a plain-text prelude but a binary ZIP tail.
html = html.decode(errors='ignore') if isinstance(html, bytes) else html
if SINGLEFILE_HEADER not in html:
raise RuntimeError(f'Not a singlefile!')
@ -1185,9 +1228,10 @@ def is_singlefile_file(path: pathlib.Path) -> bool:
if ARCHIVE_MATCHER.match(path.name):
# A file name matching exactly the Archive format from WROLPi should always be an Archive.
return True
with path.open('rt') as fh:
with path.open('rb') as fh:
# Lastly, try to read the header of this HTML file. Read a large part of the head because URLs can be long.
header = fh.read(1000)
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
header = fh.read(1000).decode(errors='ignore')
if SINGLEFILE_HEADER in header:
return True
return False
@ -1463,24 +1507,30 @@ async def extract_singlefile_artifacts(singlefile: bytes) -> SinglefileArtifacts
url = get_url_from_singlefile(singlefile)
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
# A compressed (SingleFileZ) singlefile contains the uncompressed page as index.html; use that
# for readability and screenshot, but keep the compressed bytes as the singlefile.
compressed = is_compressed_singlefile(singlefile)
extractable = get_compressed_singlefile_index(singlefile) if compressed else singlefile
if DOCKERIZED:
# Perform the archive in the Archive docker container. (Typically in the development environment).
encoded = base64.b64encode(singlefile).decode()
logger.debug(f'extract_singlefile_artifacts sending to archive service: {url}')
singlefile, readability, screenshot = await request_archive(url, singlefile=encoded)
singlefile, readability, screenshot = await request_archive(url, singlefile=encoded, compress=compressed)
singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile
else:
# JSON from readability-extractor
readability = dict()
try:
logger.debug(f'extract_singlefile_artifacts extracting readability: {url}')
readability = await html_to_readability(singlefile, url)
readability = await html_to_readability(extractable, url)
except RuntimeError as e:
logger.error(f'Failed to extract readability from: {url}', exc_info=e)
screenshot = None
try:
logger.debug(f'extract_singlefile_artifacts creating screenshot: {url}')
screenshot = html_screenshot(singlefile)
screenshot = html_screenshot(extractable)
except Exception as e:
logger.error(f'Failed to extract screenshot from: {url}', exc_info=e)
@ -1620,6 +1670,8 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path:
else:
logger.debug(f'Generating screenshot locally for Archive {archive_id}')
singlefile_contents = singlefile_path.read_bytes()
if is_compressed_singlefile(singlefile_contents):
singlefile_contents = get_compressed_singlefile_index(singlefile_contents)
screenshot_bytes = html_screenshot(singlefile_contents)
if not screenshot_bytes:

View file

@ -303,8 +303,9 @@ class Archive(Base, ModelHelper):
# This data has already been read.
return
with path.open('rt') as fh:
head = fh.read(1000)
with path.open('rb') as fh:
# Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail.
head = fh.read(1000).decode(errors='ignore')
if 'Page saved with SingleFile' not in head:
logger.error(f'Could not find SingleFile header in {self}')
return
@ -342,14 +343,16 @@ class Archive(Base, ModelHelper):
def apply_singlefile_title(self):
"""Get the title from the Singlefile, if it's missing."""
# Local import to avoid circular import within archive module
from modules.archive import lib
if self.singlefile_path and not self.file_group.title:
self.file_group.title = get_title_from_html(self.singlefile_path.read_text())
self.file_group.title = get_title_from_html(lib.read_singlefile_html(self.singlefile_path))
def apply_metadata(self):
"""Read and apply <meta> (and more) data from the Singlefile HTML."""
# Local import to avoid circular import within archive module
from modules.archive import lib
contents = self.singlefile_path.read_bytes()
contents = lib.read_singlefile_html(self.singlefile_path)
metadata = lib.parse_article_html_metadata(contents)
if metadata.author:

View file

@ -0,0 +1,187 @@
"""Tests for compressed (SingleFileZ) singlefile Archives.
A compressed singlefile is a self-extracting HTML file which is also a valid ZIP containing the
uncompressed page as index.html. Only the singlefile .html is compressed; readability files are
always written uncompressed.
"""
import pathlib
import zipfile
import pytest
import modules.archive
from modules.archive import archive_downloader, PreparedArchive, model_archive
from modules.archive.conftest import make_test_ctx
from modules.archive.lib import (
is_compressed_singlefile,
get_compressed_singlefile_index,
get_url_from_singlefile,
is_singlefile_file,
write_archive_files,
)
from modules.archive.test.test_lib import make_fake_archive_result
from wrolpi.downloader import Download
from wrolpi.files.lib import get_mimetype, _mimetype_suffix_map
from wrolpi.files.models import FileGroup
from wrolpi.schema import DownloadRequest
def test_is_compressed_singlefile(compressed_singlefile_factory, singlefile_contents_factory):
compressed = compressed_singlefile_factory()
assert is_compressed_singlefile(compressed) is True
# A regular singlefile is not compressed.
assert is_compressed_singlefile(singlefile_contents_factory().encode()) is False
# Garbage is not compressed.
assert is_compressed_singlefile(b'not a zip') is False
# The uncompressed page can be extracted.
index = get_compressed_singlefile_index(compressed)
assert b'test compressed singlefile' in index
assert b'<title>compressed title</title>' in index
def test_get_url_from_compressed_singlefile(compressed_singlefile_factory):
"""The URL can be extracted from the plain-text prelude despite the binary ZIP tail."""
compressed = compressed_singlefile_factory(url='https://wrolpi.org/some-article')
assert get_url_from_singlefile(compressed) == 'https://wrolpi.org/some-article'
@pytest.mark.parametrize(
'name,expected', [
('2000-01-01-00-00-00_Some NA.html', True), # Matches the WROLPi Archive file name.
('foo.html', True), # Detected by the SingleFile header in the prelude.
('foo.txt', False),
]
)
def test_is_singlefile_file_compressed(name, expected, make_files_structure, compressed_singlefile_factory):
"""A compressed singlefile is recognized even though it has a binary ZIP tail."""
path, = make_files_structure([name])
path.write_bytes(compressed_singlefile_factory())
assert is_singlefile_file(path) == expected
def test_compressed_singlefile_mimetype(make_files_structure, compressed_singlefile_factory):
"""A compressed singlefile must be `text/html` so the archive modeler finds it."""
# `magic` reports application/octet-stream for the real (binary) compressed singlefile.
assert _mimetype_suffix_map(pathlib.Path('foo.html'), 'application/octet-stream') == 'text/html'
path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
path.write_bytes(compressed_singlefile_factory())
assert get_mimetype(path) == 'text/html'
def test_write_archive_files_compressed(test_directory, test_session, image_bytes_factory,
compressed_singlefile_factory):
"""The compressed singlefile is written verbatim (prettifying would corrupt the ZIP);
readability files are written uncompressed."""
compressed = compressed_singlefile_factory()
readability = dict(content='<html>readability content</html>', textContent='readability text', title='a title')
destination = test_directory / 'archive/example.com'
destination.mkdir(parents=True)
written = write_archive_files('https://example.com', compressed, readability, image_bytes_factory(),
destination=destination)
singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)
# The file on disk is unchanged, and still a valid ZIP.
assert singlefile_path.read_bytes() == compressed
assert zipfile.is_zipfile(singlefile_path)
# Readability files are plain text, not compressed.
readability_path = next(i for i in written.paths if i.name.endswith('.readability.html'))
assert 'readability content' in readability_path.read_text()
readability_txt_path = next(i for i in written.paths if i.name.endswith('.readability.txt'))
assert 'readability text' in readability_txt_path.read_text()
def test_write_archive_files_compressed_title_fallback(test_directory, test_session, compressed_singlefile_factory):
"""Without a readability title, the title comes from the ZIP's index.html."""
compressed = compressed_singlefile_factory(title='the fallback title')
destination = test_directory / 'archive/example.com'
destination.mkdir(parents=True)
written = write_archive_files('https://example.com', compressed, None, None, destination=destination)
singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)
assert 'the fallback title' in singlefile_path.name
assert singlefile_path.read_bytes() == compressed
@pytest.mark.asyncio
async def test_execute_download_compressed(test_session, test_directory, monkeypatch, compressed_singlefile_factory):
"""The compress_singlefile setting is forwarded to the archive service, and the compressed
bytes returned are written to disk unchanged."""
compressed = compressed_singlefile_factory()
_, readability, screenshot = make_fake_archive_result()
request_archive_kwargs = {}
async def mock_request_archive(url, compress=False):
request_archive_kwargs['compress'] = compress
return compressed, readability, screenshot
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
destination = test_directory / 'archive/example.com'
destination.mkdir(parents=True)
prepared = PreparedArchive(
url='https://example.com',
destination=destination,
settings={'compress_singlefile': True},
)
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
assert request_archive_kwargs['compress'] is True
singlefile_path = next(i for i in executed.written.paths if i.suffix == '.html' and '.readability' not in i.name)
assert singlefile_path.read_bytes() == compressed
@pytest.mark.asyncio
async def test_model_archive_compressed(async_client, test_session, test_directory, make_files_structure,
compressed_singlefile_factory):
"""A compressed singlefile on disk (without readability files) can be modeled as an Archive;
the title and URL come from the compressed file itself.
async_client is required because get_or_create_domain_collection activates a Sanic switch."""
path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
path.write_bytes(compressed_singlefile_factory(url='https://example.com/article', title='compressed title'))
file_group = FileGroup.from_paths(test_session, path)
test_session.flush()
archive = model_archive(test_session, file_group)
test_session.commit()
assert archive.file_group.url == 'https://example.com/article'
assert archive.file_group.title == 'compressed title'
assert archive.collection.name == 'example.com'
def test_download_request_compress_singlefile():
"""compress_singlefile=True survives DownloadRequest settings validation."""
request = DownloadRequest(
urls=['https://example.com'],
downloader='archive',
settings=dict(compress_singlefile=True),
)
assert request.settings['compress_singlefile'] is True
def test_rss_forwards_compress_singlefile(test_session):
"""RSSDownloader passes compress_singlefile on to the child archive downloads."""
from wrolpi.downloader import rss_downloader, ExecutedRSS
download = Download(
url='https://example.com/feed',
downloader='rss',
sub_downloader='archive',
settings={'compress_singlefile': True},
)
executed = ExecutedRSS(yt_channel_id=None, candidate_urls=['https://example.com/article'])
result = rss_downloader.finalize_download(test_session, download, executed)
assert result.success is True
assert result.downloads == ['https://example.com/article']
assert result.settings['compress_singlefile'] is True

View file

@ -1161,7 +1161,7 @@ async def test_archive_download_uses_domain_collection_directory(async_client, t
# Mock request_archive to return fake archive data
singlefile, readability, screenshot = make_fake_archive_result()
async def mock_request_archive(url):
async def mock_request_archive(url, compress=False):
return singlefile, readability, screenshot
# Patch in both lib and archive module (imported at module level)
@ -1223,7 +1223,7 @@ async def test_archive_download_explicit_destination_overrides_domain_collection
# Mock request_archive to return fake archive data
singlefile, readability, screenshot = make_fake_archive_result()
async def mock_request_archive(url):
async def mock_request_archive(url, compress=False):
return singlefile, readability, screenshot
# Patch in both lib and archive module (imported at module level)

View file

@ -2230,6 +2230,9 @@ class RSSDownloader(Downloader):
for key in CHANNEL_INHERITABLE_SETTINGS:
if key in settings:
next_download_settings[key] = settings[key]
# Pass compression on to child archive downloads.
if 'compress_singlefile' in settings:
next_download_settings['compress_singlefile'] = settings['compress_singlefile']
# Use download.destination column (settings['destination'] is legacy)
if download.destination:
next_download_settings['destination'] = str(download.destination)

View file

@ -384,6 +384,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
return 'model/3mf'
if suffix.endswith('.mp4'):
return 'video/mp4'
if suffix.endswith('.html') or suffix.endswith('.htm'):
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
return 'text/html'
if suffix.endswith('.hgt'):
return 'application/octet-stream'
if mimetype == 'text/plain':
@ -414,6 +417,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str):
if mimetype == 'application/x-subrip':
# Fallback to old mimetype.
return 'text/srt'
if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'):
# Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail.
return 'text/html'
if mimetype == 'application/zip' and suffix == '.docx':
return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document'
if mimetype == 'application/zip' and suffix == '.cbz':

View file

@ -260,6 +260,7 @@ class DownloadSettings:
video_resolutions: List[str] = field(default_factory=list)
parent_download_url: Optional[str] = None
skip_already_downloaded: Optional[bool] = None
compress_singlefile: Optional[bool] = None
# Inheritable video settings (Optional so they're only set when user overrides)
writesubtitles: Optional[bool] = None
writeautomaticsub: Optional[bool] = None