From 0a776d030f798e231e03e4872c6543aff854f125 Mon Sep 17 00:00:00 2001 From: lrnselfreliance <33263708+lrnselfreliance@users.noreply.github.com> Date: Thu, 11 Jun 2026 17:54:22 -0600 Subject: [PATCH] Archives can be compressed (SingleFileZ self-extracting HTML) (#477) * Archives can be compressed (SingleFileZ self-extracting HTML). A new compress_singlefile download setting runs single-file with --compress-content, creating a self-extracting HTML file which is also a valid ZIP. Only the singlefile .html is compressed; readability files are always written uncompressed. Readability, screenshots, titles, and URLs are extracted from the index.html inside the ZIP. A Compress checkbox in the Archive and RSS download forms remembers the user's choice in local storage. Co-authored-by: Claude Fable 5 --- app/src/components/Download.js | 61 +++++- docker/archive/main.py | 60 ++++-- modules/archive/__init__.py | 10 +- modules/archive/conftest.py | 27 +++ modules/archive/lib.py | 80 ++++++-- modules/archive/models.py | 11 +- .../test/test_compressed_singlefile.py | 187 ++++++++++++++++++ modules/archive/test/test_lib.py | 4 +- wrolpi/downloader.py | 3 + wrolpi/files/lib.py | 6 + wrolpi/schema.py | 1 + 11 files changed, 409 insertions(+), 41 deletions(-) create mode 100644 modules/archive/test/test_compressed_singlefile.py diff --git a/app/src/components/Download.js b/app/src/components/Download.js index 04b74679..a7af9306 100644 --- a/app/src/components/Download.js +++ b/app/src/components/Download.js @@ -504,6 +504,22 @@ function AdvancedVideoSettings({form, isVideoLevel = false, isConfigLoaded = tru } +export function CompressSinglefileToggle({form}) { + return <> + + + +} + function AdvancedDownloadSettings({form}) { const [active, setActive] = React.useState(false); @@ -1096,6 +1112,9 @@ export function ArchiveDownloadForm({ }) { const [showMessage, setShowMessage] = React.useState(false); + // Remember the user's compression choice between downloads. + const [defaultCompressSinglefile, setDefaultCompressSinglefile] = useLocalStorage('compress_singlefile', false); + const submitter = propSubmitter || (async (formData) => { const downloadData = { downloader: formData.downloader, @@ -1112,6 +1131,7 @@ export function ArchiveDownloadForm({ tag_names: [], settings: { skip_already_downloaded: false, + compress_singlefile: defaultCompressSinglefile, }, }; @@ -1130,6 +1150,14 @@ export function ArchiveDownloadForm({ onSuccess, }); + // Persist the user's compression choice in local storage. + React.useEffect(() => { + const {compress_singlefile} = form.formData.settings; + if (compress_singlefile !== undefined && compress_singlefile !== defaultCompressSinglefile) { + setDefaultCompressSinglefile(compress_singlefile); + } + }, [form.formData, defaultCompressSinglefile]); + const localOnCancel = (e) => { if (e) e.preventDefault(); if (onCancel) { @@ -1156,6 +1184,11 @@ export function ArchiveDownloadForm({ + + + + + @@ -1179,6 +1212,8 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action const [userChangedResolutions, setUserChangedResolutions] = React.useState(false); const [defaultVideoFormat, setDefaultVideoFormat] = useLocalStorage('video_format', defaultVideoFormatOption); + // Remember the user's compression choice between downloads. + const [defaultCompressSinglefile, setDefaultCompressSinglefile] = useLocalStorage('compress_singlefile', false); submitter = submitter || (async (formData) => { const downloadData = { @@ -1202,6 +1237,7 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action frequency: weeklyOption.value, sub_downloader: null, settings: { + compress_singlefile: defaultCompressSinglefile, excluded_urls: null, title_exclude: null, title_include: null, @@ -1272,6 +1308,14 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action } }, [form.formData]); + // Persist the user's compression choice in local storage. + React.useEffect(() => { + const {compress_singlefile} = form.formData.settings; + if (compress_singlefile !== undefined && compress_singlefile !== defaultCompressSinglefile) { + setDefaultCompressSinglefile(compress_singlefile); + } + }, [form.formData, defaultCompressSinglefile]); + // Default to "new" download buttons. actions = actions || DownloadFormButtons; const actionsElm = actions({onDelete, onCancel, form}); @@ -1333,11 +1377,18 @@ export function RSSDownloadForm({download, submitter, onDelete, onCancel, action ; } else if (form.formData.sub_downloader === Downloaders.Archive) { - downloaderRows = - - - - ; + downloaderRows = <> + + + + + + + + + + + ; } return
diff --git a/docker/archive/main.py b/docker/archive/main.py index 0f80896e..a1eed6a7 100644 --- a/docker/archive/main.py +++ b/docker/archive/main.py @@ -5,6 +5,7 @@ This file is a simple Python/Sanic wrapper around the single-file CLI command. import asyncio import base64 import gzip +import io import json import logging import os.path @@ -13,6 +14,7 @@ import subprocess import sys import tempfile import traceback +import zipfile from json import JSONDecodeError from sanic import Sanic, response @@ -83,15 +85,17 @@ async def check_output(cmd, always_log_stderr: bool = False, cwd=None, timeout: raise TimeoutError(f"Command '{cmd}' timed out after {timeout} seconds") -async def call_single_file(url) -> bytes: +async def call_single_file(url, compress: bool = False) -> bytes: """ Call the CLI command for SingleFile. See https://github.com/gildas-lormeau/SingleFile """ logger.info(f'archiving {url}') + # --compress-content creates a compressed (SingleFileZ) self-extracting HTML file. cmd = f'{SINGLEFILE_PATH}' \ r' --dump-content ' \ + f'{" --compress-content " if compress else ""}' \ f' "{url}"' logger.debug(f'archive cmd: {cmd}') stdout, stderr, return_code = await check_output(cmd, always_log_stderr=True, timeout=3 * 60) @@ -152,6 +156,27 @@ async def take_screenshot(url: str) -> bytes: return png +def extract_compressed_singlefile(singlefile: bytes, tmp_dir: str) -> str: + """A compressed (SingleFileZ) singlefile is also a valid ZIP containing the uncompressed page as + index.html (plus its resources). Extract it so readability/screenshot can use the real HTML. + + Returns the path to the extracted index.html, or an empty string if the singlefile is not compressed. + """ + buf = io.BytesIO(singlefile) + if not zipfile.is_zipfile(buf): + return '' + try: + with zipfile.ZipFile(buf) as zip_: + if 'index.html' not in zip_.namelist(): + return '' + extracted = pathlib.Path(tmp_dir) / 'extracted' + extracted.mkdir(exist_ok=True) + zip_.extractall(extracted) + return str(extracted / 'index.html') + except zipfile.BadZipFile: + return '' + + def prepare_bytes(b: bytes) -> str: """ Compress and encode bytes for smaller response. @@ -181,16 +206,19 @@ async def post_screenshot(request: Request): # Write singlefile to temp file and screenshot it # Use html suffix so chrome screenshot recognizes it as an HTML file - with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh: - fh.write(singlefile) - fh.flush() + with tempfile.TemporaryDirectory() as tmp_dir: + singlefile_path = pathlib.Path(tmp_dir) / 'singlefile.html' + singlefile_path.write_bytes(singlefile) + + # A compressed (SingleFileZ) singlefile must be extracted before it can be screenshot. + extractable = extract_compressed_singlefile(singlefile, tmp_dir) or str(singlefile_path) screenshot = None try: # Screenshot the local singlefile - screenshot = await take_screenshot(f'file://{fh.name}') + screenshot = await take_screenshot(f'file://{extractable}') except Exception as e: - logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e) + logger.error(f'Failed to take screenshot of {extractable}', exc_info=e) # Fall back to URL if local screenshot failed if not screenshot: @@ -225,17 +253,23 @@ async def post_archive(request: Request): # May have been passed the singlefile contents, do the extractions and return. singlefile = request.json.get('singlefile') singlefile = base64.b64decode(singlefile) if singlefile else None + compress = bool(request.json.get('compress')) # Use provided singlefile, or fetch and create from the internet. - singlefile = singlefile or await call_single_file(url) + singlefile = singlefile or await call_single_file(url, compress=compress) # Use html suffix so chrome screenshot recognizes it as an HTML file. - with tempfile.NamedTemporaryFile('wb', suffix='.html') as fh: - fh.write(singlefile) - fh.flush() + with tempfile.TemporaryDirectory() as tmp_dir: + singlefile_path = pathlib.Path(tmp_dir) / 'singlefile.html' + singlefile_path.write_bytes(singlefile) + + # A compressed (SingleFileZ) singlefile contains the uncompressed page; readability and + # screenshot must use the extracted index.html. + extractable = extract_compressed_singlefile(singlefile, tmp_dir) or str(singlefile_path) + readability = None try: - readability = await extract_readability(fh.name, url) + readability = await extract_readability(extractable, url) except Exception as e: # Readability had error, but its is not required. logger.error(f'Failed to get readability', exc_info=e) @@ -243,9 +277,9 @@ async def post_archive(request: Request): screenshot = None try: # Screenshot the local singlefile, if that fails try the URL. - screenshot = await take_screenshot(f'file://{fh.name}') + screenshot = await take_screenshot(f'file://{extractable}') except Exception as e: - logger.error(f'Failed to take screenshot of {fh.name}', exc_info=e) + logger.error(f'Failed to take screenshot of {extractable}', exc_info=e) if not screenshot: logger.warning(f'Failed to screenshot local singlefile attempting to screenshot: {url}') diff --git a/modules/archive/__init__.py b/modules/archive/__init__.py index 1554c386..299f70b7 100644 --- a/modules/archive/__init__.py +++ b/modules/archive/__init__.py @@ -80,8 +80,9 @@ class ArchiveDownloader(Downloader, ABC): download: Download = None) -> ExecutedArchive: """Run the archive container or local singlefile + readability/screenshot extraction, then write the artifacts to disk. No DB.""" + compress = bool(prepared.settings.get('compress_singlefile')) if DOCKERIZED or PYTEST: - singlefile, readability, screenshot = await request_archive(prepared.url) + singlefile, readability, screenshot = await request_archive(prepared.url, compress=compress) artifacts = SinglefileArtifacts(url=prepared.url, singlefile=singlefile, readability=readability, screenshot=screenshot) else: @@ -143,6 +144,8 @@ class ArchiveDownloader(Downloader, ABC): '--user-agent', user_agent, '--dump-content', '--load-deferred-images-dispatch-scroll-event', + # Create a compressed (SingleFileZ) self-extracting HTML file. + *(('--compress-content',) if prepared.settings.get('compress_singlefile') else ()), prepared.url, ) cwd = pathlib.Path('/home/wrolpi') @@ -153,7 +156,8 @@ class ArchiveDownloader(Downloader, ABC): cmd, cwd, ctx=ctx) stderr = result.stderr.decode() - log_output = stderr or result.stdout.decode() or 'No stderr or stdout!' + # stdout may be a binary ZIP when --compress-content is used. + log_output = stderr or result.stdout.decode(errors='replace') or 'No stderr or stdout!' if result.return_code != 0: e = ChildProcessError(log_output[:1000]) @@ -244,7 +248,7 @@ def model_archive(session: Session, file_group: FileGroup) -> Archive: if readability_json_path: title = get_title(readability_json_path) if not title: - title = get_title_from_html(singlefile_path.read_text()) + title = get_title_from_html(lib.read_singlefile_html(singlefile_path)) contents = None if readability_txt_path: diff --git a/modules/archive/conftest.py b/modules/archive/conftest.py index 73ba5f78..97481a20 100644 --- a/modules/archive/conftest.py +++ b/modules/archive/conftest.py @@ -1,6 +1,8 @@ import datetime +import io import json import pathlib +import zipfile from typing import List from uuid import uuid4 @@ -101,3 +103,28 @@ def archive_factory(test_session, archive_directory, make_files_structure, image return archive return _ + + +@pytest.fixture +def compressed_singlefile_factory(): + """Mimic the file created by `single-file --compress-content`: an HTML prelude containing the + SingleFile comment header, followed by a ZIP containing the uncompressed page as index.html.""" + + def _(url: str = 'https://example.com', title: str = 'compressed title') -> bytes: + prelude = ' ' + index_html = '\n{title}test compressed singlefile' + buf = io.BytesIO() + buf.write(prelude.encode()) + # Deflate like the real single-file CLI; the page must not be readable as plain text. + with zipfile.ZipFile(buf, 'a', compression=zipfile.ZIP_DEFLATED) as zip_: + zip_.writestr('index.html', index_html) + zip_.writestr('manifest.json', '{}') + return buf.getvalue() + + return _ diff --git a/modules/archive/lib.py b/modules/archive/lib.py index 526d3af3..92091c65 100644 --- a/modules/archive/lib.py +++ b/modules/archive/lib.py @@ -1,11 +1,13 @@ import base64 import gzip +import io import json import os import pathlib import re import shlex import tempfile +import zipfile from dataclasses import dataclass, field from datetime import datetime from json import JSONDecodeError @@ -630,11 +632,12 @@ def get_new_archive_files(url: str, title: Optional[str], destination: pathlib.P ARCHIVE_TIMEOUT = Seconds.minute * 10 # Wait at most 10 minutes for response. -async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Optional[dict], Optional[str]]: +async def request_archive(url: str, singlefile: str = None, compress: bool = False) \ + -> Tuple[str | bytes, Optional[dict], Optional[str]]: """Send a request to the archive service to archive the URL.""" logger.info(f'Sending archive request to archive service: {url}') - data = dict(url=url, singlefile=singlefile) + data = dict(url=url, singlefile=singlefile, compress=compress) try: async with aiohttp_post(f'{ARCHIVE_SERVICE}/json', json_=data, timeout=ARCHIVE_TIMEOUT) as response: status = response.status @@ -662,7 +665,9 @@ async def request_archive(url: str, singlefile: str = None) -> Tuple[str, Option # Decode and decompress. singlefile = base64.b64decode(singlefile) singlefile = gzip.decompress(singlefile) - singlefile = singlefile.decode() + # A compressed (SingleFileZ) singlefile is binary and must stay bytes; only decode regular singlefiles. + if not is_compressed_singlefile(singlefile): + singlefile = singlefile.decode() if not screenshot: logger.info(f'Failed to get screenshot for {url=}') @@ -717,20 +722,23 @@ class WrittenArchiveFiles: paths: List[pathlib.Path] -def write_archive_files(url: str, singlefile: str, readability: Optional[dict], +def write_archive_files(url: str, singlefile: str | bytes, readability: Optional[dict], screenshot: Optional[bytes], destination: Optional[pathlib.Path] = None) -> WrittenArchiveFiles: """Pure I/O: write the singlefile/readability/screenshot/json files to disk. Returns the URL and the list of paths to register, leaving DB work to register_archive. """ + compressed = isinstance(singlefile, bytes) and is_compressed_singlefile(singlefile) + readability = readability.copy() if readability else readability # First try to get the title from Readability. title = readability.get('title') if readability else None if not title and singlefile: - # Try to get the title ourselves from the HTML. - title = get_title_from_html(singlefile, url=url) + # Try to get the title ourselves from the HTML. A compressed singlefile contains the + # uncompressed page as index.html. + title = get_title_from_html(get_compressed_singlefile_index(singlefile) if compressed else singlefile, url=url) if readability: # Readability could not find title, lets use ours. readability['title'] = title @@ -751,8 +759,12 @@ def write_archive_files(url: str, singlefile: str, readability: Optional[dict], archive_files.readability_txt = archive_files.readability = None # Store the single-file HTML in its own file. - singlefile = format_html_string(singlefile) - archive_files.singlefile.write_text(singlefile) + if compressed: + # A compressed (SingleFileZ) singlefile is a binary ZIP; prettifying would corrupt it. + archive_files.singlefile.write_bytes(singlefile) + else: + singlefile = format_html_string(singlefile) + archive_files.singlefile.write_text(singlefile) if screenshot: archive_files.screenshot.write_bytes(screenshot) @@ -992,10 +1004,41 @@ def link_domain_and_downloads(session: Session): SINGLEFILE_URL_EXTRACTOR = re.compile(r'Page saved with SingleFile \s+url: (http.+?)\n') +COMPRESSED_SINGLEFILE_INDEX = 'index.html' + + +def is_compressed_singlefile(singlefile: bytes) -> bool: + """A compressed (SingleFileZ) singlefile is a self-extracting HTML file which is also a valid ZIP + containing the uncompressed page as index.html.""" + buf = io.BytesIO(singlefile) + if not zipfile.is_zipfile(buf): + return False + try: + with zipfile.ZipFile(buf) as zip_: + return COMPRESSED_SINGLEFILE_INDEX in zip_.namelist() + except zipfile.BadZipFile: + return False + + +def get_compressed_singlefile_index(singlefile: bytes) -> bytes: + """Extract the uncompressed page (index.html) from a compressed (SingleFileZ) singlefile.""" + with zipfile.ZipFile(io.BytesIO(singlefile)) as zip_: + return zip_.read(COMPRESSED_SINGLEFILE_INDEX) + + +def read_singlefile_html(path: pathlib.Path) -> bytes: + """Read the page HTML of the singlefile at `path`; extract the uncompressed page when the + singlefile is compressed (SingleFileZ).""" + contents = path.read_bytes() + if is_compressed_singlefile(contents): + return get_compressed_singlefile_index(contents) + return contents + def get_url_from_singlefile(html: bytes | str) -> str: """Extract URL from SingleFile contents.""" - html = html.decode() if isinstance(html, bytes) else html + # Ignore decode errors; a compressed singlefile has a plain-text prelude but a binary ZIP tail. + html = html.decode(errors='ignore') if isinstance(html, bytes) else html if SINGLEFILE_HEADER not in html: raise RuntimeError(f'Not a singlefile!') @@ -1185,9 +1228,10 @@ def is_singlefile_file(path: pathlib.Path) -> bool: if ARCHIVE_MATCHER.match(path.name): # A file name matching exactly the Archive format from WROLPi should always be an Archive. return True - with path.open('rt') as fh: + with path.open('rb') as fh: # Lastly, try to read the header of this HTML file. Read a large part of the head because URLs can be long. - header = fh.read(1000) + # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail. + header = fh.read(1000).decode(errors='ignore') if SINGLEFILE_HEADER in header: return True return False @@ -1463,24 +1507,30 @@ async def extract_singlefile_artifacts(singlefile: bytes) -> SinglefileArtifacts url = get_url_from_singlefile(singlefile) singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile + # A compressed (SingleFileZ) singlefile contains the uncompressed page as index.html; use that + # for readability and screenshot, but keep the compressed bytes as the singlefile. + compressed = is_compressed_singlefile(singlefile) + extractable = get_compressed_singlefile_index(singlefile) if compressed else singlefile + if DOCKERIZED: # Perform the archive in the Archive docker container. (Typically in the development environment). encoded = base64.b64encode(singlefile).decode() logger.debug(f'extract_singlefile_artifacts sending to archive service: {url}') - singlefile, readability, screenshot = await request_archive(url, singlefile=encoded) + singlefile, readability, screenshot = await request_archive(url, singlefile=encoded, compress=compressed) + singlefile = singlefile.encode() if isinstance(singlefile, str) else singlefile else: # JSON from readability-extractor readability = dict() try: logger.debug(f'extract_singlefile_artifacts extracting readability: {url}') - readability = await html_to_readability(singlefile, url) + readability = await html_to_readability(extractable, url) except RuntimeError as e: logger.error(f'Failed to extract readability from: {url}', exc_info=e) screenshot = None try: logger.debug(f'extract_singlefile_artifacts creating screenshot: {url}') - screenshot = html_screenshot(singlefile) + screenshot = html_screenshot(extractable) except Exception as e: logger.error(f'Failed to extract screenshot from: {url}', exc_info=e) @@ -1620,6 +1670,8 @@ async def generate_archive_screenshot(archive_id: int) -> pathlib.Path: else: logger.debug(f'Generating screenshot locally for Archive {archive_id}') singlefile_contents = singlefile_path.read_bytes() + if is_compressed_singlefile(singlefile_contents): + singlefile_contents = get_compressed_singlefile_index(singlefile_contents) screenshot_bytes = html_screenshot(singlefile_contents) if not screenshot_bytes: diff --git a/modules/archive/models.py b/modules/archive/models.py index ef25b356..068a9902 100644 --- a/modules/archive/models.py +++ b/modules/archive/models.py @@ -303,8 +303,9 @@ class Archive(Base, ModelHelper): # This data has already been read. return - with path.open('rt') as fh: - head = fh.read(1000) + with path.open('rb') as fh: + # Read bytes because a compressed (SingleFileZ) singlefile has a binary ZIP tail. + head = fh.read(1000).decode(errors='ignore') if 'Page saved with SingleFile' not in head: logger.error(f'Could not find SingleFile header in {self}') return @@ -342,14 +343,16 @@ class Archive(Base, ModelHelper): def apply_singlefile_title(self): """Get the title from the Singlefile, if it's missing.""" + # Local import to avoid circular import within archive module + from modules.archive import lib if self.singlefile_path and not self.file_group.title: - self.file_group.title = get_title_from_html(self.singlefile_path.read_text()) + self.file_group.title = get_title_from_html(lib.read_singlefile_html(self.singlefile_path)) def apply_metadata(self): """Read and apply (and more) data from the Singlefile HTML.""" # Local import to avoid circular import within archive module from modules.archive import lib - contents = self.singlefile_path.read_bytes() + contents = lib.read_singlefile_html(self.singlefile_path) metadata = lib.parse_article_html_metadata(contents) if metadata.author: diff --git a/modules/archive/test/test_compressed_singlefile.py b/modules/archive/test/test_compressed_singlefile.py new file mode 100644 index 00000000..5eb1fcbc --- /dev/null +++ b/modules/archive/test/test_compressed_singlefile.py @@ -0,0 +1,187 @@ +"""Tests for compressed (SingleFileZ) singlefile Archives. + +A compressed singlefile is a self-extracting HTML file which is also a valid ZIP containing the +uncompressed page as index.html. Only the singlefile .html is compressed; readability files are +always written uncompressed. +""" +import pathlib +import zipfile + +import pytest + +import modules.archive +from modules.archive import archive_downloader, PreparedArchive, model_archive +from modules.archive.conftest import make_test_ctx +from modules.archive.lib import ( + is_compressed_singlefile, + get_compressed_singlefile_index, + get_url_from_singlefile, + is_singlefile_file, + write_archive_files, +) +from modules.archive.test.test_lib import make_fake_archive_result +from wrolpi.downloader import Download +from wrolpi.files.lib import get_mimetype, _mimetype_suffix_map +from wrolpi.files.models import FileGroup +from wrolpi.schema import DownloadRequest + + +def test_is_compressed_singlefile(compressed_singlefile_factory, singlefile_contents_factory): + compressed = compressed_singlefile_factory() + assert is_compressed_singlefile(compressed) is True + + # A regular singlefile is not compressed. + assert is_compressed_singlefile(singlefile_contents_factory().encode()) is False + # Garbage is not compressed. + assert is_compressed_singlefile(b'not a zip') is False + + # The uncompressed page can be extracted. + index = get_compressed_singlefile_index(compressed) + assert b'test compressed singlefile' in index + assert b'compressed title' in index + + +def test_get_url_from_compressed_singlefile(compressed_singlefile_factory): + """The URL can be extracted from the plain-text prelude despite the binary ZIP tail.""" + compressed = compressed_singlefile_factory(url='https://wrolpi.org/some-article') + assert get_url_from_singlefile(compressed) == 'https://wrolpi.org/some-article' + + +@pytest.mark.parametrize( + 'name,expected', [ + ('2000-01-01-00-00-00_Some NA.html', True), # Matches the WROLPi Archive file name. + ('foo.html', True), # Detected by the SingleFile header in the prelude. + ('foo.txt', False), + ] +) +def test_is_singlefile_file_compressed(name, expected, make_files_structure, compressed_singlefile_factory): + """A compressed singlefile is recognized even though it has a binary ZIP tail.""" + path, = make_files_structure([name]) + path.write_bytes(compressed_singlefile_factory()) + assert is_singlefile_file(path) == expected + + +def test_compressed_singlefile_mimetype(make_files_structure, compressed_singlefile_factory): + """A compressed singlefile must be `text/html` so the archive modeler finds it.""" + # `magic` reports application/octet-stream for the real (binary) compressed singlefile. + assert _mimetype_suffix_map(pathlib.Path('foo.html'), 'application/octet-stream') == 'text/html' + + path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html']) + path.write_bytes(compressed_singlefile_factory()) + assert get_mimetype(path) == 'text/html' + + +def test_write_archive_files_compressed(test_directory, test_session, image_bytes_factory, + compressed_singlefile_factory): + """The compressed singlefile is written verbatim (prettifying would corrupt the ZIP); + readability files are written uncompressed.""" + compressed = compressed_singlefile_factory() + readability = dict(content='readability content', textContent='readability text', title='a title') + destination = test_directory / 'archive/example.com' + destination.mkdir(parents=True) + + written = write_archive_files('https://example.com', compressed, readability, image_bytes_factory(), + destination=destination) + + singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name) + # The file on disk is unchanged, and still a valid ZIP. + assert singlefile_path.read_bytes() == compressed + assert zipfile.is_zipfile(singlefile_path) + + # Readability files are plain text, not compressed. + readability_path = next(i for i in written.paths if i.name.endswith('.readability.html')) + assert 'readability content' in readability_path.read_text() + readability_txt_path = next(i for i in written.paths if i.name.endswith('.readability.txt')) + assert 'readability text' in readability_txt_path.read_text() + + +def test_write_archive_files_compressed_title_fallback(test_directory, test_session, compressed_singlefile_factory): + """Without a readability title, the title comes from the ZIP's index.html.""" + compressed = compressed_singlefile_factory(title='the fallback title') + destination = test_directory / 'archive/example.com' + destination.mkdir(parents=True) + + written = write_archive_files('https://example.com', compressed, None, None, destination=destination) + + singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name) + assert 'the fallback title' in singlefile_path.name + assert singlefile_path.read_bytes() == compressed + + +@pytest.mark.asyncio +async def test_execute_download_compressed(test_session, test_directory, monkeypatch, compressed_singlefile_factory): + """The compress_singlefile setting is forwarded to the archive service, and the compressed + bytes returned are written to disk unchanged.""" + compressed = compressed_singlefile_factory() + _, readability, screenshot = make_fake_archive_result() + request_archive_kwargs = {} + + async def mock_request_archive(url, compress=False): + request_archive_kwargs['compress'] = compress + return compressed, readability, screenshot + + monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive) + + destination = test_directory / 'archive/example.com' + destination.mkdir(parents=True) + prepared = PreparedArchive( + url='https://example.com', + destination=destination, + settings={'compress_singlefile': True}, + ) + + executed = await archive_downloader.execute_download(prepared, make_test_ctx()) + + assert request_archive_kwargs['compress'] is True + singlefile_path = next(i for i in executed.written.paths if i.suffix == '.html' and '.readability' not in i.name) + assert singlefile_path.read_bytes() == compressed + + +@pytest.mark.asyncio +async def test_model_archive_compressed(async_client, test_session, test_directory, make_files_structure, + compressed_singlefile_factory): + """A compressed singlefile on disk (without readability files) can be modeled as an Archive; + the title and URL come from the compressed file itself. + + async_client is required because get_or_create_domain_collection activates a Sanic switch.""" + path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html']) + path.write_bytes(compressed_singlefile_factory(url='https://example.com/article', title='compressed title')) + + file_group = FileGroup.from_paths(test_session, path) + test_session.flush() + + archive = model_archive(test_session, file_group) + test_session.commit() + + assert archive.file_group.url == 'https://example.com/article' + assert archive.file_group.title == 'compressed title' + assert archive.collection.name == 'example.com' + + +def test_download_request_compress_singlefile(): + """compress_singlefile=True survives DownloadRequest settings validation.""" + request = DownloadRequest( + urls=['https://example.com'], + downloader='archive', + settings=dict(compress_singlefile=True), + ) + assert request.settings['compress_singlefile'] is True + + +def test_rss_forwards_compress_singlefile(test_session): + """RSSDownloader passes compress_singlefile on to the child archive downloads.""" + from wrolpi.downloader import rss_downloader, ExecutedRSS + + download = Download( + url='https://example.com/feed', + downloader='rss', + sub_downloader='archive', + settings={'compress_singlefile': True}, + ) + executed = ExecutedRSS(yt_channel_id=None, candidate_urls=['https://example.com/article']) + + result = rss_downloader.finalize_download(test_session, download, executed) + + assert result.success is True + assert result.downloads == ['https://example.com/article'] + assert result.settings['compress_singlefile'] is True diff --git a/modules/archive/test/test_lib.py b/modules/archive/test/test_lib.py index 46537a00..5b99e37b 100644 --- a/modules/archive/test/test_lib.py +++ b/modules/archive/test/test_lib.py @@ -1161,7 +1161,7 @@ async def test_archive_download_uses_domain_collection_directory(async_client, t # Mock request_archive to return fake archive data singlefile, readability, screenshot = make_fake_archive_result() - async def mock_request_archive(url): + async def mock_request_archive(url, compress=False): return singlefile, readability, screenshot # Patch in both lib and archive module (imported at module level) @@ -1223,7 +1223,7 @@ async def test_archive_download_explicit_destination_overrides_domain_collection # Mock request_archive to return fake archive data singlefile, readability, screenshot = make_fake_archive_result() - async def mock_request_archive(url): + async def mock_request_archive(url, compress=False): return singlefile, readability, screenshot # Patch in both lib and archive module (imported at module level) diff --git a/wrolpi/downloader.py b/wrolpi/downloader.py index 1fc73787..6c4e1f18 100644 --- a/wrolpi/downloader.py +++ b/wrolpi/downloader.py @@ -2230,6 +2230,9 @@ class RSSDownloader(Downloader): for key in CHANNEL_INHERITABLE_SETTINGS: if key in settings: next_download_settings[key] = settings[key] + # Pass compression on to child archive downloads. + if 'compress_singlefile' in settings: + next_download_settings['compress_singlefile'] = settings['compress_singlefile'] # Use download.destination column (settings['destination'] is legacy) if download.destination: next_download_settings['destination'] = str(download.destination) diff --git a/wrolpi/files/lib.py b/wrolpi/files/lib.py index bacba670..a0177857 100644 --- a/wrolpi/files/lib.py +++ b/wrolpi/files/lib.py @@ -384,6 +384,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str): return 'model/3mf' if suffix.endswith('.mp4'): return 'video/mp4' + if suffix.endswith('.html') or suffix.endswith('.htm'): + # Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail. + return 'text/html' if suffix.endswith('.hgt'): return 'application/octet-stream' if mimetype == 'text/plain': @@ -414,6 +417,9 @@ def _mimetype_suffix_map(path: Path, mimetype: str): if mimetype == 'application/x-subrip': # Fallback to old mimetype. return 'text/srt' + if mimetype == 'application/zip' and (suffix == '.html' or suffix == '.htm'): + # Compressed (SingleFileZ) singlefiles are HTML with a binary ZIP tail. + return 'text/html' if mimetype == 'application/zip' and suffix == '.docx': return 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' if mimetype == 'application/zip' and suffix == '.cbz': diff --git a/wrolpi/schema.py b/wrolpi/schema.py index e4128a8a..3eaeb092 100644 --- a/wrolpi/schema.py +++ b/wrolpi/schema.py @@ -260,6 +260,7 @@ class DownloadSettings: video_resolutions: List[str] = field(default_factory=list) parent_download_url: Optional[str] = None skip_already_downloaded: Optional[bool] = None + compress_singlefile: Optional[bool] = None # Inheritable video settings (Optional so they're only set when user overrides) writesubtitles: Optional[bool] = None writeautomaticsub: Optional[bool] = None