mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-19 22:46:20 -05:00
* Archives can be compressed (SingleFileZ self-extracting HTML). A new compress_singlefile download setting runs single-file with --compress-content, creating a self-extracting HTML file which is also a valid ZIP. Only the singlefile .html is compressed; readability files are always written uncompressed. Readability, screenshots, titles, and URLs are extracted from the index.html inside the ZIP. A Compress checkbox in the Archive and RSS download forms remembers the user's choice in local storage. Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
187 lines
8.2 KiB
Python
187 lines
8.2 KiB
Python
"""Tests for compressed (SingleFileZ) singlefile Archives.
|
|
|
|
A compressed singlefile is a self-extracting HTML file which is also a valid ZIP containing the
|
|
uncompressed page as index.html. Only the singlefile .html is compressed; readability files are
|
|
always written uncompressed.
|
|
"""
|
|
import pathlib
|
|
import zipfile
|
|
|
|
import pytest
|
|
|
|
import modules.archive
|
|
from modules.archive import archive_downloader, PreparedArchive, model_archive
|
|
from modules.archive.conftest import make_test_ctx
|
|
from modules.archive.lib import (
|
|
is_compressed_singlefile,
|
|
get_compressed_singlefile_index,
|
|
get_url_from_singlefile,
|
|
is_singlefile_file,
|
|
write_archive_files,
|
|
)
|
|
from modules.archive.test.test_lib import make_fake_archive_result
|
|
from wrolpi.downloader import Download
|
|
from wrolpi.files.lib import get_mimetype, _mimetype_suffix_map
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.schema import DownloadRequest
|
|
|
|
|
|
def test_is_compressed_singlefile(compressed_singlefile_factory, singlefile_contents_factory):
|
|
compressed = compressed_singlefile_factory()
|
|
assert is_compressed_singlefile(compressed) is True
|
|
|
|
# A regular singlefile is not compressed.
|
|
assert is_compressed_singlefile(singlefile_contents_factory().encode()) is False
|
|
# Garbage is not compressed.
|
|
assert is_compressed_singlefile(b'not a zip') is False
|
|
|
|
# The uncompressed page can be extracted.
|
|
index = get_compressed_singlefile_index(compressed)
|
|
assert b'test compressed singlefile' in index
|
|
assert b'<title>compressed title</title>' in index
|
|
|
|
|
|
def test_get_url_from_compressed_singlefile(compressed_singlefile_factory):
|
|
"""The URL can be extracted from the plain-text prelude despite the binary ZIP tail."""
|
|
compressed = compressed_singlefile_factory(url='https://wrolpi.org/some-article')
|
|
assert get_url_from_singlefile(compressed) == 'https://wrolpi.org/some-article'
|
|
|
|
|
|
@pytest.mark.parametrize(
|
|
'name,expected', [
|
|
('2000-01-01-00-00-00_Some NA.html', True), # Matches the WROLPi Archive file name.
|
|
('foo.html', True), # Detected by the SingleFile header in the prelude.
|
|
('foo.txt', False),
|
|
]
|
|
)
|
|
def test_is_singlefile_file_compressed(name, expected, make_files_structure, compressed_singlefile_factory):
|
|
"""A compressed singlefile is recognized even though it has a binary ZIP tail."""
|
|
path, = make_files_structure([name])
|
|
path.write_bytes(compressed_singlefile_factory())
|
|
assert is_singlefile_file(path) == expected
|
|
|
|
|
|
def test_compressed_singlefile_mimetype(make_files_structure, compressed_singlefile_factory):
|
|
"""A compressed singlefile must be `text/html` so the archive modeler finds it."""
|
|
# `magic` reports application/octet-stream for the real (binary) compressed singlefile.
|
|
assert _mimetype_suffix_map(pathlib.Path('foo.html'), 'application/octet-stream') == 'text/html'
|
|
|
|
path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
|
|
path.write_bytes(compressed_singlefile_factory())
|
|
assert get_mimetype(path) == 'text/html'
|
|
|
|
|
|
def test_write_archive_files_compressed(test_directory, test_session, image_bytes_factory,
|
|
compressed_singlefile_factory):
|
|
"""The compressed singlefile is written verbatim (prettifying would corrupt the ZIP);
|
|
readability files are written uncompressed."""
|
|
compressed = compressed_singlefile_factory()
|
|
readability = dict(content='<html>readability content</html>', textContent='readability text', title='a title')
|
|
destination = test_directory / 'archive/example.com'
|
|
destination.mkdir(parents=True)
|
|
|
|
written = write_archive_files('https://example.com', compressed, readability, image_bytes_factory(),
|
|
destination=destination)
|
|
|
|
singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)
|
|
# The file on disk is unchanged, and still a valid ZIP.
|
|
assert singlefile_path.read_bytes() == compressed
|
|
assert zipfile.is_zipfile(singlefile_path)
|
|
|
|
# Readability files are plain text, not compressed.
|
|
readability_path = next(i for i in written.paths if i.name.endswith('.readability.html'))
|
|
assert 'readability content' in readability_path.read_text()
|
|
readability_txt_path = next(i for i in written.paths if i.name.endswith('.readability.txt'))
|
|
assert 'readability text' in readability_txt_path.read_text()
|
|
|
|
|
|
def test_write_archive_files_compressed_title_fallback(test_directory, test_session, compressed_singlefile_factory):
|
|
"""Without a readability title, the title comes from the ZIP's index.html."""
|
|
compressed = compressed_singlefile_factory(title='the fallback title')
|
|
destination = test_directory / 'archive/example.com'
|
|
destination.mkdir(parents=True)
|
|
|
|
written = write_archive_files('https://example.com', compressed, None, None, destination=destination)
|
|
|
|
singlefile_path = next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)
|
|
assert 'the fallback title' in singlefile_path.name
|
|
assert singlefile_path.read_bytes() == compressed
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_download_compressed(test_session, test_directory, monkeypatch, compressed_singlefile_factory):
|
|
"""The compress_singlefile setting is forwarded to the archive service, and the compressed
|
|
bytes returned are written to disk unchanged."""
|
|
compressed = compressed_singlefile_factory()
|
|
_, readability, screenshot = make_fake_archive_result()
|
|
request_archive_kwargs = {}
|
|
|
|
async def mock_request_archive(url, compress=False):
|
|
request_archive_kwargs['compress'] = compress
|
|
return compressed, readability, screenshot
|
|
|
|
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
|
|
|
|
destination = test_directory / 'archive/example.com'
|
|
destination.mkdir(parents=True)
|
|
prepared = PreparedArchive(
|
|
url='https://example.com',
|
|
destination=destination,
|
|
settings={'compress_singlefile': True},
|
|
)
|
|
|
|
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
|
|
|
|
assert request_archive_kwargs['compress'] is True
|
|
singlefile_path = next(i for i in executed.written.paths if i.suffix == '.html' and '.readability' not in i.name)
|
|
assert singlefile_path.read_bytes() == compressed
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_model_archive_compressed(async_client, test_session, test_directory, make_files_structure,
|
|
compressed_singlefile_factory):
|
|
"""A compressed singlefile on disk (without readability files) can be modeled as an Archive;
|
|
the title and URL come from the compressed file itself.
|
|
|
|
async_client is required because get_or_create_domain_collection activates a Sanic switch."""
|
|
path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
|
|
path.write_bytes(compressed_singlefile_factory(url='https://example.com/article', title='compressed title'))
|
|
|
|
file_group = FileGroup.from_paths(test_session, path)
|
|
test_session.flush()
|
|
|
|
archive = model_archive(test_session, file_group)
|
|
test_session.commit()
|
|
|
|
assert archive.file_group.url == 'https://example.com/article'
|
|
assert archive.file_group.title == 'compressed title'
|
|
assert archive.collection.name == 'example.com'
|
|
|
|
|
|
def test_download_request_compress_singlefile():
|
|
"""compress_singlefile=True survives DownloadRequest settings validation."""
|
|
request = DownloadRequest(
|
|
urls=['https://example.com'],
|
|
downloader='archive',
|
|
settings=dict(compress_singlefile=True),
|
|
)
|
|
assert request.settings['compress_singlefile'] is True
|
|
|
|
|
|
def test_rss_forwards_compress_singlefile(test_session):
|
|
"""RSSDownloader passes compress_singlefile on to the child archive downloads."""
|
|
from wrolpi.downloader import rss_downloader, ExecutedRSS
|
|
|
|
download = Download(
|
|
url='https://example.com/feed',
|
|
downloader='rss',
|
|
sub_downloader='archive',
|
|
settings={'compress_singlefile': True},
|
|
)
|
|
executed = ExecutedRSS(yt_channel_id=None, candidate_urls=['https://example.com/article'])
|
|
|
|
result = rss_downloader.finalize_download(test_session, download, executed)
|
|
|
|
assert result.success is True
|
|
assert result.downloads == ['https://example.com/article']
|
|
assert result.settings['compress_singlefile'] is True
|