wrolpi/modules/archive/test/test_compressed_singlefile.py

"""Tests for compressed (SingleFileZ) singlefile Archives.

A compressed singlefile is a self-extracting HTML file which is also a valid ZIP containing the
uncompressed page as index.html.  Only the singlefile .html is compressed; readability files are
always written uncompressed.
"""
import base64
import pathlib
import zipfile

import pytest

import modules.archive
from modules.archive import archive_downloader, PreparedArchive, model_archive
from modules.archive.conftest import make_test_ctx
from modules.archive.lib import (
    is_compressed_singlefile,
    get_compressed_singlefile_index,
    get_url_from_singlefile,
    inline_compressed_singlefile_resources,
    is_singlefile_file,
    write_archive_files,
)
from modules.archive.test.test_lib import make_fake_archive_result
from wrolpi.downloader import Download
from wrolpi.files.lib import get_mimetype, _mimetype_suffix_map
from wrolpi.files.models import FileGroup
from wrolpi.schema import DownloadRequest


def test_compressed_singlefile_helpers(compressed_singlefile_factory, singlefile_contents_factory):
    """Compressed singlefiles are detected, and their page/URL can be extracted."""
    compressed = compressed_singlefile_factory(url='https://wrolpi.org/some-article')
    assert is_compressed_singlefile(compressed) is True

    # A regular singlefile is not compressed.
    assert is_compressed_singlefile(singlefile_contents_factory().encode()) is False
    # Garbage is not compressed.
    assert is_compressed_singlefile(b'not a zip') is False

    # The uncompressed page can be extracted.
    index = get_compressed_singlefile_index(compressed)
    assert b'test compressed singlefile' in index
    assert b'<title>compressed title</title>' in index

    # The URL can be extracted from the plain-text prelude despite the binary ZIP tail.
    assert get_url_from_singlefile(compressed) == 'https://wrolpi.org/some-article'


@pytest.mark.parametrize(
    'name,expected', [
        ('2000-01-01-00-00-00_Some NA.html', True),  # Matches the WROLPi Archive file name.
        ('foo.html', True),  # Detected by the SingleFile header in the prelude.
        ('foo.txt', False),
    ]
)
def test_is_singlefile_file_compressed(name, expected, make_files_structure, compressed_singlefile_factory):
    """A compressed singlefile is recognized even though it has a binary ZIP tail."""
    path, = make_files_structure([name])
    path.write_bytes(compressed_singlefile_factory())
    assert is_singlefile_file(path) == expected


def test_compressed_singlefile_mimetype(make_files_structure, compressed_singlefile_factory):
    """A compressed singlefile must be `text/html` so the archive modeler finds it."""
    # `magic` reports application/octet-stream for the real (binary) compressed singlefile.
    assert _mimetype_suffix_map(pathlib.Path('foo.html'), 'application/octet-stream') == 'text/html'

    path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
    path.write_bytes(compressed_singlefile_factory())
    assert get_mimetype(path) == 'text/html'


def find_singlefile_path(written):
    return next(i for i in written.paths if i.suffix == '.html' and '.readability' not in i.name)


def test_write_archive_files_compressed(test_directory, test_session, image_bytes_factory,
                                        compressed_singlefile_factory):
    """The compressed singlefile is written verbatim (prettifying would corrupt the ZIP);
    readability files are written uncompressed, with their ZIP image references inlined."""
    compressed = compressed_singlefile_factory()
    readability = dict(content='<html><img src="images/0.png">readability content</html>',
                       textContent='readability text', title='a title')
    destination = test_directory / 'archive/example.com'
    destination.mkdir(parents=True)

    written = write_archive_files('https://example.com', compressed, readability, image_bytes_factory(),
                                  destination=destination)

    singlefile_path = find_singlefile_path(written)
    # The file on disk is unchanged, and still a valid ZIP.
    assert singlefile_path.read_bytes() == compressed
    assert zipfile.is_zipfile(singlefile_path)

    # Readability files are plain text, not compressed; their ZIP image references are inlined.
    readability_path = next(i for i in written.paths if i.name.endswith('.readability.html'))
    readability_text = readability_path.read_text()
    assert 'readability content' in readability_text
    assert 'data:image/png;base64,' in readability_text
    assert 'src="images/0.png"' not in readability_text
    readability_txt_path = next(i for i in written.paths if i.name.endswith('.readability.txt'))
    assert 'readability text' in readability_txt_path.read_text()

    # Without a readability title, the title comes from the ZIP's index.html.
    compressed = compressed_singlefile_factory(title='the fallback title')
    written = write_archive_files('https://example.com', compressed, None, None, destination=destination)
    singlefile_path = find_singlefile_path(written)
    assert 'the fallback title' in singlefile_path.name
    assert singlefile_path.read_bytes() == compressed

    # A singlefile marked compressed (<html data-sfz) whose ZIP is unreadable (e.g. truncated) is
    # stored verbatim; prettifying would destroy whatever remains.
    broken = b'<!DOCTYPE html> <html data-sfz><!--\n Page saved with SingleFile \n' \
             b' url: https://example.com \n--><body hidden></body></html>' \
             b'PK\x03\x04truncated zip data without a central directory'
    written = write_archive_files('https://example.com', broken, None, None, destination=destination)
    assert find_singlefile_path(written).read_bytes() == broken


def test_inline_compressed_singlefile_resources(compressed_singlefile_factory):
    """Readability extracted from a compressed singlefile references images by relative ZIP paths;
    they are inlined as data URIs so the readability file is self-contained.

    Relative and absolutized ZIP paths are inlined; data/external URLs and unknown paths are
    left alone.  readability-extractor absolutizes relative srcs against the page URL."""
    compressed = compressed_singlefile_factory()
    expected_data_uri = 'data:image/png;base64,' + base64.b64encode(b'\x89PNG fake image bytes').decode()

    content = '<html><body>' \
              '<img src="images/0.png" srcset="images/0.png 1x">' \
              '<img src="./images/0.png">' \
              '<img src="../images/0.png">' \
              '<img src="https://example.com/images/0.png">' \
              '<img src="https://other.com/remote.png">' \
              '<img src="data:image/png;base64,QUJD">' \
              '<img src="images/does-not-exist.png">' \
              '<p>article text</p></body></html>'
    inlined = inline_compressed_singlefile_resources(content, compressed, 'https://example.com/page')
    assert inlined.count(expected_data_uri) == 3  # The relative, ./-relative, and absolutized forms.
    assert 'srcset' not in inlined
    # A path outside the ZIP directory must not match a ZIP entry.
    assert '../images/0.png' in inlined
    assert 'https://other.com/remote.png' in inlined
    assert 'data:image/png;base64,QUJD' in inlined
    assert 'images/does-not-exist.png' in inlined


@pytest.mark.asyncio
async def test_execute_download_compressed(test_session, test_directory, monkeypatch, compressed_singlefile_factory):
    """The compress_singlefile setting is forwarded to the archive service, and the compressed
    bytes returned are written to disk unchanged."""
    compressed = compressed_singlefile_factory()
    _, readability, screenshot = make_fake_archive_result()
    request_archive_kwargs = {}

    async def mock_request_archive(url, compress=False):
        request_archive_kwargs['compress'] = compress
        return compressed, readability, screenshot

    monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)

    destination = test_directory / 'archive/example.com'
    destination.mkdir(parents=True)
    prepared = PreparedArchive(
        url='https://example.com',
        destination=destination,
        settings={'compress_singlefile': True},
    )

    executed = await archive_downloader.execute_download(prepared, make_test_ctx())

    assert request_archive_kwargs['compress'] is True
    singlefile_path = next(i for i in executed.written.paths if i.suffix == '.html' and '.readability' not in i.name)
    assert singlefile_path.read_bytes() == compressed


@pytest.mark.asyncio
async def test_model_archive_compressed(async_client, test_session, test_directory, make_files_structure,
                                        compressed_singlefile_factory):
    """A compressed singlefile on disk (without readability files) can be modeled as an Archive;
    the title and URL come from the compressed file itself.

    async_client is required because get_or_create_domain_collection activates a Sanic switch."""
    path, = make_files_structure(['archive/2000-01-01-00-00-00_compressed.html'])
    path.write_bytes(compressed_singlefile_factory(url='https://example.com/article', title='compressed title'))

    file_group = FileGroup.from_paths(test_session, path)
    test_session.flush()

    archive = model_archive(test_session, file_group)
    test_session.commit()

    assert archive.file_group.url == 'https://example.com/article'
    assert archive.file_group.title == 'compressed title'
    assert archive.collection.name == 'example.com'


def test_html_indexer_compressed(make_files_structure, compressed_singlefile_factory):
    """The generic HTML indexer is binary-safe: it indexes the page inside a compressed
    singlefile instead of crashing on the binary ZIP tail.

    The archive modeler claims WROLPi's own singlefiles; this indexer handles foreign HTML
    files, e.g. a SingleFileZ file saved by the browser extension and dropped into the media
    directory."""
    from wrolpi.files.indexers import HTMLIndexer

    path, = make_files_structure(['foreign.html'])
    path.write_bytes(compressed_singlefile_factory(title='foreign singlefilez'))

    title, _, _, words = HTMLIndexer.create_index(path)
    assert title == 'foreign singlefilez'
    assert 'test compressed singlefile' in words


def test_compress_singlefile_setting_flow(test_session):
    """compress_singlefile survives DownloadRequest settings validation, and the RSSDownloader
    passes it on to the child archive downloads."""
    from wrolpi.downloader import rss_downloader, ExecutedRSS

    request = DownloadRequest(
        urls=['https://example.com'],
        downloader='archive',
        settings=dict(compress_singlefile=True),
    )
    assert request.settings['compress_singlefile'] is True

    download = Download(
        url='https://example.com/feed',
        downloader='rss',
        sub_downloader='archive',
        settings={'compress_singlefile': True},
    )
    executed = ExecutedRSS(yt_channel_id=None, candidate_urls=['https://example.com/article'])

    result = rss_downloader.finalize_download(test_session, download, executed)

    assert result.success is True
    assert result.downloads == ['https://example.com/article']
    assert result.settings['compress_singlefile'] is True