wrolpi/modules/archive/test/test_archive_downloader.py
lrnselfreliance 32405a9132
A download records the sha256 of its files in the wrolpi section (#904)
* A download records the sha256 of its files in the wrolpi section

A video download writes `wrolpi.sha256` into its .info.json; an archive download writes it into
its .readability.json.  Sums are keyed by suffix, so a rename or Reorganize keeps them valid,
and the metadata JSON holding them is not hashed.

Large files are hashed in execute_download (in a thread, without the DB session).  Finalize
reuses those sums while a file's mtime is unchanged and hashes only files finalize created or
changed (a converted poster, the ffprobe json).

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

* A video download hashes only for an info json, and finalize never suspends

Finalize holds the write session; awaiting the hash let the loop run a switch whose
write waited on that session, stalling the loop for the busy timeout.  Finalize now
hashes inline: the large files were hashed in execute_download.

A download that writes no info json has nowhere to keep the sums, so it no longer
reads its files to hash them.

Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com>

---------

Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
2026-10-30 19:34:49 -06:00

300 lines
14 KiB
Python

"""Phase-split ArchiveDownloader tests.
These tests exercise prepare_download / execute_download / finalize_download in
isolation. None of them use async_client or test_download_manager — that's the
whole point of the refactor.
"""
import asyncio
import hashlib
import json
import pathlib
import pytest
import modules.archive
from modules.archive import (
ExecutedArchive,
PreparedArchive,
archive_downloader,
)
from modules.archive.conftest import make_test_ctx
from modules.archive.lib import write_archive_files
from modules.archive.models import Archive
from modules.archive.test.test_lib import make_fake_archive_result
from wrolpi.collections import Collection
from wrolpi.downloader import Download, DownloadResult
from wrolpi.errors import UnrecoverableDownloadError
def test_prepare_max_attempts_raises(test_session):
"""prepare_download enforces the attempt cap before any I/O."""
download = Download(url='https://example.com/x', downloader='archive', attempts=4)
with pytest.raises(UnrecoverableDownloadError):
archive_downloader.prepare_download(test_session, download)
def test_prepare_uses_collection_directory(test_session, test_directory):
"""When a domain Collection has a directory, prepare_download routes the archive there."""
custom_dir = test_directory / 'archive/News/example.com'
custom_dir.mkdir(parents=True, exist_ok=True)
test_session.add(Collection(name='example.com', kind='domain', directory=custom_dir))
test_session.commit()
download = Download(url='https://example.com/some-article', downloader='archive', settings={})
prepared = archive_downloader.prepare_download(test_session, download)
assert isinstance(prepared, PreparedArchive)
assert prepared.url == 'https://example.com/some-article'
assert prepared.destination == custom_dir
def test_prepare_explicit_destination_overrides_collection(test_session, test_directory):
"""An explicit settings['destination'] beats the domain collection's directory."""
collection_dir = test_directory / 'archive/News/example.com'
collection_dir.mkdir(parents=True, exist_ok=True)
test_session.add(Collection(name='example.com', kind='domain', directory=collection_dir))
test_session.commit()
explicit_dir = test_directory / 'archive/Special/my-archives'
explicit_dir.mkdir(parents=True, exist_ok=True)
download = Download(
url='https://example.com/some-article',
downloader='archive',
settings={'destination': str(explicit_dir)},
)
prepared = archive_downloader.prepare_download(test_session, download)
assert prepared.destination == explicit_dir
@pytest.mark.asyncio
async def test_execute_cancels_via_ctx(test_session, test_directory, monkeypatch):
"""When ctx reports cancelled, the cancel_wrapper that the dispatch puts around
execute_download returns a cancel DownloadResult — no global download_manager,
no Sanic shared_ctx."""
# Force the local-singlefile path so the run_command stub is what blocks.
import modules.archive
monkeypatch.setattr(modules.archive, 'DOCKERIZED', False)
monkeypatch.setattr(modules.archive, 'PYTEST', False)
async def never_completes(*args, **kwargs):
await asyncio.sleep(60)
raise AssertionError('should have been cancelled')
# Pre-cancelled: the very first poll inside cancel_wrapper aborts the run.
ctx = make_test_ctx(is_cancelled=lambda: True, run_command_=never_completes)
prepared = PreparedArchive(
url='https://example.com/x',
destination=test_directory / 'archive/example.com',
settings={},
)
prepared.destination.mkdir(parents=True, exist_ok=True)
# Mirror the dispatch layer: wrap execute_download in cancel_wrapper.
coro = archive_downloader.execute_download(prepared, ctx)
result = await archive_downloader.cancel_wrapper(coro, Download(url=prepared.url), ctx=ctx)
assert isinstance(result, DownloadResult)
assert result.success is False
assert 'cancel' in (result.error or '').lower()
def make_process_runner_recorder(singlefile: bytes, compressed: bytes = None):
"""Stub Downloader.process_runner: record the call, write the singlefile to the output file
(the last argument of the command, like single-file does), return a successful CommandResult.
Capture runs (no --compress-content) write `singlefile`; conversion runs write `compressed`
(or nothing, when compressed is None, to simulate a failed conversion)."""
from wrolpi.cmd import CommandResult
calls = []
async def process_runner(download, cmd, cwd, **kwargs):
calls.append(dict(cmd=cmd, cwd=cwd, **kwargs))
if '--compress-content' not in cmd:
pathlib.Path(cmd[-1]).write_bytes(singlefile)
elif compressed is not None:
pathlib.Path(cmd[-1]).write_bytes(compressed)
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
return process_runner, calls
@pytest.mark.asyncio
async def test_do_singlefile_uses_deno(test_session, test_directory, monkeypatch, compressed_singlefile_factory):
"""single-file is run under Deno (it is designed for it; under node it cannot reach the
browser CDP on IPv6-enabled hosts and never exits). The page is captured uncompressed,
then converted to a compressed (SingleFileZ) file with a second local single-file run."""
import modules.archive
singlefile = b'<html><!--\n Page saved with SingleFile \n url: https://example.com \n--></html>'
compressed = compressed_singlefile_factory()
process_runner, calls = make_process_runner_recorder(singlefile, compressed)
monkeypatch.setattr(modules.archive, 'DENO_BIN', '/usr/local/bin/deno')
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_DENO_SCRIPT',
'/usr/local/lib/node_modules/single-file-cli/single-file')
monkeypatch.setattr(modules.archive, 'CHROMIUM', '/usr/bin/chromium')
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
prepared = PreparedArchive(url='https://example.com', destination=None,
settings={'compress_singlefile': True})
result, page_html = await archive_downloader.do_singlefile(prepared, make_test_ctx())
# The compressed file is stored; the uncompressed page is used for readability/screenshot.
assert result == compressed
assert page_html == singlefile
capture, convert = calls
cmd = capture['cmd']
assert cmd[2:9] == ('/usr/local/bin/deno', 'run', '--node-modules-dir=manual',
'--allow-read', '--allow-write', '--allow-net', '--allow-env')
assert '/usr/local/lib/node_modules/single-file-cli/single-file' in cmd
# The capture is always uncompressed.
assert '--compress-content' not in cmd
# The singlefile is written to a file; Deno >= 2.3 truncates large --dump-content output.
assert '--dump-content' not in cmd
assert cmd[-2] == 'https://example.com'
assert cmd[-1].endswith('page.html')
# The browser single-file leaves behind is reaped with the process group.
assert capture['start_new_session'] is True
# The browser runs via the stderr-discarding wrapper (Deno >= 2.3 kills it otherwise).
wrapper = cmd[cmd.index('--browser-executable-path') + 1]
assert wrapper.endswith('scripts/singlefile_browser_wrapper.sh')
assert capture['env']['WROLPI_BROWSER'] == '/usr/bin/chromium'
# The conversion is a local (file://) operation; the page is not fetched again.
cmd = convert['cmd']
assert '--compress-content' in cmd
assert cmd[-2].startswith('file://') and cmd[-2].endswith('page.html')
assert cmd[-1].endswith('compressed.html')
# Without the compress setting there is one run, and the page is stored as-is.
calls.clear()
prepared = PreparedArchive(url='https://example.com', destination=None, settings={})
result, page_html = await archive_downloader.do_singlefile(prepared, make_test_ctx())
assert result == page_html == singlefile
assert len(calls) == 1
# When the compression run fails, the uncompressed singlefile is stored instead.
process_runner, calls = make_process_runner_recorder(singlefile, compressed=None) # Conversion writes nothing.
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
prepared = PreparedArchive(url='https://example.com', destination=None,
settings={'compress_singlefile': True})
result, page_html = await archive_downloader.do_singlefile(prepared, make_test_ctx())
assert len(calls) == 2 # The conversion was attempted.
assert result == page_html == singlefile
# A page that has not finished archiving in ten minutes is hung; the timeout reaps it.
assert archive_downloader.timeout == 10 * 60
@pytest.mark.asyncio
async def test_do_singlefile_node_fallback(test_session, test_directory, monkeypatch):
"""Without Deno, single-file runs under node with the IPv4-fallback fix."""
import modules.archive
singlefile = b'<html><!--\n Page saved with SingleFile \n url: https://example.com \n--></html>'
process_runner, calls = make_process_runner_recorder(singlefile)
monkeypatch.setattr(modules.archive, 'DENO_BIN', None)
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_DENO_SCRIPT', None)
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_BIN', '/usr/local/bin/single-file')
monkeypatch.setattr(modules.archive, 'CHROMIUM', '/usr/bin/chromium')
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
prepared = PreparedArchive(url='https://example.com', destination=None, settings={})
await archive_downloader.do_singlefile(prepared, make_test_ctx())
call, = calls
assert call['cmd'][2] == '/usr/local/bin/single-file'
# node 18 has no IPv4 fallback when "localhost" resolves to ::1 first.
assert call['env']['NODE_OPTIONS'] == '--enable-network-family-autoselection'
assert call['start_new_session'] is True
@pytest.mark.asyncio
async def test_finalize_adds_tags(async_client, test_session, test_directory, tag_factory):
"""finalize_download applies download.tag_names to the registered Archive.
Uses async_client because tag_factory's upsert_tag activates a Sanic switch
(shared_ctx.switches_lock). That's an orthogonal coupling — the rest of
finalize_download is Sanic-free.
"""
singlefile, readability, screenshot = make_fake_archive_result()
destination = test_directory / 'archive/example.com'
destination.mkdir(parents=True, exist_ok=True)
written = write_archive_files('https://example.com/x', singlefile, readability, screenshot,
destination=destination)
executed = ExecutedArchive(written=written)
tag = await tag_factory()
download = Download(
url='https://example.com/x',
downloader='archive',
tag_names=[tag.name],
)
result = archive_downloader.finalize_download(test_session, download, executed)
test_session.commit()
assert result.success is True
archives = test_session.query(Archive).all()
assert len(archives) == 1
archive_tag_names = [tf.tag.name for tf in archives[0].file_group.tag_files]
assert tag.name in archive_tag_names
@pytest.mark.asyncio
@pytest.mark.parametrize('configured, expected', [
# A config can outlive the OS it was written on: Raspberry Pi OS names Chromium "chromium-browser".
('/nonexistent/chromium-browser', '/usr/bin/chromium'),
(None, '/usr/bin/chromium'),
('/bin/sh', '/bin/sh'),
])
async def test_do_singlefile_configured_browser(test_session, test_directory, monkeypatch, configured, expected):
"""The configured browser is used when it is installed; otherwise one is detected."""
from types import SimpleNamespace
import modules.archive
singlefile = b'<html><!--\n Page saved with SingleFile \n url: https://example.com \n--></html>'
process_runner, calls = make_process_runner_recorder(singlefile)
monkeypatch.setattr(modules.archive, 'DENO_BIN', '/usr/local/bin/deno')
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_DENO_SCRIPT',
'/usr/local/lib/node_modules/single-file-cli/single-file')
monkeypatch.setattr(modules.archive, 'CHROMIUM', '/usr/bin/chromium')
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
monkeypatch.setattr(modules.archive.lib, 'get_archive_downloader_config',
lambda: SimpleNamespace(browser_executable=configured, browser_args=None, user_agent=None))
prepared = PreparedArchive(url='https://example.com', destination=None, settings={})
await archive_downloader.do_singlefile(prepared, make_test_ctx())
capture, = calls
assert capture['env']['WROLPI_BROWSER'] == expected
@pytest.mark.asyncio
async def test_download_records_sha256(async_client, test_session, test_directory, monkeypatch):
"""An archive download records the sha256 of each file in its group in the readability json's wrolpi section."""
singlefile, readability, screenshot = make_fake_archive_result()
async def mock_request_archive(url, compress=False):
return singlefile, readability, screenshot
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
download = Download(url='https://example.com/x', downloader='archive',
settings={'parent_download_url': 'https://example.com/feed'})
prepared = archive_downloader.prepare_download(test_session, download)
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
result = archive_downloader.finalize_download(test_session, download, executed)
test_session.commit()
assert result.success is True
archive = test_session.query(Archive).one()
readability_json_path = archive.file_group.readability_json_path
wrolpi = json.loads(readability_json_path.read_text())['wrolpi']
assert wrolpi['parent_download_url'] == 'https://example.com/feed'
stem = archive.singlefile_path.name[:-len('.html')]
expected = {path.name[len(stem):]: hashlib.sha256(path.read_bytes()).hexdigest()
for path in archive.file_group.my_paths() if path != readability_json_path}
assert set(expected) == {'.html', '.readability.html', '.readability.txt', '.png'}
assert wrolpi['sha256'] == expected