mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-11-21 15:36:18 -05:00
* A download records the sha256 of its files in the wrolpi section A video download writes `wrolpi.sha256` into its .info.json; an archive download writes it into its .readability.json. Sums are keyed by suffix, so a rename or Reorganize keeps them valid, and the metadata JSON holding them is not hashed. Large files are hashed in execute_download (in a thread, without the DB session). Finalize reuses those sums while a file's mtime is unchanged and hashes only files finalize created or changed (a converted poster, the ffprobe json). Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> * A video download hashes only for an info json, and finalize never suspends Finalize holds the write session; awaiting the hash let the loop run a switch whose write waited on that session, stalling the loop for the busy timeout. Finalize now hashes inline: the large files were hashed in execute_download. A download that writes no info json has nowhere to keep the sums, so it no longer reads its files to hash them. Co-Authored-By: Claude Opus 5.5 <noreply@anthropic.com> --------- Co-authored-by: Claude Opus 5.5 <noreply@anthropic.com>
300 lines
14 KiB
Python
300 lines
14 KiB
Python
"""Phase-split ArchiveDownloader tests.
|
|
|
|
These tests exercise prepare_download / execute_download / finalize_download in
|
|
isolation. None of them use async_client or test_download_manager — that's the
|
|
whole point of the refactor.
|
|
"""
|
|
import asyncio
|
|
import hashlib
|
|
import json
|
|
import pathlib
|
|
|
|
import pytest
|
|
|
|
import modules.archive
|
|
from modules.archive import (
|
|
ExecutedArchive,
|
|
PreparedArchive,
|
|
archive_downloader,
|
|
)
|
|
from modules.archive.conftest import make_test_ctx
|
|
from modules.archive.lib import write_archive_files
|
|
from modules.archive.models import Archive
|
|
from modules.archive.test.test_lib import make_fake_archive_result
|
|
from wrolpi.collections import Collection
|
|
from wrolpi.downloader import Download, DownloadResult
|
|
from wrolpi.errors import UnrecoverableDownloadError
|
|
|
|
|
|
def test_prepare_max_attempts_raises(test_session):
|
|
"""prepare_download enforces the attempt cap before any I/O."""
|
|
download = Download(url='https://example.com/x', downloader='archive', attempts=4)
|
|
|
|
with pytest.raises(UnrecoverableDownloadError):
|
|
archive_downloader.prepare_download(test_session, download)
|
|
|
|
|
|
def test_prepare_uses_collection_directory(test_session, test_directory):
|
|
"""When a domain Collection has a directory, prepare_download routes the archive there."""
|
|
custom_dir = test_directory / 'archive/News/example.com'
|
|
custom_dir.mkdir(parents=True, exist_ok=True)
|
|
test_session.add(Collection(name='example.com', kind='domain', directory=custom_dir))
|
|
test_session.commit()
|
|
|
|
download = Download(url='https://example.com/some-article', downloader='archive', settings={})
|
|
|
|
prepared = archive_downloader.prepare_download(test_session, download)
|
|
|
|
assert isinstance(prepared, PreparedArchive)
|
|
assert prepared.url == 'https://example.com/some-article'
|
|
assert prepared.destination == custom_dir
|
|
|
|
|
|
def test_prepare_explicit_destination_overrides_collection(test_session, test_directory):
|
|
"""An explicit settings['destination'] beats the domain collection's directory."""
|
|
collection_dir = test_directory / 'archive/News/example.com'
|
|
collection_dir.mkdir(parents=True, exist_ok=True)
|
|
test_session.add(Collection(name='example.com', kind='domain', directory=collection_dir))
|
|
test_session.commit()
|
|
|
|
explicit_dir = test_directory / 'archive/Special/my-archives'
|
|
explicit_dir.mkdir(parents=True, exist_ok=True)
|
|
download = Download(
|
|
url='https://example.com/some-article',
|
|
downloader='archive',
|
|
settings={'destination': str(explicit_dir)},
|
|
)
|
|
|
|
prepared = archive_downloader.prepare_download(test_session, download)
|
|
|
|
assert prepared.destination == explicit_dir
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_execute_cancels_via_ctx(test_session, test_directory, monkeypatch):
|
|
"""When ctx reports cancelled, the cancel_wrapper that the dispatch puts around
|
|
execute_download returns a cancel DownloadResult — no global download_manager,
|
|
no Sanic shared_ctx."""
|
|
# Force the local-singlefile path so the run_command stub is what blocks.
|
|
import modules.archive
|
|
monkeypatch.setattr(modules.archive, 'DOCKERIZED', False)
|
|
monkeypatch.setattr(modules.archive, 'PYTEST', False)
|
|
|
|
async def never_completes(*args, **kwargs):
|
|
await asyncio.sleep(60)
|
|
raise AssertionError('should have been cancelled')
|
|
|
|
# Pre-cancelled: the very first poll inside cancel_wrapper aborts the run.
|
|
ctx = make_test_ctx(is_cancelled=lambda: True, run_command_=never_completes)
|
|
prepared = PreparedArchive(
|
|
url='https://example.com/x',
|
|
destination=test_directory / 'archive/example.com',
|
|
settings={},
|
|
)
|
|
prepared.destination.mkdir(parents=True, exist_ok=True)
|
|
|
|
# Mirror the dispatch layer: wrap execute_download in cancel_wrapper.
|
|
coro = archive_downloader.execute_download(prepared, ctx)
|
|
result = await archive_downloader.cancel_wrapper(coro, Download(url=prepared.url), ctx=ctx)
|
|
|
|
assert isinstance(result, DownloadResult)
|
|
assert result.success is False
|
|
assert 'cancel' in (result.error or '').lower()
|
|
|
|
|
|
def make_process_runner_recorder(singlefile: bytes, compressed: bytes = None):
|
|
"""Stub Downloader.process_runner: record the call, write the singlefile to the output file
|
|
(the last argument of the command, like single-file does), return a successful CommandResult.
|
|
|
|
Capture runs (no --compress-content) write `singlefile`; conversion runs write `compressed`
|
|
(or nothing, when compressed is None, to simulate a failed conversion)."""
|
|
from wrolpi.cmd import CommandResult
|
|
calls = []
|
|
|
|
async def process_runner(download, cmd, cwd, **kwargs):
|
|
calls.append(dict(cmd=cmd, cwd=cwd, **kwargs))
|
|
if '--compress-content' not in cmd:
|
|
pathlib.Path(cmd[-1]).write_bytes(singlefile)
|
|
elif compressed is not None:
|
|
pathlib.Path(cmd[-1]).write_bytes(compressed)
|
|
return CommandResult(return_code=0, cancelled=False, stdout=b'', stderr=b'', elapsed=1)
|
|
|
|
return process_runner, calls
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_do_singlefile_uses_deno(test_session, test_directory, monkeypatch, compressed_singlefile_factory):
|
|
"""single-file is run under Deno (it is designed for it; under node it cannot reach the
|
|
browser CDP on IPv6-enabled hosts and never exits). The page is captured uncompressed,
|
|
then converted to a compressed (SingleFileZ) file with a second local single-file run."""
|
|
import modules.archive
|
|
singlefile = b'<html><!--\n Page saved with SingleFile \n url: https://example.com \n--></html>'
|
|
compressed = compressed_singlefile_factory()
|
|
process_runner, calls = make_process_runner_recorder(singlefile, compressed)
|
|
monkeypatch.setattr(modules.archive, 'DENO_BIN', '/usr/local/bin/deno')
|
|
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_DENO_SCRIPT',
|
|
'/usr/local/lib/node_modules/single-file-cli/single-file')
|
|
monkeypatch.setattr(modules.archive, 'CHROMIUM', '/usr/bin/chromium')
|
|
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
|
|
|
|
prepared = PreparedArchive(url='https://example.com', destination=None,
|
|
settings={'compress_singlefile': True})
|
|
result, page_html = await archive_downloader.do_singlefile(prepared, make_test_ctx())
|
|
|
|
# The compressed file is stored; the uncompressed page is used for readability/screenshot.
|
|
assert result == compressed
|
|
assert page_html == singlefile
|
|
|
|
capture, convert = calls
|
|
cmd = capture['cmd']
|
|
assert cmd[2:9] == ('/usr/local/bin/deno', 'run', '--node-modules-dir=manual',
|
|
'--allow-read', '--allow-write', '--allow-net', '--allow-env')
|
|
assert '/usr/local/lib/node_modules/single-file-cli/single-file' in cmd
|
|
# The capture is always uncompressed.
|
|
assert '--compress-content' not in cmd
|
|
# The singlefile is written to a file; Deno >= 2.3 truncates large --dump-content output.
|
|
assert '--dump-content' not in cmd
|
|
assert cmd[-2] == 'https://example.com'
|
|
assert cmd[-1].endswith('page.html')
|
|
# The browser single-file leaves behind is reaped with the process group.
|
|
assert capture['start_new_session'] is True
|
|
# The browser runs via the stderr-discarding wrapper (Deno >= 2.3 kills it otherwise).
|
|
wrapper = cmd[cmd.index('--browser-executable-path') + 1]
|
|
assert wrapper.endswith('scripts/singlefile_browser_wrapper.sh')
|
|
assert capture['env']['WROLPI_BROWSER'] == '/usr/bin/chromium'
|
|
|
|
# The conversion is a local (file://) operation; the page is not fetched again.
|
|
cmd = convert['cmd']
|
|
assert '--compress-content' in cmd
|
|
assert cmd[-2].startswith('file://') and cmd[-2].endswith('page.html')
|
|
assert cmd[-1].endswith('compressed.html')
|
|
|
|
# Without the compress setting there is one run, and the page is stored as-is.
|
|
calls.clear()
|
|
prepared = PreparedArchive(url='https://example.com', destination=None, settings={})
|
|
result, page_html = await archive_downloader.do_singlefile(prepared, make_test_ctx())
|
|
assert result == page_html == singlefile
|
|
assert len(calls) == 1
|
|
|
|
# When the compression run fails, the uncompressed singlefile is stored instead.
|
|
process_runner, calls = make_process_runner_recorder(singlefile, compressed=None) # Conversion writes nothing.
|
|
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
|
|
prepared = PreparedArchive(url='https://example.com', destination=None,
|
|
settings={'compress_singlefile': True})
|
|
result, page_html = await archive_downloader.do_singlefile(prepared, make_test_ctx())
|
|
assert len(calls) == 2 # The conversion was attempted.
|
|
assert result == page_html == singlefile
|
|
|
|
# A page that has not finished archiving in ten minutes is hung; the timeout reaps it.
|
|
assert archive_downloader.timeout == 10 * 60
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_do_singlefile_node_fallback(test_session, test_directory, monkeypatch):
|
|
"""Without Deno, single-file runs under node with the IPv4-fallback fix."""
|
|
import modules.archive
|
|
singlefile = b'<html><!--\n Page saved with SingleFile \n url: https://example.com \n--></html>'
|
|
process_runner, calls = make_process_runner_recorder(singlefile)
|
|
monkeypatch.setattr(modules.archive, 'DENO_BIN', None)
|
|
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_DENO_SCRIPT', None)
|
|
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_BIN', '/usr/local/bin/single-file')
|
|
monkeypatch.setattr(modules.archive, 'CHROMIUM', '/usr/bin/chromium')
|
|
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
|
|
|
|
prepared = PreparedArchive(url='https://example.com', destination=None, settings={})
|
|
await archive_downloader.do_singlefile(prepared, make_test_ctx())
|
|
|
|
call, = calls
|
|
assert call['cmd'][2] == '/usr/local/bin/single-file'
|
|
# node 18 has no IPv4 fallback when "localhost" resolves to ::1 first.
|
|
assert call['env']['NODE_OPTIONS'] == '--enable-network-family-autoselection'
|
|
assert call['start_new_session'] is True
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_finalize_adds_tags(async_client, test_session, test_directory, tag_factory):
|
|
"""finalize_download applies download.tag_names to the registered Archive.
|
|
|
|
Uses async_client because tag_factory's upsert_tag activates a Sanic switch
|
|
(shared_ctx.switches_lock). That's an orthogonal coupling — the rest of
|
|
finalize_download is Sanic-free.
|
|
"""
|
|
singlefile, readability, screenshot = make_fake_archive_result()
|
|
destination = test_directory / 'archive/example.com'
|
|
destination.mkdir(parents=True, exist_ok=True)
|
|
written = write_archive_files('https://example.com/x', singlefile, readability, screenshot,
|
|
destination=destination)
|
|
executed = ExecutedArchive(written=written)
|
|
|
|
tag = await tag_factory()
|
|
download = Download(
|
|
url='https://example.com/x',
|
|
downloader='archive',
|
|
tag_names=[tag.name],
|
|
)
|
|
|
|
result = archive_downloader.finalize_download(test_session, download, executed)
|
|
test_session.commit()
|
|
|
|
assert result.success is True
|
|
archives = test_session.query(Archive).all()
|
|
assert len(archives) == 1
|
|
archive_tag_names = [tf.tag.name for tf in archives[0].file_group.tag_files]
|
|
assert tag.name in archive_tag_names
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
@pytest.mark.parametrize('configured, expected', [
|
|
# A config can outlive the OS it was written on: Raspberry Pi OS names Chromium "chromium-browser".
|
|
('/nonexistent/chromium-browser', '/usr/bin/chromium'),
|
|
(None, '/usr/bin/chromium'),
|
|
('/bin/sh', '/bin/sh'),
|
|
])
|
|
async def test_do_singlefile_configured_browser(test_session, test_directory, monkeypatch, configured, expected):
|
|
"""The configured browser is used when it is installed; otherwise one is detected."""
|
|
from types import SimpleNamespace
|
|
import modules.archive
|
|
singlefile = b'<html><!--\n Page saved with SingleFile \n url: https://example.com \n--></html>'
|
|
process_runner, calls = make_process_runner_recorder(singlefile)
|
|
monkeypatch.setattr(modules.archive, 'DENO_BIN', '/usr/local/bin/deno')
|
|
monkeypatch.setattr(modules.archive, 'SINGLE_FILE_DENO_SCRIPT',
|
|
'/usr/local/lib/node_modules/single-file-cli/single-file')
|
|
monkeypatch.setattr(modules.archive, 'CHROMIUM', '/usr/bin/chromium')
|
|
monkeypatch.setattr(archive_downloader, 'process_runner', process_runner)
|
|
monkeypatch.setattr(modules.archive.lib, 'get_archive_downloader_config',
|
|
lambda: SimpleNamespace(browser_executable=configured, browser_args=None, user_agent=None))
|
|
|
|
prepared = PreparedArchive(url='https://example.com', destination=None, settings={})
|
|
await archive_downloader.do_singlefile(prepared, make_test_ctx())
|
|
|
|
capture, = calls
|
|
assert capture['env']['WROLPI_BROWSER'] == expected
|
|
|
|
|
|
@pytest.mark.asyncio
|
|
async def test_download_records_sha256(async_client, test_session, test_directory, monkeypatch):
|
|
"""An archive download records the sha256 of each file in its group in the readability json's wrolpi section."""
|
|
singlefile, readability, screenshot = make_fake_archive_result()
|
|
|
|
async def mock_request_archive(url, compress=False):
|
|
return singlefile, readability, screenshot
|
|
|
|
monkeypatch.setattr(modules.archive, 'request_archive', mock_request_archive)
|
|
|
|
download = Download(url='https://example.com/x', downloader='archive',
|
|
settings={'parent_download_url': 'https://example.com/feed'})
|
|
prepared = archive_downloader.prepare_download(test_session, download)
|
|
executed = await archive_downloader.execute_download(prepared, make_test_ctx())
|
|
result = archive_downloader.finalize_download(test_session, download, executed)
|
|
test_session.commit()
|
|
assert result.success is True
|
|
|
|
archive = test_session.query(Archive).one()
|
|
readability_json_path = archive.file_group.readability_json_path
|
|
wrolpi = json.loads(readability_json_path.read_text())['wrolpi']
|
|
assert wrolpi['parent_download_url'] == 'https://example.com/feed'
|
|
stem = archive.singlefile_path.name[:-len('.html')]
|
|
expected = {path.name[len(stem):]: hashlib.sha256(path.read_bytes()).hexdigest()
|
|
for path in archive.file_group.my_paths() if path != readability_json_path}
|
|
assert set(expected) == {'.html', '.readability.html', '.readability.txt', '.png'}
|
|
assert wrolpi['sha256'] == expected
|