mirror of
https://github.com/lrnselfreliance/wrolpi.git
synced 2026-10-26 19:01:39 -04:00
1acb148c90
* Singlefile runs under Deno 2.8.3; browser leftovers are reaped.
single-file-cli is designed for Deno. Under node 18 (Debian 12/RPi OS) it
cannot reach the browser CDP on IPv6-enabled hosts ("localhost" resolves to
::1 first with no IPv4 fallback) and it never exits after a successful save,
blocking the download worker forever.
The native path and the Archive docker container now run single-file with
"deno run". Deno is pinned at v2.8.3 everywhere because yt-dlp 2026.06.09
requires at least 2.3.0 (our node is Debian's v18, below yt-dlp's v22
minimum). Deno >= 2.3 kills the browser single-file spawns (piped stderr
that single-file never reads), so the browser is run through
singlefile_browser_wrapper.sh which discards its stderr.
Commands are run in their own process group which is killed when they
finish, reaping the browser single-file leaves behind. The Archive
downloader now has a ten minute timeout. Install scripts install Deno
(previously only upgrade.sh and the image builds did).
* Singlefile writes to a file instead of dumping to stdout.
Deno >= 2.3 truncates large --dump-content output at exit; compressed
(SingleFileZ) singlefiles of large pages lost their ZIP tail, failed the
compression check in write_archive_files, and were destroyed by the HTML
prettifier (blank white archives). single-file now writes its result to a
temporary file which WROLPi reads back.
As insurance, a singlefile marked compressed (<html data-sfz) whose ZIP is
unreadable is now stored verbatim instead of being prettified.
* Compressed Archive screenshots and readability images are no longer broken.
The index.html inside a compressed (SingleFileZ) singlefile references its
resources by relative ZIP paths. Screenshots are now taken from the fully
extracted ZIP so those references resolve. Readability images are inlined
as data URIs read from the ZIP (readability-extractor absolutizes relative
srcs against the page URL, so both forms are matched).
* Compressed Archives are created by converting the uncompressed capture.
The page is always captured uncompressed; readability, screenshot, and
title come from that canonical page (its images are inline data URIs, so
no ZIP extraction or image inlining is needed). When compress_singlefile
is set, a second single-file run converts the captured file into a
compressed (SingleFileZ) self-extracting file. The conversion is a local
file:// operation - the page is not fetched again, and single-file
preserves the original URL header when re-saving its own pages. If the
conversion fails, the uncompressed singlefile is stored instead.
* Consolidating duplicate compressed-singlefile tests.
The write_archive_files tests (verbatim write, title fallback, broken-ZIP
guard, readability inlining) are combined into one test; the helper probes
and the setting-plumbing tests are each merged into one. The do_singlefile
compression-failure scenario and the timeout constant are folded into the
main do_singlefile test, which shared their setup.
* Addressing review comments: removeprefix, PermissionError when killing.
* Bumping version to 0.24.1-beta.
* Install scripts assert the deno.zip hash to avoid malicious or invalid files.
The hashes are the official sha256sum assets of the Deno v2.8.3 release,
per architecture.
* Archive Dockerfile asserts the deno.zip hash to avoid malicious or invalid files.
---------
Co-authored-by: Claude Fable 5 <noreply@anthropic.com>
454 lines
20 KiB
Python
454 lines
20 KiB
Python
import asyncio
|
|
import json
|
|
import os
|
|
import pathlib
|
|
import tempfile
|
|
from abc import ABC
|
|
from dataclasses import dataclass
|
|
from typing import Callable, List, Optional, Tuple, Iterable
|
|
|
|
from sqlalchemy import not_
|
|
from sqlalchemy.orm import Session
|
|
|
|
from wrolpi.cmd import SINGLE_FILE_BIN, CHROMIUM, FIREFOX, DENO_BIN, SINGLE_FILE_DENO_SCRIPT
|
|
from wrolpi.collections import Collection
|
|
from wrolpi.common import logger, register_modeler, register_refresh_cleanup, limit_concurrent, split_lines_by_length, \
|
|
slow_logger, get_title_from_html, TRACE_LEVEL
|
|
from wrolpi.db import get_db_session
|
|
from wrolpi.downloader import Downloader, Download, DownloadContext, DownloadResult
|
|
from wrolpi.errors import UnrecoverableDownloadError
|
|
from wrolpi.files.models import FileGroup
|
|
from wrolpi.vars import PYTEST, DOCKERIZED, DOWNLOAD_USER_AGENT, PROJECT_DIR
|
|
from . import lib
|
|
from .api import archive_bp # noqa
|
|
from .errors import InvalidArchive
|
|
from .lib import is_singlefile_file, request_archive, SINGLEFILE_HEADER, get_url_from_singlefile, \
|
|
SinglefileArtifacts, WrittenArchiveFiles, extract_singlefile_artifacts, register_archive, write_archive_files
|
|
from .models import Archive
|
|
|
|
PRETTY_NAME = 'Archive'
|
|
|
|
logger = logger.getChild(__name__)
|
|
|
|
__all__ = ['ArchiveDownloader', 'archive_downloader', 'model_archive']
|
|
|
|
|
|
@dataclass
|
|
class PreparedArchive:
|
|
"""Plan produced by ArchiveDownloader.prepare_download.
|
|
|
|
Holds everything execute_download needs (URL, destination, settings) so the I/O
|
|
phase has no DB session.
|
|
"""
|
|
url: str
|
|
destination: Optional[pathlib.Path]
|
|
settings: dict
|
|
|
|
|
|
@dataclass
|
|
class ExecutedArchive:
|
|
"""Output of ArchiveDownloader.execute_download: files are on disk and ready to register."""
|
|
written: WrittenArchiveFiles
|
|
|
|
|
|
class ArchiveDownloader(Downloader, ABC):
|
|
name = 'archive'
|
|
pretty_name = 'Archive'
|
|
# A page that has not finished archiving after this long is hung; the config-level
|
|
# download_timeout still takes precedence when set.
|
|
timeout = 10 * 60
|
|
|
|
def __repr__(self):
|
|
return f'<ArchiveDownloader>'
|
|
|
|
def already_downloaded(self, session: Session, *urls: List[str]) -> List:
|
|
file_groups = list(session.query(FileGroup).filter(FileGroup.url.in_(urls), FileGroup.model == 'archive'))
|
|
return file_groups
|
|
|
|
def prepare_download(self, session: Session, download: Download) -> PreparedArchive:
|
|
"""Resolve destination directory. Caller commits (or rolls back on raise)."""
|
|
if (download.attempts or 0) > 3:
|
|
raise UnrecoverableDownloadError(f'Max download attempts reached for {download.url}')
|
|
|
|
settings = download.settings or dict()
|
|
# Destination from settings (e.g. set by the RSS sub-downloader) wins over collection lookup.
|
|
destination = pathlib.Path(settings['destination']) if settings.get('destination') else None
|
|
|
|
if not destination:
|
|
collection = lib.get_or_create_domain_collection(session, download.url)
|
|
if collection:
|
|
destination = collection.get_or_set_directory(session)
|
|
|
|
return PreparedArchive(url=download.url, destination=destination, settings=settings)
|
|
|
|
async def execute_download(self, prepared: PreparedArchive, ctx: DownloadContext,
|
|
download: Download = None) -> ExecutedArchive:
|
|
"""Run the archive container or local singlefile + readability/screenshot extraction,
|
|
then write the artifacts to disk. No DB."""
|
|
compress = bool(prepared.settings.get('compress_singlefile'))
|
|
if DOCKERIZED or PYTEST:
|
|
singlefile, readability, screenshot = await request_archive(prepared.url, compress=compress)
|
|
artifacts = SinglefileArtifacts(url=prepared.url, singlefile=singlefile,
|
|
readability=readability, screenshot=screenshot)
|
|
else:
|
|
singlefile_bytes, page_html = await self.do_singlefile(prepared, ctx, download=download)
|
|
# Readability/screenshot/title come from the uncompressed page; only the stored
|
|
# singlefile may be compressed.
|
|
artifacts = await extract_singlefile_artifacts(page_html)
|
|
artifacts.singlefile = singlefile_bytes
|
|
|
|
written = write_archive_files(artifacts.url, artifacts.singlefile, artifacts.readability,
|
|
artifacts.screenshot, destination=prepared.destination)
|
|
return ExecutedArchive(written=written)
|
|
|
|
def finalize_download(self, session: Session, download: Download,
|
|
executed: ExecutedArchive) -> DownloadResult:
|
|
"""Register the Archive record, apply tags, persist parent_download_url. Caller commits."""
|
|
archive = register_archive(session, executed.written)
|
|
|
|
if tag_names := download.tag_names:
|
|
for name in tag_names:
|
|
archive.add_tag(session, name)
|
|
|
|
settings = download.settings or dict()
|
|
if parent_download_url := settings.get('parent_download_url'):
|
|
try:
|
|
archive.file_group.update_wrolpi_json({'parent_download_url': parent_download_url})
|
|
except ValueError as e:
|
|
logger.warning(f'Failed to update wrolpi json for {archive}: {e}')
|
|
|
|
logger.info(f'Successfully downloaded Archive {download.url} {archive}')
|
|
return DownloadResult(success=True, location=archive.location)
|
|
|
|
async def do_singlefile(self, prepared: PreparedArchive, ctx: DownloadContext,
|
|
download: Download = None) -> Tuple[bytes, bytes]:
|
|
"""Create a Singlefile from the archive's URL.
|
|
|
|
Returns (singlefile, page_html): the singlefile to store, and the uncompressed page used
|
|
for readability/screenshot/title. The page is always captured uncompressed; when
|
|
compress_singlefile is set, a second single-file run converts the captured file into a
|
|
compressed (SingleFileZ) self-extracting file (single-file preserves the original URL
|
|
header when re-saving its own pages). If the conversion fails, the uncompressed
|
|
singlefile is stored instead.
|
|
|
|
download is the original Download row when called via the manager dispatch; tests
|
|
may omit it and a stub is built so process_runner has a url for log messages.
|
|
"""
|
|
# Get browser settings from config
|
|
config = lib.get_archive_downloader_config()
|
|
|
|
# Determine browser executable
|
|
browser_executable = config.browser_executable
|
|
if browser_executable:
|
|
# Use configured browser path
|
|
browser = browser_executable
|
|
else:
|
|
# Auto-detect: prefer Chromium, fallback to Firefox
|
|
browser = CHROMIUM or FIREFOX
|
|
if not browser:
|
|
raise RuntimeError('No browser found. Install Chromium or Firefox, or configure browser_executable.')
|
|
|
|
# Get browser args and user agent from config
|
|
browser_args = config.browser_args or '["--no-sandbox"]'
|
|
user_agent = config.user_agent or DOWNLOAD_USER_AGENT
|
|
|
|
env = None
|
|
browser_wrapper = PROJECT_DIR / 'scripts/singlefile_browser_wrapper.sh'
|
|
if DENO_BIN and SINGLE_FILE_DENO_SCRIPT and browser_wrapper.is_file():
|
|
# single-file-cli is designed for Deno. Under node 18 (Debian 12/RPi OS) it cannot
|
|
# reach the browser CDP on IPv6-enabled hosts ("localhost" resolves to ::1 first with
|
|
# no IPv4 fallback) and it never exits after a successful save.
|
|
runtime = (str(DENO_BIN), 'run', '--node-modules-dir=manual',
|
|
'--allow-read', '--allow-write', '--allow-net', '--allow-env', '--allow-run',
|
|
str(SINGLE_FILE_DENO_SCRIPT))
|
|
# single-file pipes the browser's stderr but never reads it; under Deno >= 2.3 that
|
|
# kills the browser during startup. The wrapper discards the browser's stderr.
|
|
env = {**os.environ, 'WROLPI_BROWSER': str(browser)}
|
|
browser = browser_wrapper
|
|
else:
|
|
logger.warning('deno not found, running single-file under node. Upgrade this WROLPi to install deno.')
|
|
runtime = (str(SINGLE_FILE_BIN),)
|
|
# node 18 has no IPv4 fallback when "localhost" resolves to ::1 first; without this
|
|
# single-file cannot reach the browser CDP and fails with "fetch failed".
|
|
env = {**os.environ, 'NODE_OPTIONS': '--enable-network-family-autoselection'}
|
|
|
|
cwd = pathlib.Path('/home/wrolpi')
|
|
cwd = cwd if cwd.is_dir() else None
|
|
download = download or Download(url=prepared.url)
|
|
|
|
async def run_single_file(input_url: str, output_path: pathlib.Path, compress: bool):
|
|
cmd = ('/usr/bin/nice', '-n15', # Nice above map importing, but very low priority.
|
|
*runtime,
|
|
'--browser-executable-path', str(browser),
|
|
'--browser-args', browser_args,
|
|
'--user-agent', user_agent,
|
|
'--load-deferred-images-dispatch-scroll-event',
|
|
# Create a compressed (SingleFileZ) self-extracting HTML file.
|
|
*(('--compress-content',) if compress else ()),
|
|
input_url,
|
|
str(output_path),
|
|
)
|
|
# cancel_wrapper reads cancellation from ctx; the Download is forwarded so
|
|
# the real download id is used for unkill cleanup and log attribution.
|
|
# start_new_session reaps the browser single-file leaves behind (it does not close it).
|
|
result = await self.process_runner(download, cmd, cwd, env=env, start_new_session=True, ctx=ctx)
|
|
stderr = result.stderr.decode()
|
|
log_output = stderr or result.stdout.decode(errors='replace') or 'No stderr or stdout!'
|
|
output = output_path.read_bytes() if output_path.is_file() else b''
|
|
return result.return_code, output, log_output
|
|
|
|
# single-file writes the result to a file; Deno >= 2.3 truncates large --dump-content
|
|
# output at exit, which corrupted compressed (ZIP) singlefiles.
|
|
with tempfile.TemporaryDirectory(prefix='wrolpi-singlefile-') as tmp_dir:
|
|
page_path = pathlib.Path(tmp_dir) / 'page.html'
|
|
return_code, page_html, log_output = await run_single_file(prepared.url, page_path, compress=False)
|
|
|
|
if return_code != 0:
|
|
e = ChildProcessError(log_output[:1000])
|
|
raise RuntimeError(f'singlefile exited with {return_code}') from e
|
|
|
|
if not page_html:
|
|
e = ChildProcessError(log_output[:1000])
|
|
raise RuntimeError(f'Singlefile created was empty: {prepared.url}') from e
|
|
|
|
if SINGLEFILE_HEADER.encode() not in page_html:
|
|
e = ChildProcessError(log_output[:1000])
|
|
raise RuntimeError(f'Singlefile created was invalid: {prepared.url}') from e
|
|
|
|
singlefile = page_html
|
|
if prepared.settings.get('compress_singlefile'):
|
|
# Convert the captured page into a compressed (SingleFileZ) self-extracting file.
|
|
# This is a local (file://) operation; the page is not fetched again.
|
|
compressed_path = pathlib.Path(tmp_dir) / 'compressed.html'
|
|
return_code, compressed, log_output = await run_single_file(
|
|
f'file://{page_path}', compressed_path, compress=True)
|
|
if return_code == 0 and lib.is_compressed_singlefile(compressed):
|
|
singlefile = compressed
|
|
else:
|
|
logger.error(f'Failed to compress singlefile ({return_code=}), '
|
|
f'storing it uncompressed: {prepared.url}\n{log_output[:1000]}')
|
|
|
|
return singlefile, page_html
|
|
|
|
|
|
archive_downloader = ArchiveDownloader()
|
|
|
|
|
|
def model_archive(session: Session, file_group: FileGroup) -> Archive:
|
|
"""
|
|
Models an Archive from a given FileGroup.
|
|
|
|
This function takes in a FileGroup and attempts to create an Archive object.
|
|
It does this by checking for the presence of HTML files within the FileGroup,
|
|
determining if any of these are SingleFiles, and then extracting relevant data
|
|
(title, contents) from either the JSON or text readability files.
|
|
|
|
Args:
|
|
session: Database session.
|
|
file_group: The FileGroup to model as an Archive.
|
|
|
|
Returns:
|
|
An Archive object representing the modeled archive.
|
|
|
|
Raises:
|
|
InvalidArchive: If no HTML SingleFile is found in the FileGroup, or if
|
|
any other error occurs during modeling.
|
|
"""
|
|
file_group_id = file_group.id
|
|
if not file_group_id:
|
|
session.flush([file_group])
|
|
file_group_id = file_group.id
|
|
|
|
# All Archives have an HTML Singlefile. Singlefile may be text HTML, or bytes HTML.
|
|
html_paths = file_group.my_html_paths()
|
|
if not html_paths:
|
|
logger.error('Query returned a group without an HTML file!')
|
|
raise InvalidArchive('FileGroup does not contain any html files')
|
|
|
|
# All Archives have a Singlefile.
|
|
singlefile_path = None
|
|
for file in html_paths:
|
|
try:
|
|
if is_singlefile_file(file):
|
|
singlefile_path = file
|
|
break
|
|
except Exception as e:
|
|
if PYTEST:
|
|
raise
|
|
logger.debug(f'Cannot check is_singlefile_file of {repr(file)}', exc_info=e)
|
|
|
|
if singlefile_path is None:
|
|
logger.debug(f'No Archive singlefile found in {file_group}')
|
|
raise InvalidArchive('FileGroup does not contain a singlefile')
|
|
|
|
readability_json_path = None
|
|
json_files = file_group.my_json_files()
|
|
for json_file in json_files:
|
|
path = json_file['path']
|
|
if path.name.endswith('.readability.json'):
|
|
readability_json_path = path
|
|
break
|
|
|
|
readability_txt_path = None
|
|
text_files = file_group.my_text_files()
|
|
for text_file in text_files:
|
|
path = text_file['path']
|
|
if path.name.endswith('.readability.txt'):
|
|
readability_txt_path = path
|
|
break
|
|
|
|
try:
|
|
file_group.model = 'archive'
|
|
file_group.primary_path = singlefile_path
|
|
|
|
title = None
|
|
if readability_json_path:
|
|
title = get_title(readability_json_path)
|
|
if not title:
|
|
title = get_title_from_html(lib.read_singlefile_html(singlefile_path))
|
|
|
|
contents = None
|
|
if readability_txt_path:
|
|
contents = get_article(readability_txt_path)
|
|
|
|
# Check if an Archive already exists for this FileGroup
|
|
archive = session.query(Archive).filter_by(file_group_id=file_group_id).one_or_none()
|
|
if not archive:
|
|
# Create new Archive if it doesn't exist
|
|
archive = Archive(file_group_id=file_group_id, file_group=file_group)
|
|
|
|
# Set collection_id BEFORE adding to session to avoid autoflush constraint violation
|
|
from modules.archive.lib import get_or_create_domain_collection
|
|
|
|
if file_group.url:
|
|
# URL is already set on the FileGroup
|
|
collection = get_or_create_domain_collection(session, file_group.url)
|
|
archive.collection_id = collection.id if collection else None
|
|
else:
|
|
# No URL - try to extract from singlefile
|
|
try:
|
|
url = get_url_from_singlefile(singlefile_path.read_bytes())
|
|
file_group.url = url
|
|
collection = get_or_create_domain_collection(session, url)
|
|
archive.collection_id = collection.id if collection else None
|
|
except (RuntimeError, ValueError) as e:
|
|
# Could not extract URL from singlefile - archive will have no collection
|
|
logger.debug(f'Could not extract URL from singlefile: {e}')
|
|
archive.collection_id = None
|
|
|
|
session.add(archive)
|
|
|
|
archive.validate()
|
|
archive.flush()
|
|
|
|
# Note: We intentionally do NOT clear collection.directory when archive is outside it.
|
|
# Archives are associated with domain collections based on URL, not file location.
|
|
# The collection's directory should remain intact for other archives.
|
|
|
|
file_group.title = file_group.a_text = title or archive.file_group.title
|
|
file_group.d_text = contents
|
|
archive.rebuild_data()
|
|
|
|
return archive
|
|
except Exception as e:
|
|
logger.error('Failed to model Archive', exc_info=e)
|
|
raise InvalidArchive(f'Failed to model Archive {file_group}') from e
|
|
|
|
|
|
@register_modeler
|
|
async def archive_modeler(progress_callback: Callable[[int], None] = None):
|
|
"""Searches DB for FileGroups that contain an HTML file. If the HTML file is a SingleFile, we model it as an
|
|
Archive."""
|
|
invalid_archives = set()
|
|
total_processed = 0
|
|
|
|
while True:
|
|
with get_db_session(commit=True) as session:
|
|
results = session.query(FileGroup, Archive) \
|
|
.filter(
|
|
# Get all groups that contain an HTML file that have not been indexed.
|
|
FileGroup.indexed != True,
|
|
FileGroup.mimetype == 'text/html',
|
|
).filter(not_(FileGroup.id.in_(list(invalid_archives)))) \
|
|
.outerjoin(Archive, Archive.file_group_id == FileGroup.id) \
|
|
.limit(20)
|
|
results: Iterable[Tuple[FileGroup, Archive]]
|
|
results_list = list(results)
|
|
|
|
processed = 0
|
|
for processed, (file_group, archive) in enumerate(results_list):
|
|
with slow_logger(1, f'Modeling archive took %(elapsed)s seconds: {file_group}',
|
|
logger__=logger):
|
|
if archive:
|
|
try:
|
|
archive_id = archive.id
|
|
archive.validate()
|
|
# Successfully validated, mark as indexed
|
|
file_group.indexed = True
|
|
except Exception as e:
|
|
logger.error(f'Unable to validate Archive {archive_id}')
|
|
# Don't mark as indexed - will retry later
|
|
if PYTEST:
|
|
raise
|
|
else:
|
|
try:
|
|
model_archive(session, file_group)
|
|
# Successfully modeled, mark as indexed
|
|
file_group.indexed = True
|
|
except InvalidArchive:
|
|
# It was not a real Archive. Many HTML files will not be an Archive.
|
|
file_group.indexed = False
|
|
invalid_archives.add(file_group.id)
|
|
except Exception as e:
|
|
# Some other error occurred during modeling - don't mark as indexed so we can retry
|
|
logger.error(f'Failed to model Archive for FileGroup {file_group.id}: {e}')
|
|
if PYTEST:
|
|
raise
|
|
|
|
session.commit()
|
|
|
|
# Report batch progress
|
|
batch_count = len(results_list)
|
|
total_processed += batch_count
|
|
if progress_callback and batch_count > 0:
|
|
progress_callback(total_processed)
|
|
|
|
if processed < 19:
|
|
# Did not reach limit (enumerate is 0-indexed, so 19 = 20 items), do not query again.
|
|
if __debug__ and logger.isEnabledFor(TRACE_LEVEL):
|
|
logger.trace(f'archive_modeler: DONE (processed {processed + 1} files)')
|
|
break
|
|
|
|
# Sleep to catch cancel.
|
|
await asyncio.sleep(0)
|
|
|
|
|
|
@register_refresh_cleanup
|
|
@limit_concurrent(1)
|
|
def archive_cleanup():
|
|
with get_db_session(commit=True) as session:
|
|
# Remove any domain Collections without any Archives.
|
|
# Get all collection_ids that have archives
|
|
collection_ids = [i[0] for i in session.execute('SELECT DISTINCT collection_id FROM archive') if i[0]]
|
|
# Find domain collections that have no archives
|
|
for collection in session.query(Collection).filter_by(kind='domain'):
|
|
if collection.id not in collection_ids:
|
|
logger.info(f'Deleting empty domain collection: {collection.name}')
|
|
session.delete(collection)
|
|
|
|
|
|
def get_title(path):
|
|
with path.open('rt') as fh:
|
|
try:
|
|
return json.load(fh)['title']
|
|
except Exception:
|
|
if not PYTEST:
|
|
logger.warning(f'Archive readability json file exists, but cannot get title. {path}')
|
|
return None
|
|
|
|
|
|
def get_article(path):
|
|
if path.is_file():
|
|
text = path.read_text()
|
|
text = split_lines_by_length(text)
|
|
return text
|