Added rngit media conversion max input size cap, improved logging and error handling

This commit is contained in:
Mark Qvist 2026-11-19 17:50:22 +01:00
parent cff4d03fdb
commit 840eb3699b
No known key found for this signature in database
2 changed files with 152 additions and 13 deletions

View file

@ -31,6 +31,7 @@
import os
import time
import shutil
import selectors
import subprocess
import tempfile
import RNS
@ -50,6 +51,11 @@ BACKENDS = ( ("magick", ["magick", "-", "webp:-"]),
("ffmpeg", ["ffmpeg", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]),
("avconv", ["avconv", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]) )
SUPPORTED_IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"]
CONV_SIZE_MAX = 32*1024*1024
SNIFF_BYTES = 64
RELAY_CHUNK = 256*1024
_ENV_BACKEND = os.environ.get("RNGIT_MEDIA_BACKEND")
_winner = None
_no_backend_logged = False
@ -106,6 +112,24 @@ def _webp_info(data):
if width > 0 and height > 0: return (width, height)
return None
def _is_bmp_header(data):
if len(data) < 18 or data[:2] != b"BM": return False
try: dib_size = int.from_bytes(data[14:18], "little")
except Exception: return False
return dib_size in (12, 40, 52, 56, 64, 108, 124)
_IMAGE_SIGNATURES = ( ("png", lambda data: len(data) >= 8 and data[:8] == b"\x89PNG\r\n\x1a\n"),
("jpeg", lambda data: len(data) >= 3 and data[:3] == b"\xff\xd8\xff"),
("gif", lambda data: len(data) >= 6 and data[:6] in (b"GIF87a", b"GIF89a")),
("tiff", lambda data: len(data) >= 4 and data[:4] in (b"II*\x00", b"MM\x00*")),
("webp", lambda data: _webp_info(data) is not None),
("bmp", lambda data: _is_bmp_header(data)) )
def _classify_image(data):
for format_name, signature_matches in _IMAGE_SIGNATURES:
if signature_matches(data): return format_name
return None
def _valid_webp(path):
try:
with open(path, "rb") as fh: return _webp_info(fh.read(30)) is not None
@ -115,7 +139,7 @@ def _unlink_output(path):
try: os.unlink(path)
except Exception: pass
def _configured_backend(quality=None, max_dimension=None):
def _configured_backend(quality=None, max_dimension=None, input_spec=None):
global _no_backend_logged
backend = _selected_backend()
if backend is None:
@ -139,10 +163,15 @@ def _configured_backend(quality=None, max_dimension=None):
encoder_argv = list(encoder_argv)
if backend_name in ("magick", "convert", "gm"):
if input_spec is not None:
try: input_index = encoder_argv.index("-")
except ValueError: input_index = None
if input_index is not None: encoder_argv[input_index] = input_spec
options = []
if quality_arg is not None: options += ["-quality", str(quality_arg)]
if dimension_arg is not None: options += ["-resize", f"{dimension_arg}x{dimension_arg}>"]
encoder_argv = encoder_argv[:-1] + options + encoder_argv[-1:]
elif backend_name in ("ffmpeg", "avconv"):
options = []
if quality_arg is not None: options += ["-quality", str(quality_arg)]
@ -163,13 +192,13 @@ def _await(backend_name, encoder_proc, input_proc=None, timeout=None):
except subprocess.TimeoutExpired:
_terminate(input_proc)
_terminate(encoder_proc)
RNS.log(f"Media conversion via {backend_name} timed out after {timeout} seconds", RNS.LOG_WARNING)
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
return False
if encoder_proc.returncode != 0:
detail = ""
encoder_tail = _stderr_tail(encoder_proc)
input_tail = _stderr_tail(input_proc) if input_proc is not None else ""
input_tail = _stderr_tail(input_proc) if input_proc is not None else ""
if encoder_tail: detail += f" Encoder: {encoder_tail}"
if input_tail: detail += f" Input: {input_tail}"
RNS.log(f"Media conversion via {backend_name} failed.{detail}", RNS.LOG_WARNING)
@ -177,21 +206,107 @@ def _await(backend_name, encoder_proc, input_proc=None, timeout=None):
return True
def _await_readable(fd, timeout, selector):
try: return bool(selector.select(max(0.0, timeout)))
except Exception: return False
def _read_prefix(fd, amount, deadline):
data = bytearray()
selector = selectors.DefaultSelector()
try:
selector.register(fd, selectors.EVENT_READ)
while len(data) < amount:
if not _await_readable(fd, deadline - time.time(), selector): return None
chunk = os.read(fd, amount - len(data))
if not chunk: break
data += chunk
return bytes(data)
finally: selector.close()
def _pump(fd, encoder_stdin, prefix, deadline):
written = len(prefix)
if written > CONV_SIZE_MAX: return None
encoder_stdin.write(prefix)
selector = selectors.DefaultSelector()
try:
selector.register(fd, selectors.EVENT_READ)
while True:
if not _await_readable(fd, deadline - time.time(), selector):
raise subprocess.TimeoutExpired("input relay", max(0.0, deadline - time.time()))
chunk = os.read(fd, RELAY_CHUNK)
if not chunk: break
written += len(chunk)
if written > CONV_SIZE_MAX: return None
encoder_stdin.write(chunk)
return written
finally: selector.close()
def detect_image_format(data):
return _classify_image(data)
def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=None, max_dimension=None):
configured = _configured_backend(quality=quality, max_dimension=max_dimension)
if configured is None: return False
backend_name = configured[0]
backend_name, encoder_argv = configured
if timeout is None: timeout = CONVERSION_TIMEOUT
deadline = time.time() + timeout
try:
with open(output_path, "wb") as output_fh:
input_proc = subprocess.Popen(input_argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
encoder_proc = subprocess.Popen(encoder_argv, stdin=input_proc.stdout, stdout=output_fh, stderr=subprocess.PIPE)
input_fd = input_proc.stdout.fileno()
# The parent must not retain a copy of the input pipe's write
# end, or the input process would never see EOF.
input_proc.stdout.close()
if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=timeout): return False
prefix = _read_prefix(input_fd, SNIFF_BYTES, deadline)
if prefix is None:
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
_terminate(input_proc)
return False
detected = _classify_image(prefix)
if detected is None:
RNS.log("Media conversion failed: Input format is not supported", RNS.LOG_DEBUG)
_terminate(input_proc)
return False
configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-")
if configured is None:
_terminate(input_proc)
return False
backend_name, encoder_argv = configured
encoder_proc = subprocess.Popen(encoder_argv, stdin=subprocess.PIPE, stdout=output_fh, stderr=subprocess.PIPE)
aborted = False
try:
if _pump(input_fd, encoder_proc.stdin, prefix, deadline) is None:
RNS.log(f"Media conversion failed: Streaming input exceeded maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG)
aborted = True
except subprocess.TimeoutExpired:
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
aborted = True
except (BrokenPipeError, OSError):
RNS.log(f"Media conversion via {backend_name} failed: Encoder exited before input was consumed", RNS.LOG_DEBUG)
aborted = True
if aborted:
_terminate(input_proc)
_terminate(encoder_proc)
return False
try: encoder_proc.stdin.close()
except (BrokenPipeError, OSError): pass
try: input_proc.stdout.close()
except Exception: pass
if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=max(0.0, deadline - time.time())): return False
if not _valid_webp(output_path):
RNS.log(f"Media conversion via {backend_name} produced invalid WebP output", RNS.LOG_WARNING)
@ -207,19 +322,42 @@ def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=Non
def convert_file_to_webp(source_path, quality=85, max_dimension=None, timeout=None):
configured = _configured_backend(quality=quality, max_dimension=max_dimension)
if configured is None: return False
backend_name, encoder_argv = configured
backend_name = configured[0]
tmp_path = None
success = False
try:
if not os.path.isfile(source_path):
RNS.log(f"Cannot convert media: source file does not exist: {source_path}", RNS.LOG_WARNING)
RNS.log(f"Cannot convert media: Source file does not exist: {source_path}", RNS.LOG_WARNING)
return False
try: source_size = os.path.getsize(source_path)
except Exception: source_size = None
if source_size is not None and source_size > CONV_SIZE_MAX:
RNS.log(f"Cannot convert media: Source file exceeds maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG)
return False
fd, tmp_path = tempfile.mkstemp(prefix="rns_media_", suffix=".webp")
os.close(fd)
with open(source_path, "rb") as input_fh, open(tmp_path, "wb") as output_fh:
with open(source_path, "rb", buffering=0) as input_fh, open(tmp_path, "wb") as output_fh:
prefix = b""
while len(prefix) < SNIFF_BYTES:
chunk = input_fh.read(SNIFF_BYTES - len(prefix))
if not chunk: break
prefix += chunk
detected = _classify_image(prefix)
if detected is None:
RNS.log("Cannot convert media: Source file is not a supported media format", RNS.LOG_DEBUG)
return False
configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-")
if configured is None: return False
backend_name, encoder_argv = configured
input_fh.seek(0)
encoder_proc = subprocess.Popen(encoder_argv, stdin=input_fh, stdout=output_fh, stderr=subprocess.PIPE)
if not _await(backend_name, encoder_proc, timeout=timeout): return False

View file

@ -46,6 +46,7 @@ from RNS.vendor.configobj import ConfigObj
from RNS.vendor import umsgpack as mp
from RNS._version import __version__
from RNS.Utilities.rnid import validate_rsg, extract_signed_rsg_data
from RNS.Utilities.rngit.media import SUPPORTED_IMAGE_EXTS
from RNS.Utilities.rngit.commitsigs import unarmor_ssh_signature, parse_ssh_signature, extract_commit_author
class NomadNetworkNode():
@ -127,7 +128,7 @@ class NomadNetworkNode():
CONVERTABLE_EXTS = [".md"]
RENDERABLE_EXTS = [".md", ".mu"]
RENDER_DEFAULT = [".md", ".mu"]
IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"]
IMAGE_EXTS = SUPPORTED_IMAGE_EXTS
def __init__(self, owner=None):
if not owner: raise TypeError(f"Invalid owner {owner} for {self}")