mirror of
https://github.com/markqvist/Reticulum.git
synced 2026-11-20 15:06:31 -05:00
Added rngit media conversion max input size cap, improved logging and error handling
This commit is contained in:
parent
cff4d03fdb
commit
840eb3699b
2 changed files with 152 additions and 13 deletions
|
|
@ -31,6 +31,7 @@
|
|||
import os
|
||||
import time
|
||||
import shutil
|
||||
import selectors
|
||||
import subprocess
|
||||
import tempfile
|
||||
import RNS
|
||||
|
|
@ -50,6 +51,11 @@ BACKENDS = ( ("magick", ["magick", "-", "webp:-"]),
|
|||
("ffmpeg", ["ffmpeg", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]),
|
||||
("avconv", ["avconv", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]) )
|
||||
|
||||
SUPPORTED_IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"]
|
||||
CONV_SIZE_MAX = 32*1024*1024
|
||||
SNIFF_BYTES = 64
|
||||
RELAY_CHUNK = 256*1024
|
||||
|
||||
_ENV_BACKEND = os.environ.get("RNGIT_MEDIA_BACKEND")
|
||||
_winner = None
|
||||
_no_backend_logged = False
|
||||
|
|
@ -106,6 +112,24 @@ def _webp_info(data):
|
|||
if width > 0 and height > 0: return (width, height)
|
||||
return None
|
||||
|
||||
def _is_bmp_header(data):
|
||||
if len(data) < 18 or data[:2] != b"BM": return False
|
||||
try: dib_size = int.from_bytes(data[14:18], "little")
|
||||
except Exception: return False
|
||||
return dib_size in (12, 40, 52, 56, 64, 108, 124)
|
||||
|
||||
_IMAGE_SIGNATURES = ( ("png", lambda data: len(data) >= 8 and data[:8] == b"\x89PNG\r\n\x1a\n"),
|
||||
("jpeg", lambda data: len(data) >= 3 and data[:3] == b"\xff\xd8\xff"),
|
||||
("gif", lambda data: len(data) >= 6 and data[:6] in (b"GIF87a", b"GIF89a")),
|
||||
("tiff", lambda data: len(data) >= 4 and data[:4] in (b"II*\x00", b"MM\x00*")),
|
||||
("webp", lambda data: _webp_info(data) is not None),
|
||||
("bmp", lambda data: _is_bmp_header(data)) )
|
||||
|
||||
def _classify_image(data):
|
||||
for format_name, signature_matches in _IMAGE_SIGNATURES:
|
||||
if signature_matches(data): return format_name
|
||||
return None
|
||||
|
||||
def _valid_webp(path):
|
||||
try:
|
||||
with open(path, "rb") as fh: return _webp_info(fh.read(30)) is not None
|
||||
|
|
@ -115,7 +139,7 @@ def _unlink_output(path):
|
|||
try: os.unlink(path)
|
||||
except Exception: pass
|
||||
|
||||
def _configured_backend(quality=None, max_dimension=None):
|
||||
def _configured_backend(quality=None, max_dimension=None, input_spec=None):
|
||||
global _no_backend_logged
|
||||
backend = _selected_backend()
|
||||
if backend is None:
|
||||
|
|
@ -139,10 +163,15 @@ def _configured_backend(quality=None, max_dimension=None):
|
|||
|
||||
encoder_argv = list(encoder_argv)
|
||||
if backend_name in ("magick", "convert", "gm"):
|
||||
if input_spec is not None:
|
||||
try: input_index = encoder_argv.index("-")
|
||||
except ValueError: input_index = None
|
||||
if input_index is not None: encoder_argv[input_index] = input_spec
|
||||
options = []
|
||||
if quality_arg is not None: options += ["-quality", str(quality_arg)]
|
||||
if dimension_arg is not None: options += ["-resize", f"{dimension_arg}x{dimension_arg}>"]
|
||||
encoder_argv = encoder_argv[:-1] + options + encoder_argv[-1:]
|
||||
|
||||
elif backend_name in ("ffmpeg", "avconv"):
|
||||
options = []
|
||||
if quality_arg is not None: options += ["-quality", str(quality_arg)]
|
||||
|
|
@ -163,13 +192,13 @@ def _await(backend_name, encoder_proc, input_proc=None, timeout=None):
|
|||
except subprocess.TimeoutExpired:
|
||||
_terminate(input_proc)
|
||||
_terminate(encoder_proc)
|
||||
RNS.log(f"Media conversion via {backend_name} timed out after {timeout} seconds", RNS.LOG_WARNING)
|
||||
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
||||
if encoder_proc.returncode != 0:
|
||||
detail = ""
|
||||
encoder_tail = _stderr_tail(encoder_proc)
|
||||
input_tail = _stderr_tail(input_proc) if input_proc is not None else ""
|
||||
input_tail = _stderr_tail(input_proc) if input_proc is not None else ""
|
||||
if encoder_tail: detail += f" Encoder: {encoder_tail}"
|
||||
if input_tail: detail += f" Input: {input_tail}"
|
||||
RNS.log(f"Media conversion via {backend_name} failed.{detail}", RNS.LOG_WARNING)
|
||||
|
|
@ -177,21 +206,107 @@ def _await(backend_name, encoder_proc, input_proc=None, timeout=None):
|
|||
|
||||
return True
|
||||
|
||||
def _await_readable(fd, timeout, selector):
|
||||
try: return bool(selector.select(max(0.0, timeout)))
|
||||
except Exception: return False
|
||||
|
||||
def _read_prefix(fd, amount, deadline):
|
||||
data = bytearray()
|
||||
selector = selectors.DefaultSelector()
|
||||
try:
|
||||
selector.register(fd, selectors.EVENT_READ)
|
||||
while len(data) < amount:
|
||||
if not _await_readable(fd, deadline - time.time(), selector): return None
|
||||
chunk = os.read(fd, amount - len(data))
|
||||
if not chunk: break
|
||||
data += chunk
|
||||
return bytes(data)
|
||||
finally: selector.close()
|
||||
|
||||
def _pump(fd, encoder_stdin, prefix, deadline):
|
||||
written = len(prefix)
|
||||
if written > CONV_SIZE_MAX: return None
|
||||
encoder_stdin.write(prefix)
|
||||
|
||||
selector = selectors.DefaultSelector()
|
||||
try:
|
||||
selector.register(fd, selectors.EVENT_READ)
|
||||
while True:
|
||||
if not _await_readable(fd, deadline - time.time(), selector):
|
||||
raise subprocess.TimeoutExpired("input relay", max(0.0, deadline - time.time()))
|
||||
chunk = os.read(fd, RELAY_CHUNK)
|
||||
if not chunk: break
|
||||
|
||||
written += len(chunk)
|
||||
if written > CONV_SIZE_MAX: return None
|
||||
encoder_stdin.write(chunk)
|
||||
|
||||
return written
|
||||
|
||||
finally: selector.close()
|
||||
|
||||
def detect_image_format(data):
|
||||
return _classify_image(data)
|
||||
|
||||
def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=None, max_dimension=None):
|
||||
configured = _configured_backend(quality=quality, max_dimension=max_dimension)
|
||||
if configured is None: return False
|
||||
backend_name = configured[0]
|
||||
|
||||
backend_name, encoder_argv = configured
|
||||
if timeout is None: timeout = CONVERSION_TIMEOUT
|
||||
deadline = time.time() + timeout
|
||||
|
||||
try:
|
||||
with open(output_path, "wb") as output_fh:
|
||||
input_proc = subprocess.Popen(input_argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
|
||||
encoder_proc = subprocess.Popen(encoder_argv, stdin=input_proc.stdout, stdout=output_fh, stderr=subprocess.PIPE)
|
||||
input_fd = input_proc.stdout.fileno()
|
||||
|
||||
# The parent must not retain a copy of the input pipe's write
|
||||
# end, or the input process would never see EOF.
|
||||
input_proc.stdout.close()
|
||||
if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=timeout): return False
|
||||
prefix = _read_prefix(input_fd, SNIFF_BYTES, deadline)
|
||||
if prefix is None:
|
||||
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
|
||||
_terminate(input_proc)
|
||||
return False
|
||||
|
||||
detected = _classify_image(prefix)
|
||||
if detected is None:
|
||||
RNS.log("Media conversion failed: Input format is not supported", RNS.LOG_DEBUG)
|
||||
_terminate(input_proc)
|
||||
return False
|
||||
|
||||
configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-")
|
||||
if configured is None:
|
||||
_terminate(input_proc)
|
||||
return False
|
||||
backend_name, encoder_argv = configured
|
||||
|
||||
encoder_proc = subprocess.Popen(encoder_argv, stdin=subprocess.PIPE, stdout=output_fh, stderr=subprocess.PIPE)
|
||||
|
||||
aborted = False
|
||||
try:
|
||||
if _pump(input_fd, encoder_proc.stdin, prefix, deadline) is None:
|
||||
RNS.log(f"Media conversion failed: Streaming input exceeded maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG)
|
||||
aborted = True
|
||||
|
||||
except subprocess.TimeoutExpired:
|
||||
RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING)
|
||||
aborted = True
|
||||
|
||||
except (BrokenPipeError, OSError):
|
||||
RNS.log(f"Media conversion via {backend_name} failed: Encoder exited before input was consumed", RNS.LOG_DEBUG)
|
||||
aborted = True
|
||||
|
||||
if aborted:
|
||||
_terminate(input_proc)
|
||||
_terminate(encoder_proc)
|
||||
return False
|
||||
|
||||
try: encoder_proc.stdin.close()
|
||||
except (BrokenPipeError, OSError): pass
|
||||
|
||||
try: input_proc.stdout.close()
|
||||
except Exception: pass
|
||||
|
||||
if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=max(0.0, deadline - time.time())): return False
|
||||
|
||||
if not _valid_webp(output_path):
|
||||
RNS.log(f"Media conversion via {backend_name} produced invalid WebP output", RNS.LOG_WARNING)
|
||||
|
|
@ -207,19 +322,42 @@ def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=Non
|
|||
def convert_file_to_webp(source_path, quality=85, max_dimension=None, timeout=None):
|
||||
configured = _configured_backend(quality=quality, max_dimension=max_dimension)
|
||||
if configured is None: return False
|
||||
backend_name, encoder_argv = configured
|
||||
backend_name = configured[0]
|
||||
|
||||
tmp_path = None
|
||||
success = False
|
||||
try:
|
||||
if not os.path.isfile(source_path):
|
||||
RNS.log(f"Cannot convert media: source file does not exist: {source_path}", RNS.LOG_WARNING)
|
||||
RNS.log(f"Cannot convert media: Source file does not exist: {source_path}", RNS.LOG_WARNING)
|
||||
return False
|
||||
|
||||
try: source_size = os.path.getsize(source_path)
|
||||
except Exception: source_size = None
|
||||
if source_size is not None and source_size > CONV_SIZE_MAX:
|
||||
RNS.log(f"Cannot convert media: Source file exceeds maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG)
|
||||
return False
|
||||
|
||||
fd, tmp_path = tempfile.mkstemp(prefix="rns_media_", suffix=".webp")
|
||||
os.close(fd)
|
||||
|
||||
with open(source_path, "rb") as input_fh, open(tmp_path, "wb") as output_fh:
|
||||
with open(source_path, "rb", buffering=0) as input_fh, open(tmp_path, "wb") as output_fh:
|
||||
prefix = b""
|
||||
while len(prefix) < SNIFF_BYTES:
|
||||
chunk = input_fh.read(SNIFF_BYTES - len(prefix))
|
||||
if not chunk: break
|
||||
prefix += chunk
|
||||
|
||||
detected = _classify_image(prefix)
|
||||
if detected is None:
|
||||
RNS.log("Cannot convert media: Source file is not a supported media format", RNS.LOG_DEBUG)
|
||||
return False
|
||||
|
||||
configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-")
|
||||
if configured is None: return False
|
||||
backend_name, encoder_argv = configured
|
||||
|
||||
input_fh.seek(0)
|
||||
|
||||
encoder_proc = subprocess.Popen(encoder_argv, stdin=input_fh, stdout=output_fh, stderr=subprocess.PIPE)
|
||||
if not _await(backend_name, encoder_proc, timeout=timeout): return False
|
||||
|
||||
|
|
|
|||
|
|
@ -46,6 +46,7 @@ from RNS.vendor.configobj import ConfigObj
|
|||
from RNS.vendor import umsgpack as mp
|
||||
from RNS._version import __version__
|
||||
from RNS.Utilities.rnid import validate_rsg, extract_signed_rsg_data
|
||||
from RNS.Utilities.rngit.media import SUPPORTED_IMAGE_EXTS
|
||||
from RNS.Utilities.rngit.commitsigs import unarmor_ssh_signature, parse_ssh_signature, extract_commit_author
|
||||
|
||||
class NomadNetworkNode():
|
||||
|
|
@ -127,7 +128,7 @@ class NomadNetworkNode():
|
|||
CONVERTABLE_EXTS = [".md"]
|
||||
RENDERABLE_EXTS = [".md", ".mu"]
|
||||
RENDER_DEFAULT = [".md", ".mu"]
|
||||
IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"]
|
||||
IMAGE_EXTS = SUPPORTED_IMAGE_EXTS
|
||||
|
||||
def __init__(self, owner=None):
|
||||
if not owner: raise TypeError(f"Invalid owner {owner} for {self}")
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue