diff --git a/RNS/Utilities/rngit/media.py b/RNS/Utilities/rngit/media.py index 75008c29..d5891b61 100644 --- a/RNS/Utilities/rngit/media.py +++ b/RNS/Utilities/rngit/media.py @@ -31,6 +31,7 @@ import os import time import shutil +import selectors import subprocess import tempfile import RNS @@ -50,6 +51,11 @@ BACKENDS = ( ("magick", ["magick", "-", "webp:-"]), ("ffmpeg", ["ffmpeg", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]), ("avconv", ["avconv", "-y", "-loglevel", "error", "-i", "-", "-f", "webp", "pipe:1"]) ) +SUPPORTED_IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"] +CONV_SIZE_MAX = 32*1024*1024 +SNIFF_BYTES = 64 +RELAY_CHUNK = 256*1024 + _ENV_BACKEND = os.environ.get("RNGIT_MEDIA_BACKEND") _winner = None _no_backend_logged = False @@ -106,6 +112,24 @@ def _webp_info(data): if width > 0 and height > 0: return (width, height) return None +def _is_bmp_header(data): + if len(data) < 18 or data[:2] != b"BM": return False + try: dib_size = int.from_bytes(data[14:18], "little") + except Exception: return False + return dib_size in (12, 40, 52, 56, 64, 108, 124) + +_IMAGE_SIGNATURES = ( ("png", lambda data: len(data) >= 8 and data[:8] == b"\x89PNG\r\n\x1a\n"), + ("jpeg", lambda data: len(data) >= 3 and data[:3] == b"\xff\xd8\xff"), + ("gif", lambda data: len(data) >= 6 and data[:6] in (b"GIF87a", b"GIF89a")), + ("tiff", lambda data: len(data) >= 4 and data[:4] in (b"II*\x00", b"MM\x00*")), + ("webp", lambda data: _webp_info(data) is not None), + ("bmp", lambda data: _is_bmp_header(data)) ) + +def _classify_image(data): + for format_name, signature_matches in _IMAGE_SIGNATURES: + if signature_matches(data): return format_name + return None + def _valid_webp(path): try: with open(path, "rb") as fh: return _webp_info(fh.read(30)) is not None @@ -115,7 +139,7 @@ def _unlink_output(path): try: os.unlink(path) except Exception: pass -def _configured_backend(quality=None, max_dimension=None): +def _configured_backend(quality=None, max_dimension=None, input_spec=None): global _no_backend_logged backend = _selected_backend() if backend is None: @@ -139,10 +163,15 @@ def _configured_backend(quality=None, max_dimension=None): encoder_argv = list(encoder_argv) if backend_name in ("magick", "convert", "gm"): + if input_spec is not None: + try: input_index = encoder_argv.index("-") + except ValueError: input_index = None + if input_index is not None: encoder_argv[input_index] = input_spec options = [] if quality_arg is not None: options += ["-quality", str(quality_arg)] if dimension_arg is not None: options += ["-resize", f"{dimension_arg}x{dimension_arg}>"] encoder_argv = encoder_argv[:-1] + options + encoder_argv[-1:] + elif backend_name in ("ffmpeg", "avconv"): options = [] if quality_arg is not None: options += ["-quality", str(quality_arg)] @@ -163,13 +192,13 @@ def _await(backend_name, encoder_proc, input_proc=None, timeout=None): except subprocess.TimeoutExpired: _terminate(input_proc) _terminate(encoder_proc) - RNS.log(f"Media conversion via {backend_name} timed out after {timeout} seconds", RNS.LOG_WARNING) + RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING) return False if encoder_proc.returncode != 0: detail = "" encoder_tail = _stderr_tail(encoder_proc) - input_tail = _stderr_tail(input_proc) if input_proc is not None else "" + input_tail = _stderr_tail(input_proc) if input_proc is not None else "" if encoder_tail: detail += f" Encoder: {encoder_tail}" if input_tail: detail += f" Input: {input_tail}" RNS.log(f"Media conversion via {backend_name} failed.{detail}", RNS.LOG_WARNING) @@ -177,21 +206,107 @@ def _await(backend_name, encoder_proc, input_proc=None, timeout=None): return True +def _await_readable(fd, timeout, selector): + try: return bool(selector.select(max(0.0, timeout))) + except Exception: return False + +def _read_prefix(fd, amount, deadline): + data = bytearray() + selector = selectors.DefaultSelector() + try: + selector.register(fd, selectors.EVENT_READ) + while len(data) < amount: + if not _await_readable(fd, deadline - time.time(), selector): return None + chunk = os.read(fd, amount - len(data)) + if not chunk: break + data += chunk + return bytes(data) + finally: selector.close() + +def _pump(fd, encoder_stdin, prefix, deadline): + written = len(prefix) + if written > CONV_SIZE_MAX: return None + encoder_stdin.write(prefix) + + selector = selectors.DefaultSelector() + try: + selector.register(fd, selectors.EVENT_READ) + while True: + if not _await_readable(fd, deadline - time.time(), selector): + raise subprocess.TimeoutExpired("input relay", max(0.0, deadline - time.time())) + chunk = os.read(fd, RELAY_CHUNK) + if not chunk: break + + written += len(chunk) + if written > CONV_SIZE_MAX: return None + encoder_stdin.write(chunk) + + return written + + finally: selector.close() + +def detect_image_format(data): + return _classify_image(data) + def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=None, max_dimension=None): configured = _configured_backend(quality=quality, max_dimension=max_dimension) if configured is None: return False + backend_name = configured[0] - backend_name, encoder_argv = configured + if timeout is None: timeout = CONVERSION_TIMEOUT + deadline = time.time() + timeout try: with open(output_path, "wb") as output_fh: input_proc = subprocess.Popen(input_argv, cwd=cwd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) - encoder_proc = subprocess.Popen(encoder_argv, stdin=input_proc.stdout, stdout=output_fh, stderr=subprocess.PIPE) + input_fd = input_proc.stdout.fileno() - # The parent must not retain a copy of the input pipe's write - # end, or the input process would never see EOF. - input_proc.stdout.close() - if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=timeout): return False + prefix = _read_prefix(input_fd, SNIFF_BYTES, deadline) + if prefix is None: + RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING) + _terminate(input_proc) + return False + + detected = _classify_image(prefix) + if detected is None: + RNS.log("Media conversion failed: Input format is not supported", RNS.LOG_DEBUG) + _terminate(input_proc) + return False + + configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-") + if configured is None: + _terminate(input_proc) + return False + backend_name, encoder_argv = configured + + encoder_proc = subprocess.Popen(encoder_argv, stdin=subprocess.PIPE, stdout=output_fh, stderr=subprocess.PIPE) + + aborted = False + try: + if _pump(input_fd, encoder_proc.stdin, prefix, deadline) is None: + RNS.log(f"Media conversion failed: Streaming input exceeded maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG) + aborted = True + + except subprocess.TimeoutExpired: + RNS.log(f"Media conversion via {backend_name} timed out after {RNS.prettytime(timeout)}", RNS.LOG_WARNING) + aborted = True + + except (BrokenPipeError, OSError): + RNS.log(f"Media conversion via {backend_name} failed: Encoder exited before input was consumed", RNS.LOG_DEBUG) + aborted = True + + if aborted: + _terminate(input_proc) + _terminate(encoder_proc) + return False + + try: encoder_proc.stdin.close() + except (BrokenPipeError, OSError): pass + + try: input_proc.stdout.close() + except Exception: pass + + if not _await(backend_name, encoder_proc, input_proc=input_proc, timeout=max(0.0, deadline - time.time())): return False if not _valid_webp(output_path): RNS.log(f"Media conversion via {backend_name} produced invalid WebP output", RNS.LOG_WARNING) @@ -207,19 +322,42 @@ def convert_to_webp(input_argv, output_path, cwd=None, timeout=None, quality=Non def convert_file_to_webp(source_path, quality=85, max_dimension=None, timeout=None): configured = _configured_backend(quality=quality, max_dimension=max_dimension) if configured is None: return False - backend_name, encoder_argv = configured + backend_name = configured[0] tmp_path = None success = False try: if not os.path.isfile(source_path): - RNS.log(f"Cannot convert media: source file does not exist: {source_path}", RNS.LOG_WARNING) + RNS.log(f"Cannot convert media: Source file does not exist: {source_path}", RNS.LOG_WARNING) + return False + + try: source_size = os.path.getsize(source_path) + except Exception: source_size = None + if source_size is not None and source_size > CONV_SIZE_MAX: + RNS.log(f"Cannot convert media: Source file exceeds maximum size of {RNS.prettysize(CONV_SIZE_MAX)}", RNS.LOG_DEBUG) return False fd, tmp_path = tempfile.mkstemp(prefix="rns_media_", suffix=".webp") os.close(fd) - with open(source_path, "rb") as input_fh, open(tmp_path, "wb") as output_fh: + with open(source_path, "rb", buffering=0) as input_fh, open(tmp_path, "wb") as output_fh: + prefix = b"" + while len(prefix) < SNIFF_BYTES: + chunk = input_fh.read(SNIFF_BYTES - len(prefix)) + if not chunk: break + prefix += chunk + + detected = _classify_image(prefix) + if detected is None: + RNS.log("Cannot convert media: Source file is not a supported media format", RNS.LOG_DEBUG) + return False + + configured = _configured_backend(quality=quality, max_dimension=max_dimension, input_spec=detected+":-") + if configured is None: return False + backend_name, encoder_argv = configured + + input_fh.seek(0) + encoder_proc = subprocess.Popen(encoder_argv, stdin=input_fh, stdout=output_fh, stderr=subprocess.PIPE) if not _await(backend_name, encoder_proc, timeout=timeout): return False diff --git a/RNS/Utilities/rngit/pages.py b/RNS/Utilities/rngit/pages.py index 86523feb..cf7b1e0e 100644 --- a/RNS/Utilities/rngit/pages.py +++ b/RNS/Utilities/rngit/pages.py @@ -46,6 +46,7 @@ from RNS.vendor.configobj import ConfigObj from RNS.vendor import umsgpack as mp from RNS._version import __version__ from RNS.Utilities.rnid import validate_rsg, extract_signed_rsg_data +from RNS.Utilities.rngit.media import SUPPORTED_IMAGE_EXTS from RNS.Utilities.rngit.commitsigs import unarmor_ssh_signature, parse_ssh_signature, extract_commit_author class NomadNetworkNode(): @@ -127,7 +128,7 @@ class NomadNetworkNode(): CONVERTABLE_EXTS = [".md"] RENDERABLE_EXTS = [".md", ".mu"] RENDER_DEFAULT = [".md", ".mu"] - IMAGE_EXTS = [".webp", ".png", ".jpg", ".jpeg", ".gif", ".tiff", ".tif", ".bmp"] + IMAGE_EXTS = SUPPORTED_IMAGE_EXTS def __init__(self, owner=None): if not owner: raise TypeError(f"Invalid owner {owner} for {self}")