From 3e83fcc18b4cae4372b36b0d0cb48f2a904d0974 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sun, 25 Oct 2026 00:21:32 +0200 Subject: [PATCH] Memory management --- Changelog.md | 1 + RNS/Transport.py | 90 +++++++++++++++++++++++++++++++------------ docs/source/using.rst | 2 + 3 files changed, 69 insertions(+), 24 deletions(-) diff --git a/Changelog.md b/Changelog.md index ae976152..13c6b540 100644 --- a/Changelog.md +++ b/Changelog.md @@ -11,6 +11,7 @@ - Added managed storage for known destinations and ratchets - Added managed storage for cached announces - Added more comprehensive metrics to status output structure +- Added heap-tracking memory management - Optimized announce rate table implementation - Optimized packet filter hashlist implementation - Fixed `RNodeInterface` phy stats regression on Android diff --git a/RNS/Transport.py b/RNS/Transport.py index 4d4af1d4..3d1874bd 100755 --- a/RNS/Transport.py +++ b/RNS/Transport.py @@ -281,8 +281,11 @@ class Transport: last_mgmt_announce = 0 mem_trim_throttle = 300 last_mem_trim = 0 - collect_throttle = 1.0 + collect_throttle = 1440 last_collect = 0 + mem_trim_pad = 0 + mem_trim_gate_mb = 8 + last_trim_rss_kb = -1 mgmt_announce_interval = 2*60*60 blackhole_last_checked = 0 blackhole_check_interval = 60 @@ -1297,10 +1300,13 @@ class Transport: except Exception as e: RNS.log(f"Error while checking blackholed identities: {e}", RNS.LOG_ERROR) - if Transport.should_collect and time.time() > Transport.last_collect+Transport.collect_throttle: - garbage_collect() + if Transport.should_collect: + Transport.mem_dirty = True + if time.time() > Transport.last_collect+Transport.collect_throttle: + garbage_collect() - if Transport.mem_dirty and time.time() > Transport.last_mem_trim+Transport.mem_trim_throttle: + trim_surplus = _trim_due(time.time()) + if trim_surplus is not None: Transport.mem_dirty = False; trim_heap() Transport.last_mem_trim = time.time() @@ -3063,7 +3069,7 @@ class Transport: with Transport.announce_handler_lock: while handler in Transport.announce_handlers: Transport.announce_handlers.remove(handler) - garbage_collect() + Transport.should_collect = True @staticmethod def interface_hashes(): @@ -3902,7 +3908,7 @@ class Transport: interface.announce_queue = [] RNS.log("Dropped "+na_str+" on "+str(interface), RNS.LOG_VERBOSE) if RNS.sl(RNS.LOG_VERBOSE) else None - garbage_collect() + Transport.should_collect = True @staticmethod def timebase_from_random_blob(random_blob): @@ -3964,9 +3970,6 @@ class Transport: except Exception as e: RNS.log("Could not save packet hashlist to storage, the contained exception was: "+str(e), RNS.LOG_ERROR) finally: Transport.saving_packet_hashlist = False - garbage_collect() - - @staticmethod def save_path_table(background=False): if not Transport.owner.is_connected_to_shared_instance: @@ -4057,8 +4060,6 @@ class Transport: RNS.trace_exception(e) Transport.saving_path_table = False - garbage_collect() - @staticmethod def save_tunnel_table(background=False): @@ -4143,7 +4144,6 @@ class Transport: RNS.log("Could not save tunnel table to storage, the contained exception was: "+str(e), RNS.LOG_ERROR) Transport.saving_tunnel_table = False - garbage_collect() @staticmethod def persist_data(background=False): @@ -4152,7 +4152,6 @@ class Transport: Transport.save_packet_hashlist(background=background) Transport.save_path_table(background=background) Transport.save_tunnel_table(background=background) - garbage_collect() @staticmethod def void_queues(): @@ -4305,44 +4304,87 @@ class Transport: # Memory management +MEM_TRIM_FLOOR = 60 +_trim_impl = None +_trim_resolved = False + def garbage_collect(): Transport.should_collect = False gc.collect(); Transport.mem_dirty = True Transport.last_collect = time.time() -_trim_impl = None -_trim_resolved = False def trim_heap(): global _trim_impl, _trim_resolved if not _trim_resolved: _trim_resolved = True - _trim_impl = _resolve_trim() + _trim_impl = _resolve_trim() + if _trim_impl is not None: - try: _trim_impl() - except Exception as e: RNS.log(f"Memory trim error: {e}", RNS.LOG_DEBUG) # TODO: Reset to no-op + try: + _trim_impl(Transport.mem_trim_pad) + Transport.last_trim_rss_kb = _proc_status_kb("VmRSS")["VmRSS"] + if Transport.last_trim_rss_kb >= 0: + RNS.log(f"Heap trimmed, RSS {RNS.prettysize(Transport.last_trim_rss_kb*1024)}", RNS.LOG_DEBUG) + except Exception as e: RNS.log(f"Memory trim error: {e}", RNS.LOG_DEBUG) + +def _proc_status_kb(*fields): + # Map missing fields to -1 + values = {field: -1 for field in fields} + try: + with open("/proc/self/status") as status_file: + for line in status_file: + for field in fields: + if line.startswith(field): + values[field] = int(line.split()[1]) + except (OSError, ValueError): pass + return values + +def _trim_due(now): + if not Transport.mem_dirty: return None + elapsed = now - Transport.last_mem_trim + + # If no gate configured, rely solely on throttle + if Transport.mem_trim_gate_mb <= 0: + if now > Transport.last_mem_trim+Transport.mem_trim_throttle: return -1 + return None + + # Hard limit trim cadence + if elapsed < MEM_TRIM_FLOOR: return None + + # First gated trim, no baseline yet + if Transport.last_trim_rss_kb < 0: return -1 + + rss = _proc_status_kb("VmRSS")["VmRSS"] + if rss < 0: # No RSS introspection available, fall back to the timer + if now > Transport.last_mem_trim+Transport.mem_trim_throttle: return -1 + return None + + surplus = rss - Transport.last_trim_rss_kb + if surplus >= Transport.mem_trim_gate_mb*1024: return surplus + return None def _resolve_trim(): try: if RNS.vendor.platformutils.is_linux(): - f = getattr(ctypes.CDLL(None), "malloc_trim", None) # None on musl + f = getattr(ctypes.CDLL(None), "malloc_trim", None) # Not available on musl if f is None: return None - return lambda: f(0) + return lambda pad: f(pad) elif RNS.vendor.platformutils.is_darwin(): libc = ctypes.CDLL(ctypes.util.find_library("c")) f = getattr(libc, "malloc_zone_pressure_relief", None) if f is None: return None - def trim(): f(None, 0); f(None, 0) # Yes, it's strange, see Apple's malloc.h + def trim(pad): f(None, 0); f(None, 0) # Yes, it's strange, see Apple's malloc.h return trim elif RNS.vendor.platformutils.is_android(): libc = ctypes.CDLL("libc.so") - return lambda: libc.mallopt(-101, 0) # M_PURGE, API 28+ + return lambda pad: libc.mallopt(-101, 0) # M_PURGE, API 28+ elif RNS.vendor.platformutils.is_windows(): k32 = ctypes.WinDLL("kernel32") heap = k32.GetProcessHeap() f = k32.HeapTrim # Will raise AttributeError on Windows < 8 - return lambda: f(heap, 0) + return lambda pad: f(heap, 0) except Exception as e: RNS.log(f"Heap trim unavailable: {e}", RNS.LOG_DEBUG) - RNS.log(f"Could not resolve memory trim function on this platform, memory management will be sub-optimal", RNS.LOG_WARNING) + RNS.log(f"No working memory trim mechanism on this platform, memory management will be sub-optimal", RNS.LOG_WARNING) return None # Table entry indices diff --git a/docs/source/using.rst b/docs/source/using.rst index c31ce7dd..ee47a30f 100644 --- a/docs/source/using.rst +++ b/docs/source/using.rst @@ -1470,6 +1470,7 @@ following content: Restart=always RestartSec=3 User=USERNAMEHERE + Environment=MALLOC_ARENA_MAX=2 ExecStart=rnsd --service [Install] @@ -1511,6 +1512,7 @@ Create a user systemd service files ``~/.config/systemd/user/rnsd.service`` with Type=simple Restart=always RestartSec=3 + Environment=MALLOC_ARENA_MAX=2 ExecStart=RNS_BIN_DIR/rnsd --service [Install]