Added canonical gc handling and platform-aware malloc trim

This commit is contained in:
Mark Qvist 2026-10-24 14:25:09 +02:00
parent 044d4c9adc
commit 156290733e
No known key found for this signature in database

View file

@ -33,6 +33,8 @@ import gc
import RNS
import time
import math
import ctypes
import ctypes.util
import struct
import inspect
import threading
@ -277,9 +279,15 @@ class Transport:
interface_last_jobs = 0.0
interface_jobs_interval = 5.0
last_mgmt_announce = 0
mem_trim_throttle = 300
last_mem_trim = 0
collect_throttle = 1.0
last_collect = 0
mgmt_announce_interval = 2*60*60
blackhole_last_checked = 0
blackhole_check_interval = 60
should_collect = False
mem_dirty = False
unused_path_linger = None
interface_announcer = None
discovery_handler = None
@ -422,6 +430,10 @@ class Transport:
# Defer path table unused pool cull
Transport.pool_size_last_checked = time.time()
# Defer memory trim
Transport.last_collect = time.time()
Transport.last_mem_trim = time.time()
# Defer sending management announces for 15 seconds
Transport.last_mgmt_announce = time.time() - Transport.mgmt_announce_interval + 15
@ -489,11 +501,11 @@ class Transport:
else: specifier = "entries"
RNS.log("Loaded "+str(len(Transport.path_table))+" path table "+specifier+" from storage", RNS.LOG_VERBOSE) if RNS.sl(RNS.LOG_VERBOSE) else None
gc.collect()
garbage_collect()
except Exception as e:
RNS.log("Could not load destination table from storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
gc.collect()
garbage_collect()
if os.path.isfile(tunnel_table_path) and not Transport.owner.is_connected_to_shared_instance:
serialised_tunnels = []
@ -540,11 +552,11 @@ class Transport:
else: specifier = "entries"
RNS.log("Loaded "+str(len(Transport.tunnels))+" tunnel table "+specifier+" from storage", RNS.LOG_VERBOSE) if RNS.sl(RNS.LOG_VERBOSE) else None
gc.collect()
garbage_collect()
except Exception as e:
RNS.log("Could not load tunnel table from storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
gc.collect()
garbage_collect()
if RNS.Reticulum.probe_destination_enabled():
Transport.probe_destination = RNS.Destination(Transport.identity, RNS.Destination.IN, RNS.Destination.SINGLE, Transport.APP_NAME, "probe")
@ -574,7 +586,7 @@ class Transport:
if hasattr(interface, "wants_tunnel") and interface.wants_tunnel:
Transport.synthesize_tunnel(interface)
gc.collect()
garbage_collect()
@staticmethod
def add_interface(interface):
@ -730,8 +742,6 @@ class Transport:
try:
with Transport.jobs_lock:
should_collect = False
# Process active and pending link lists
if time.time() > Transport.links_last_checked+Transport.links_check_interval:
@ -787,7 +797,7 @@ class Transport:
culled_receipt = Transport.receipts.pop(0)
culled_receipt.timeout = -1
culled_receipt.check_timeout()
should_collect = True
Transport.should_collect = True
# TODO: Actually, this whole thing might need re-
# thinking a bit, but may require moving receipts
@ -1030,15 +1040,15 @@ class Transport:
if now > path_expiry:
stale_paths.append(destination_hash)
should_collect = True
Transport.should_collect = True
RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
elif not attached_interface in Transport.interfaces:
stale_paths.append(destination_hash)
should_collect = True
Transport.should_collect = True
RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
elif path_used == 0 and now > path_timestamp + Transport.unused_path_linger:
stale_paths.append(destination_hash)
should_collect = True
Transport.should_collect = True
RNS.log(f"Removing never used path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
if len(unused_paths) > Transport.PATH_POOL_UNUSED_TARGET:
@ -1084,7 +1094,7 @@ class Transport:
if time.time() > entry["timeout"]:
stale_discovery_path_requests.append(destination_hash)
should_collect = True
Transport.should_collect = True
RNS.log("Waiting path request for "+RNS.prettyhexrep(destination_hash)+" timed out and was removed", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
# Cull the tunnel table
@ -1095,11 +1105,11 @@ class Transport:
expires = tunnel_entry[IDX_TT_EXPIRES]
if expires > time.time() + Transport.TUNNEL_TIMEOUT*2:
stale_tunnels.append(tunnel_id); should_collect = True
stale_tunnels.append(tunnel_id); Transport.should_collect = True
RNS.log("Tunnel "+RNS.prettyhexrep(tunnel_id)+" with excessive expiry was removed", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
elif time.time() > expires:
stale_tunnels.append(tunnel_id); should_collect = True
stale_tunnels.append(tunnel_id); Transport.should_collect = True
RNS.log("Tunnel "+RNS.prettyhexrep(tunnel_id)+" timed out and was removed", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
else:
@ -1113,7 +1123,7 @@ class Transport:
tunnel_path_entry = tunnel_paths[tunnel_path]
if time.time() > tunnel_path_entry[0] + Transport.TUNNEL_PATH_TIMEOUT:
stale_tunnel_paths.append(tunnel_path); should_collect = True
stale_tunnel_paths.append(tunnel_path); Transport.should_collect = True
RNS.log("Tunnel path to "+RNS.prettyhexrep(tunnel_path)+" timed out and was removed", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
else:
@ -1128,7 +1138,7 @@ class Transport:
tunnel_announce_timebase = Transport.timebase_from_random_blobs(random_blobs)
if current_path_timebase > tunnel_announce_timebase:
stale_tunnel_paths.append(tunnel_path); should_collect = True
stale_tunnel_paths.append(tunnel_path); Transport.should_collect = True
RNS.log("Tunnel path to "+RNS.prettyhexrep(tunnel_path)+" was removed due to more recent active path", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
for tunnel_path in stale_tunnel_paths:
@ -1287,7 +1297,12 @@ class Transport:
except Exception as e:
RNS.log(f"Error while checking blackholed identities: {e}", RNS.LOG_ERROR)
if should_collect: gc.collect()
if Transport.should_collect and time.time() > Transport.last_collect+Transport.collect_throttle:
garbage_collect()
if Transport.mem_dirty and time.time() > Transport.last_mem_trim+Transport.mem_trim_throttle:
Transport.mem_dirty = False; trim_heap()
Transport.last_mem_trim = time.time()
except Exception as e:
RNS.log("An exception occurred while running Transport jobs.", RNS.LOG_ERROR)
@ -3048,7 +3063,7 @@ class Transport:
with Transport.announce_handler_lock:
while handler in Transport.announce_handlers: Transport.announce_handlers.remove(handler)
gc.collect()
garbage_collect()
@staticmethod
def interface_hashes():
@ -3887,7 +3902,7 @@ class Transport:
interface.announce_queue = []
RNS.log("Dropped "+na_str+" on "+str(interface), RNS.LOG_VERBOSE) if RNS.sl(RNS.LOG_VERBOSE) else None
gc.collect()
garbage_collect()
@staticmethod
def timebase_from_random_blob(random_blob):
@ -3949,7 +3964,7 @@ class Transport:
except Exception as e: RNS.log("Could not save packet hashlist to storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
finally: Transport.saving_packet_hashlist = False
gc.collect()
garbage_collect()
@staticmethod
@ -4042,7 +4057,7 @@ class Transport:
RNS.trace_exception(e)
Transport.saving_path_table = False
gc.collect()
garbage_collect()
@staticmethod
@ -4128,7 +4143,7 @@ class Transport:
RNS.log("Could not save tunnel table to storage, the contained exception was: "+str(e), RNS.LOG_ERROR)
Transport.saving_tunnel_table = False
gc.collect()
garbage_collect()
@staticmethod
def persist_data(background=False):
@ -4137,7 +4152,7 @@ class Transport:
Transport.save_packet_hashlist(background=background)
Transport.save_path_table(background=background)
Transport.save_tunnel_table(background=background)
gc.collect()
garbage_collect()
@staticmethod
def void_queues():
@ -4288,6 +4303,47 @@ class Transport:
RNS.log(f"Error while persisting blackhole list: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
# Memory management
def garbage_collect():
Transport.should_collect = False
gc.collect(); Transport.mem_dirty = True
Transport.last_collect = time.time()
_trim_impl = None
_trim_resolved = False
def trim_heap():
global _trim_impl, _trim_resolved
if not _trim_resolved:
_trim_resolved = True
_trim_impl = _resolve_trim()
if _trim_impl is not None:
try: _trim_impl()
except Exception as e: RNS.log(f"Memory trim error: {e}", RNS.LOG_DEBUG) # TODO: Reset to no-op
def _resolve_trim():
try:
if RNS.vendor.platformutils.is_linux():
f = getattr(ctypes.CDLL(None), "malloc_trim", None) # None on musl
if f is None: return None
return lambda: f(0)
elif RNS.vendor.platformutils.is_darwin():
libc = ctypes.CDLL(ctypes.util.find_library("c"))
f = getattr(libc, "malloc_zone_pressure_relief", None)
if f is None: return None
def trim(): f(None, 0); f(None, 0) # Yes, it's strange, see Apple's malloc.h
return trim
elif RNS.vendor.platformutils.is_android():
libc = ctypes.CDLL("libc.so")
return lambda: libc.mallopt(-101, 0) # M_PURGE, API 28+
elif RNS.vendor.platformutils.is_windows():
k32 = ctypes.WinDLL("kernel32")
heap = k32.GetProcessHeap()
f = k32.HeapTrim # Will raise AttributeError on Windows < 8
return lambda: f(heap, 0)
except Exception as e: RNS.log(f"Heap trim unavailable: {e}", RNS.LOG_DEBUG)
RNS.log(f"Could not resolve memory trim function on this platform, memory management will be sub-optimal", RNS.LOG_WARNING)
return None
# Table entry indices