Added dataplane ingress control

This commit is contained in:
Mark Qvist 2026-08-25 23:30:59 +02:00
parent d80245b62c
commit de76b8939f
No known key found for this signature in database
3 changed files with 180 additions and 15 deletions

View file

@ -61,12 +61,26 @@ class BackboneInterface(Interface):
fast_flapping_lock = threading.Lock()
fast_flapping = {}
DP_IC_HIGH_WM_PCT = 85
DP_IC_MID_WM_PCT = 10
DP_IC_LOW_WM_PCT = 1
DP_IC_INTERVAL = 0.250
DP_IC_TRIGGER = 1.5
DP_IC_RCVBUF = 32768
DP_IC_IF_HEADROOM = 32
DP_IC_PENALTY = 1.5
epoll = None
listener_filenos = {}
spawned_interface_filenos = {}
epoll = None
_job_active = False
_job_lock = threading.Lock()
_job_active = False
_ic_job_active = False
_job_lock = threading.Lock()
_ic_job_lock = threading.Lock()
_dp_ic_lock = threading.Lock()
_dp_ic_snapshot = 0
@staticmethod
def get_address_for_if(name, bind_port, prefer_ipv6=False):
@ -247,7 +261,8 @@ class BackboneInterface(Interface):
@staticmethod
def start():
if not BackboneInterface._job_active: threading.Thread(target=BackboneInterface.__job, daemon=True).start()
if not BackboneInterface._job_active: threading.Thread(target=BackboneInterface.__job, daemon=True).start()
if not BackboneInterface._ic_job_active: threading.Thread(target=BackboneInterface.__dp_ic_job, daemon=True).start()
@staticmethod
def ensure_epoll():
@ -288,7 +303,7 @@ class BackboneInterface(Interface):
RNS.log(f"Attempt to register invalid file descriptor {fileno}", RNS.LOG_WARNING)
return
try: BackboneInterface.epoll.register(fileno, select.EPOLLIN)
try: BackboneInterface.epoll.register(fileno, select.EPOLLIN | select.EPOLLHUP)
except Exception as e:
RNS.log(f"An error occurred while registering EPOLLIN for file descriptor {fileno}: {e}", RNS.LOG_WARNING)
@ -319,13 +334,147 @@ class BackboneInterface(Interface):
if interface.socket:
fileno = interface.socket.fileno()
if fileno in BackboneInterface.spawned_interface_filenos:
try: BackboneInterface.epoll.modify(fileno, select.EPOLLIN | select.EPOLLOUT)
try:
events = select.EPOLLOUT | select.EPOLLHUP
if not interface.dp_ingress_gated: events |= select.EPOLLIN
BackboneInterface.epoll.modify(fileno, events)
except Exception as e:
if str(e).endswith("No such file or directory"): pass
elif str(e).endswith("Bad file descriptor"): pass
else: RNS.log(f"Error occurred on {interface} while modifying socket EPOLL state: {e}", RNS.LOG_WARNING)
raise e
@staticmethod
def __throttle_ingress(interface, hold=None):
if interface.dp_ingress_gated: return False
try:
fileno = interface.socket.fileno()
if fileno in BackboneInterface.spawned_interface_filenos:
interface.dp_ingress_gated = True
interface.dp_ingress_tcount += 1
events = select.EPOLLHUP
if len(interface.transmit_buffer) > 0: events |= select.EPOLLOUT
BackboneInterface.epoll.modify(fileno, events)
if hold: interface.dp_ingress_hold = time.time() + hold
RNS.log(f"Ingress throttled on {interface}", RNS.LOG_NOTICE) if RNS.sl(RNS.LOG_NOTICE) else None
return True
except Exception as e: RNS.log(f"Error throttling {interface}: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return False
@staticmethod
def __release_ingress(interface):
if not interface.dp_ingress_gated: return False
try:
fileno = interface.socket.fileno()
if fileno in BackboneInterface.spawned_interface_filenos:
events = select.EPOLLIN | select.EPOLLHUP
if len(interface.transmit_buffer) > 0: events |= select.EPOLLOUT
BackboneInterface.epoll.modify(fileno, events)
interface.dp_ingress_gated = False
RNS.log(f"Released ingress throttle on {interface}", RNS.LOG_NOTICE) if RNS.sl(RNS.LOG_NOTICE) else None
return True
except Exception as e: RNS.log(f"Error releasing throttle on {interface}: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return False
@staticmethod
def _throttle_immediate(q_depth):
with BackboneInterface._dp_ic_lock:
st = time.time()
interfaces = list(BackboneInterface.spawned_interface_filenos.values())
producers = [iface for iface in interfaces if iface.dp_ingress_packets > 0]
if producers:
producers.sort(key=lambda p: p.dp_ingress_packets, reverse=True)
selected = producers[0]
now = time.time()
span = now - BackboneInterface._dp_ic_snapshot
total = sum(iface.dp_ingress_packets for iface in producers)
avail = sum(iface.dp_ingress_bytes for iface in interfaces) / span
share = selected.dp_ingress_packets / total
rate = selected.dp_ingress_packets / span
speed = selected.dp_ingress_bytes / span
data = selected.dp_ingress_bytes
pkts = selected.dp_ingress_packets
alloc = 1/((max(len(interfaces), BackboneInterface.DP_IC_IF_HEADROOM))*BackboneInterface.DP_IC_PENALTY)
hold = (avail/(avail*alloc))*span
if BackboneInterface.__throttle_ingress(selected, hold=hold):
taken = time.time()-st
if RNS.sl(RNS.LOG_DEBUG):
RNS.log(f"Throttled producer with {pkts} packets at {RNS.prettysize(rate, suffix='pps')} / {RNS.prettyspeed(speed*8)}, {round(share*100.0, 2)}% ingress share", RNS.LOG_DEBUG)
RNS.log(f"Available ingress budget {RNS.prettyspeed(avail*8)}, hard-allocated {round(alloc*100.0, 2)}% ({RNS.prettyspeed(avail*alloc*8)}), handled in {RNS.prettyshorttime(taken, compact=True, tight=True)}", RNS.LOG_DEBUG)
RNS.log(f"Holding for {RNS.prettyshorttime(hold, compact=True, tight=True)}, throttling handled in {RNS.prettyshorttime(taken, compact=True, tight=True)} at depth {q_depth}", RNS.LOG_DEBUG)
@staticmethod
def __dp_ic_job():
with BackboneInterface._ic_job_lock:
if BackboneInterface._ic_job_active: return
else:
dql = RNS.Reticulum.default_data_queue_length() or RNS.Transport.INBOUND_DA_QUEUE_LENGTH
BackboneInterface.DP_IC_HIGH_WM = max(4, int((BackboneInterface.DP_IC_HIGH_WM_PCT/100.0) * dql))
BackboneInterface.DP_IC_MID_WM = max(2, int((BackboneInterface.DP_IC_MID_WM_PCT/100.0) * dql))
BackboneInterface.DP_IC_LOW_WM = max(0, int((BackboneInterface.DP_IC_LOW_WM_PCT/100.0) * dql))
BackboneInterface._ic_job_active = True
RNS.log(f"Started dataplane ingress control: High/mid/low water marks are {BackboneInterface.DP_IC_HIGH_WM}/{BackboneInterface.DP_IC_MID_WM}/{BackboneInterface.DP_IC_LOW_WM} packets", RNS.LOG_DEBUG)
try:
while True:
time.sleep(BackboneInterface.DP_IC_INTERVAL)
try:
if RNS.Transport.inbound_queues != None: q_depth = RNS.Transport.inbound_queues.qsize(RNS.Transport.TC_DATA)
else: continue
except Exception: continue
with BackboneInterface._dp_ic_lock:
st = time.time()
interfaces = list(BackboneInterface.spawned_interface_filenos.values())
# active = [iface for iface in interfaces if iface.dp_ingress_bytes > 0]
if q_depth > BackboneInterface.DP_IC_MID_WM:
producers = [iface for iface in interfaces if iface.dp_ingress_packets > 0]
if producers:
producers.sort(key=lambda p: p.dp_ingress_packets, reverse=True)
selected = producers[0]
now = time.time()
span = max(now - BackboneInterface._dp_ic_snapshot, 0.001)
total = sum(iface.dp_ingress_packets for iface in producers)
avail = sum(iface.dp_ingress_bytes for iface in interfaces) / span
mean = total / max(len(producers), 2)
share = selected.dp_ingress_packets / total
rate = selected.dp_ingress_packets / span
speed = selected.dp_ingress_bytes / span
data = selected.dp_ingress_bytes
pkts = selected.dp_ingress_packets
alloc = 1/((max(len(interfaces), BackboneInterface.DP_IC_IF_HEADROOM))*BackboneInterface.DP_IC_PENALTY)
hold = (avail/max(avail*alloc, 1))*span
if BackboneInterface.__throttle_ingress(selected, hold=hold):
taken = time.time()-st
if RNS.sl(RNS.LOG_DEBUG):
RNS.log(f"Throttled producer with {pkts} packets at {RNS.prettysize(rate, suffix='pps')} / {RNS.prettyspeed(speed*8)}, {round(share*100.0, 2)}% ingress share", RNS.LOG_DEBUG)
RNS.log(f"Available ingress budget {RNS.prettyspeed(avail*8)}, hard-allocated {round(alloc*100.0, 2)}% ({RNS.prettyspeed(avail*alloc*8)}), handled in {RNS.prettyshorttime(taken, compact=True, tight=True)}", RNS.LOG_DEBUG)
RNS.log(f"Holding for {RNS.prettyshorttime(hold, compact=True, tight=True)}, throttling handled in {RNS.prettyshorttime(taken, compact=True, tight=True)}", RNS.LOG_DEBUG)
elif q_depth < BackboneInterface.DP_IC_LOW_WM:
now = time.time()
for iface in interfaces:
if iface.dp_ingress_gated:
if iface.dp_ingress_hold and now < iface.dp_ingress_hold: continue
BackboneInterface.__release_ingress(iface)
break
BackboneInterface._dp_ic_snapshot = time.time()
for iface in interfaces:
iface.dp_ingress_bytes = 0
iface.dp_ingress_packets = 0
except Exception as e:
RNS.log(f"BackboneInterface ingress control error: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
@staticmethod
def __job():
with BackboneInterface._job_lock:
@ -340,13 +489,16 @@ class BackboneInterface(Interface):
spawned_interface = BackboneInterface.spawned_interface_filenos[fileno]
client_socket = spawned_interface.socket
socket_valid = client_socket and fileno == client_socket.fileno()
if socket_valid and (event & select.EPOLLIN):
if socket_valid and (event & select.EPOLLIN) and not spawned_interface.dp_ingress_gated:
if spawned_interface.dp_ingress_gated: continue
try: received_bytes = client_socket.recv(spawned_interface.HW_MTU)
except Exception as e:
RNS.log(f"Error while reading from {spawned_interface}: {e}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
received_bytes = b""
if len(received_bytes): spawned_interface.receive(received_bytes)
if len(received_bytes):
spawned_interface.dp_ingress_bytes += len(received_bytes)
spawned_interface.receive(received_bytes)
else:
BackboneInterface.deregister_fileno(fileno); client_socket.close()
try:
@ -393,9 +545,11 @@ class BackboneInterface(Interface):
spawned_interface.transmit_buffer = spawned_interface.transmit_buffer[written:]
try:
if len(spawned_interface.transmit_buffer) == 0: BackboneInterface.epoll.modify(fileno, select.EPOLLIN)
except Exception as e:
RNS.log(f"Error while setting EPOLLIN on {spawned_interface}: {e}", RNS.LOG_ERROR)
if len(spawned_interface.transmit_buffer) == 0:
events = select.EPOLLHUP
if not spawned_interface.dp_ingress_gated: events |= select.EPOLLIN
BackboneInterface.epoll.modify(fileno, events)
except Exception as e: RNS.log(f"Error while setting EPOLLIN on {spawned_interface}: {e}", RNS.LOG_ERROR)
spawned_interface.txb += written
if spawned_interface.parent_interface: spawned_interface.parent_interface.txb += written
@ -423,6 +577,7 @@ class BackboneInterface(Interface):
try:
client_socket, address = server_socket.accept()
client_socket.setblocking(0)
client_socket.setsockopt(socket.SOL_SOCKET, socket.SO_RCVBUF, BackboneInterface.DP_IC_RCVBUF)
if not owner_interface.incoming_connection(client_socket):
try: client_socket.close()
except Exception as e: RNS.log(f"Error while closing socket for failed incoming connection: {e}", RNS.LOG_WARNING)
@ -791,7 +946,8 @@ class BackboneClientInterface(Interface):
self.rxb += len(data)
if hasattr(self, "parent_interface") and self.parent_interface != None:
self.parent_interface.rxb += len(data)
self.dp_ingress_packets += 1
self.owner.inbound(data, self)
def process_outgoing(self, data):

View file

@ -157,6 +157,11 @@ class Interface:
self.protocol_violations = 0
self.ifac_violations = 0
self.packet_filter_hits = 0
self.dp_ingress_tcount = 0
self.dp_ingress_bytes = 0
self.dp_ingress_packets = 0
self.dp_ingress_hold = None
self.dp_ingress_gated = False
self.reports_phy_stats = False
self.r_stat_rssi = None

View file

@ -44,18 +44,22 @@ from .vendor import umsgpack as umsgpack
from RNS.Interfaces.BackboneInterface import BackboneInterface
class InboundQueues:
__slots__ = ("_cond", "_queues", "_sizes", "_dropped")
__slots__ = ("_cond", "_queues", "_sizes", "_dropped", "_hw_mark")
def __init__(self, sizes):
def __init__(self, sizes, hw_mark=None):
self._cond = Condition()
self._queues = [deque() for _ in sizes]
self._sizes = sizes
self._dropped = [0 for _ in sizes]
self._hw_mark = hw_mark or max(128, BackboneInterface.DP_IC_HIGH_WM)
RNS.log(f"hw mark at {self._hw_mark}", RNS.LOG_CRITICAL)
def put(self, traffic_class, item, block=False, timeout=None):
with self._cond:
q = self._queues[traffic_class]
if len(q) >= self._sizes[traffic_class]:
q = self._queues[traffic_class]; ql = len(q)
if traffic_class == Transport.TC_DATA and ql >= self._hw_mark:
BackboneInterface._throttle_immediate(ql)
if ql >= self._sizes[traffic_class]:
self._dropped[traffic_class] += 1
raise Full
q.append(item)