Merged staging

This commit is contained in:
Mark Qvist 2026-10-24 00:20:27 +02:00
parent f3ebec9cb1
commit e24d207312
No known key found for this signature in database
25 changed files with 1123 additions and 174 deletions

View file

@ -1,3 +1,21 @@
### RNS 1.6.0
*Pending*.
**Changes**
- Added path usage tracking
- Added automatic path table culling based on traffic and usage
- Added unused path pool monitoring and handling
- Added managed storage backend using SQLite
- Added fallback to legacy file-based storage if SQLite is unavailable
- Added managed storage for known destinations and ratchets
- Added managed storage for cached announces
- Added more comprehensive metrics to status output structure
- Optimized announce rate table implementation
- Optimized packet filter hashlist implementation
- Fixed `RNodeInterface` phy stats regression on Android
- Fixed `rngit` default config path typo for `~/.config/rngit` case
### 2026-10-05: RNS 1.5.7
This maintenance release fixes a bug in the dataplane egress control handling.

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@ -40,6 +40,7 @@ from .vendor import umsgpack as umsgpack
from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey
from RNS.Cryptography import Token
from RNS.Storage import KnownDestinationsStore
class Identity:
@ -92,6 +93,7 @@ class Identity:
# Storage
known_destinations = {}
known_destinations_store = None
known_ratchets = {}
ratchet_persist_lock = threading.Lock()
@ -104,7 +106,7 @@ class Identity:
else:
with Identity.known_destinations_lock:
if not destination_hash in Identity.known_destinations:
Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0]
Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0, None, 0]
else:
entry = Identity.known_destinations[destination_hash]
entry[0] = time.time()
@ -112,6 +114,8 @@ class Identity:
entry[2] = public_key
entry[3] = app_data
Identity._kd_mark_dirty(destination_hash)
@staticmethod
def recall(target_hash, from_identity_hash=False, _no_use=False):
"""
@ -177,7 +181,12 @@ class Identity:
def save_known_destinations(background=False, recombine=False):
if recombine: RNS.log(f"Recombining known destinations from disk cache on persist is deprecated, argument ignored", RNS.LOG_WARNING)
if RNS.Transport.owner.is_connected_to_shared_instance: return
if Identity.known_destinations_store != None: Identity.flush_known_destinations()
else: Identity.save_known_destinations_legacy(background=background)
@staticmethod
def save_known_destinations_legacy(background=False):
try:
if hasattr(Identity, "saving_known_destinations"):
wait_interval = 0.2
@ -214,6 +223,72 @@ class Identity:
@staticmethod
def load_known_destinations():
if RNS.Reticulum.get_instance().is_connected_to_shared_instance:
try:
store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True)
count = 0
with Identity.known_destinations_lock:
for destination_hash, entry in store.load_all():
Identity.known_destinations[destination_hash] = entry
count += 1
store.close()
RNS.log(f"Loaded {count} known destinations from shared instance database", RNS.LOG_VERBOSE)
except Exception as e:
with Identity.known_destinations_lock: Identity.known_destinations = {}
RNS.log("Could not load known destinations from shared instance database, continuing with an empty table. The contained exception was: "+str(e), RNS.LOG_ERROR)
return
store = None
try: store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8)
except Exception as e:
RNS.log("Could not initialise known destinations database, falling back to legacy snapshot persistence. The contained exception was: "+str(e), RNS.LOG_ERROR)
Identity.known_destinations_store = None
if store != None:
Identity.known_destinations_store = store
legacy_path = RNS.Reticulum.storagepath+"/known_destinations"
if os.path.isfile(legacy_path):
try:
imported = store.import_legacy(legacy_path)
os.rename(legacy_path, legacy_path+".imported")
RNS.log(f"Imported {imported} known destinations from legacy storage into the known destinations database", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log("Could not import legacy known destinations file into the known destinations database. The contained exception was: "+str(e), RNS.LOG_ERROR)
RNS.trace_exception(e)
st = time.time()
try:
count = 0
with Identity.known_destinations_lock:
for destination_hash, entry in store.load_all():
Identity.known_destinations[destination_hash] = entry
count += 1
RNS.log(f"Loaded {count} known destinations from storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log("Error loading known destinations from storage, continuing with an empty table", RNS.LOG_ERROR)
RNS.trace_exception(e)
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
if os.path.isdir(ratchetdir) and len(os.listdir(ratchetdir)) > 0:
try:
imported = store.import_legacy_ratchets(ratchetdir, time.time()-Identity.RATCHET_EXPIRY, Identity.known_destinations)
RNS.log(f"Imported {imported} ratchets from legacy file storage", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log(f"Could not import legacy ratchet files into the known destinations database. The contained exception was: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
else: Identity.load_known_destinations_legacy()
@staticmethod
def load_known_destinations_legacy():
if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"):
st = time.time()
try:
@ -244,6 +319,7 @@ class Identity:
if destination_hash in Identity.known_destinations:
if not Identity.known_destinations[destination_hash][4] < 0:
Identity.known_destinations[destination_hash][4] = time.time()
Identity._kd_mark_dirty(destination_hash)
return True
return False
@ -253,6 +329,7 @@ class Identity:
with Identity.known_destinations_lock:
if destination_hash in Identity.known_destinations:
Identity.known_destinations[destination_hash][4] = -1
Identity._kd_mark_dirty(destination_hash)
return True
return False
@ -262,6 +339,7 @@ class Identity:
with Identity.known_destinations_lock:
if destination_hash in Identity.known_destinations:
Identity.known_destinations[destination_hash][4] = time.time()
Identity._kd_mark_dirty(destination_hash)
return True
return False
@ -327,7 +405,7 @@ class Identity:
if not is_retained and not has_path:
if not was_used and now - last_announce > RNS.Transport.UNUSED_DESTINATION_LINGER: stale.append(destination_hash)
elif unused_for > RNS.Transport.DESTINATION_TIMEOUT*1.25: stale.append(destination_hash)
elif unused_for > RNS.Transport.PATH_TIMEOUT*1.25: stale.append(destination_hash)
except Exception as e: RNS.log(f"Faulty entry for {RNS.prettyhexrep(destination_hash)} while cleaning known destinations: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
@ -336,13 +414,23 @@ class Identity:
with Identity.known_destinations_lock:
if destination_hash in Identity.known_destinations:
Identity.known_destinations.pop(destination_hash)
Identity._kd_mark_deleted(destination_hash)
if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash)
removed += 1
try:
hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_path = f"{ratchetdir}/{hexhash}"
if os.path.isfile(ratchet_path): os.unlink(ratchet_path)
except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
if not Identity.known_destinations_store:
try:
hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_path = f"{ratchetdir}/{hexhash}"
if os.path.isfile(ratchet_path): os.unlink(ratchet_path)
except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
if Identity.known_destinations_store != None:
expired_ratchets = Identity.known_destinations_store.expire_ratchets(time.time()-Identity.RATCHET_EXPIRY)
if len(expired_ratchets) > 0:
for destination_hash in expired_ratchets:
with Identity.known_destinations_lock:
if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash)
RNS.log(f"Cleaned known destinations in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
RNS.log(f"Total: {total}, stale: {len(stale)}, removed: {removed}, no path: {no_path}, never used: {never_used}, with path: {total-no_path}, used: {total-never_used}, retained: {retained}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
@ -409,33 +497,40 @@ class Identity:
@staticmethod
def _remember_ratchet(destination_hash, ratchet):
try:
if destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet:
ratchet_exists = True
# Don't re-insert existing ratchet
if Identity.known_ratchets.get(destination_hash) == ratchet: return
store = Identity.known_destinations_store
if store != None:
# Persist to store, only update in-memory cache if this
# destination already exists there.
store.remember_ratchet(destination_hash, ratchet, time.time())
if destination_hash in Identity.known_ratchets:
Identity.known_ratchets[destination_hash] = ratchet
# Legacy store: Use file system
# Shared instance client: Just keep in memory cache
else:
ratchet_exists = False
ratchet_exists = destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet
if not ratchet_exists:
RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
Identity.known_ratchets[destination_hash] = ratchet
if not RNS.Transport.owner.is_connected_to_shared_instance:
def persist_job():
with Identity.ratchet_persist_lock:
hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_data = {"ratchet": ratchet, "received": time.time()}
if not ratchet_exists:
RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
Identity.known_ratchets[destination_hash] = ratchet
if not RNS.Transport.owner.is_connected_to_shared_instance:
def persist_job():
with Identity.ratchet_persist_lock:
hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_data = {"ratchet": ratchet, "received": time.time()}
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
if not os.path.isdir(ratchetdir):
os.makedirs(ratchetdir)
if not os.path.isdir(ratchetdir): os.makedirs(ratchetdir)
outpath = f"{ratchetdir}/{hexhash}.out"
finalpath = f"{ratchetdir}/{hexhash}"
with open(outpath, "wb") as ratchet_file:
ratchet_file.write(umsgpack.packb(ratchet_data))
os.replace(outpath, finalpath)
outpath = f"{ratchetdir}/{hexhash}.out"
finalpath = f"{ratchetdir}/{hexhash}"
with open(outpath, "wb") as ratchet_file: ratchet_file.write(umsgpack.packb(ratchet_data))
os.replace(outpath, finalpath)
threading.Thread(target=persist_job, daemon=True).start()
threading.Thread(target=persist_job, daemon=True).start()
except Exception as e:
RNS.log(f"Could not persist ratchet for {RNS.prettyhexrep(destination_hash)} to storage.", RNS.LOG_ERROR)
@ -483,7 +578,43 @@ class Identity:
@staticmethod
def get_ratchet(destination_hash):
if not destination_hash in Identity.known_ratchets:
if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash]
store = Identity.known_destinations_store
if store != None:
ratchet_data = store.fetch_ratchet(destination_hash)
if ratchet_data != None:
ratchet, received = ratchet_data
if time.time() < received+Identity.RATCHET_EXPIRY:
Identity.known_ratchets[destination_hash] = ratchet
return ratchet
else: return None
else:
RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
elif RNS.Reticulum.get_instance().is_connected_to_shared_instance:
try:
# TODO: Ideally, refactor this to an RPC method instead
store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True)
ratchet_data = store.fetch_ratchet(destination_hash)
store.close()
if ratchet_data != None:
ratchet, received = ratchet_data
if time.time() < received+Identity.RATCHET_EXPIRY:
Identity.known_ratchets[destination_hash] = ratchet
return ratchet
except Exception as e: RNS.log(f"Could not fetch ratchet for {RNS.prettyhexrep(destination_hash)} from shared instance database. The contained exception was: {e}", RNS.LOG_ERROR)
return None
else:
if not os.path.isfile(RNS.Reticulum.storagepath+"/ratchets/"+RNS.hexrep(destination_hash, delimit=False)): return None
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
hexhash = RNS.hexrep(destination_hash, delimit=False)
ratchet_path = f"{ratchetdir}/{hexhash}"
@ -493,16 +624,14 @@ class Identity:
ratchet_data = umsgpack.unpackb(ratchet_file.read())
if time.time() < ratchet_data["received"]+Identity.RATCHET_EXPIRY and len(ratchet_data["ratchet"]) == Identity.RATCHETSIZE//8:
Identity.known_ratchets[destination_hash] = ratchet_data["ratchet"]
else:
return None
else: return None
except Exception as e:
RNS.log(f"An error occurred while loading ratchet data for {RNS.prettyhexrep(destination_hash)} from storage.", RNS.LOG_ERROR)
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR)
return None
if destination_hash in Identity.known_ratchets:
return Identity.known_ratchets[destination_hash]
if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash]
else:
RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
@ -614,12 +743,35 @@ class Identity:
@staticmethod
def persist_data(background=False):
if not RNS.Transport.owner.is_connected_to_shared_instance:
Identity.save_known_destinations(background=background)
if Identity.known_destinations_store != None: Identity.flush_known_destinations()
else: Identity.save_known_destinations_legacy(background=background)
@staticmethod
def exit_handler():
Identity.persist_data()
def flush_known_destinations():
store = Identity.known_destinations_store
if store == None: return False
with Identity.known_destinations_lock: upserts, deletes, ratchet_upserts, ratchet_deletes = store.take_pending()
try: return store.apply(upserts, deletes, ratchet_upserts, ratchet_deletes, Identity.known_destinations)
except Exception as e:
RNS.log(f"Could not flush known destinations to storage. The contained exception was: {e}", RNS.LOG_ERROR)
return False
@staticmethod
def _kd_mark_dirty(destination_hash):
store = Identity.known_destinations_store
if store != None: store.mark_dirty(destination_hash)
@staticmethod
def _kd_mark_deleted(destination_hash):
store = Identity.known_destinations_store
if store != None:
store.mark_deleted(destination_hash)
store.mark_ratchet_deleted(destination_hash)
@staticmethod
def exit_handler(): Identity.persist_data()
@staticmethod
def from_bytes(prv_bytes):

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@ -425,6 +425,7 @@ class Link:
self.activated_at = time.time()
self.last_proof = self.activated_at
RNS.Transport.activate_link(self)
RNS.Transport.mark_path_responsive(self.destination.hash)
RNS.log(f"Link {self} established with {self.destination}, RTT is {RNS.prettyshorttime(self.rtt)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0:

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1350,7 +1350,7 @@ class Reticulum:
filepath = self.cachepath + "/" + filename
mtime = os.path.getmtime(filepath)
age = now - mtime
if age > RNS.Transport.DESTINATION_TIMEOUT: os.unlink(filepath)
if age > RNS.Transport.PATH_TIMEOUT: os.unlink(filepath)
if background: time.sleep(0.001)
except Exception as e:
@ -1387,6 +1387,7 @@ class Reticulum:
if path == "lowest_interface_bitrate": self.rpc_return(conn, self.get_lowest_interface_bitrate())
if path == "medium_path_timeout": self.rpc_return(conn, self.get_medium_path_timeout())
if path == "link_count": self.rpc_return(conn, self.get_link_count())
if path == "link_distribution": self.rpc_return(conn, self.get_link_distribution())
if path == "active_link_count": self.rpc_return(conn, self.get_active_link_count())
if path == "packet_rssi": self.rpc_return(conn, self.get_packet_rssi(call["packet_hash"]))
if path == "packet_snr": self.rpc_return(conn, self.get_packet_snr(call["packet_hash"]))
@ -1684,6 +1685,7 @@ class Reticulum:
pql = RNS.Transport.INBOUND_PR_QUEUE_LENGTH
ilql = RNS.Transport.INBOUND_IL_QUEUE_LENGTH
tql = dql+aql+pql+ilql
ppcs = RNS.Transport.path_pool_counts()
stats = {}
stats["interfaces"] = interfaces
@ -1705,6 +1707,12 @@ class Reticulum:
stats["ptxf"] = RNS.Transport.pr_freq_tx
stats["rxpps"] = RNS.Transport.rx_pps
stats["txpps"] = RNS.Transport.tx_pps
stats["rxpkts"] = RNS.Transport.rx_packets
stats["txpkts"] = RNS.Transport.tx_packets
stats["arxpkts"] = RNS.Transport.announce_rxc
stats["atxpkts"] = RNS.Transport.announce_txc
stats["prxpkts"] = RNS.Transport.pr_rxc
stats["ptxpkts"] = RNS.Transport.pr_txc
stats["rxqt"] = qsnapshot[0]
stats["rxqd"] = qsnapshot[1][0]
stats["rxqa"] = qsnapshot[1][1]
@ -1721,6 +1729,12 @@ class Reticulum:
stats["pqpressure"] = qsnapshot[1][2]/pql if qsnapshot[1][2] else 0
stats["ilqpressure"] = qsnapshot[1][3]/ilql if qsnapshot[1][3] else 0
stats["txq"] = None
stats["tlinks"] = RNS.Transport.link_count()
stats["alinks"] = RNS.Transport.active_link_count()
stats["plinks"] = stats["tlinks"]-stats["alinks"]
stats["tpcount"] = ppcs[0]
stats["unupcount"] = ppcs[1]
stats["actpcount"] = stats["tpcount"] - stats["unupcount"]
if Reticulum.transport_enabled():
stats["transport_id"] = RNS.Transport.identity.hash
@ -1753,6 +1767,7 @@ class Reticulum:
entry = { "hash": dst_hash, "timestamp": RNS.Transport.path_table[dst_hash][0],
"via": RNS.Transport.path_table[dst_hash][1], "hops": path_hops,
"expires": RNS.Transport.path_table[dst_hash][3],
"used": RNS.Transport.path_table[dst_hash][7],
"interface": str(RNS.Transport.path_table[dst_hash][5]) }
path_table.append(entry)
@ -1767,15 +1782,10 @@ class Reticulum:
else:
rate_table = []
for dst_hash in RNS.Transport.announce_rate_table:
entry = {
"hash": dst_hash,
"last": RNS.Transport.announce_rate_table[dst_hash]["last"],
"rate_violations": RNS.Transport.announce_rate_table[dst_hash]["rate_violations"],
"blocked_until": RNS.Transport.announce_rate_table[dst_hash]["blocked_until"],
"timestamps": RNS.Transport.announce_rate_table[dst_hash]["timestamps"],
}
rate_table.append(entry)
with RNS.Transport.announce_rate_table_lock:
for dst_hash, entry in RNS.Transport.announce_rate_table.items():
entry["hash"] = dst_hash
rate_table.append(entry)
return rate_table
@ -1925,6 +1935,16 @@ class Reticulum:
else:
return RNS.Transport.active_link_count()
def get_link_distribution(self):
if self.is_connected_to_shared_instance:
rpc_connection = self.get_rpc_client()
rpc_connection.send_bytes(mp.packb({"get": "link_distribution"}))
response = mp.unpackb(rpc_connection.recv_bytes())
return response
else:
return RNS.Transport.link_distribution()
def get_packet_rssi(self, packet_hash):
if self.is_connected_to_shared_instance:
rpc_connection = self.get_rpc_client()

580
RNS/Storage.py Normal file
View file

@ -0,0 +1,580 @@
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from array import array
import hashlib
import threading
import os
import time
class AnnounceRateTable:
# Columnar store for announce rate tracking. Layout:
# _index : dict[destination_hash -> slot]
# _keys : list[slot] -> destination_hash (for swap-remove bookkeeping)
# _last : array('d') last non-violating announce time
# _blocked: array('d') blocked_until
# _viol : array('i') rate_violations
# _ts : array('d') flat ring of max_timestamps doubles per slot
# _ts_len : array('B') valid timestamp count per slot
# _ts_head: array('B') ring write cursor per slot
def __init__(self, max_timestamps=16):
assert max_timestamps <= 255, "Timestamp ring size must fit in a single byte"
self.max_ts = max_timestamps
self._index = {}
self._keys = []
self._last = array('d')
self._blocked = array('d')
self._viol = array('i')
self._ts = array('d')
self._ts_len = array('B')
self._ts_head = array('B')
self._cursor = 0
def __len__(self): return len(self._keys)
def __contains__(self, destination_hash): return destination_hash in self._index
# Perform the announce-rate check-and-update for one announce.
# Caller must hold the announce_rate_table_lock.
def check_and_update(self, destination_hash, now, target, grace, penalty):
slot = self._index.get(destination_hash)
if slot is None:
slot = len(self._keys)
self._index[destination_hash] = slot
self._keys.append(destination_hash)
self._last.append(now)
self._blocked.append(0.0)
self._viol.append(0)
self._ts.extend([0.0]*self.max_ts)
self._ts[slot*self.max_ts] = now
self._ts_len.append(1)
self._ts_head.append(1 % self.max_ts)
return False
base = slot*self.max_ts
head = self._ts_head[slot]
self._ts[base+head] = now
self._ts_head[slot] = (head+1) % self.max_ts
if self._ts_len[slot] < self.max_ts: self._ts_len[slot] += 1
current_rate = now - self._last[slot]
rate_blocked = False
if now > self._blocked[slot]:
if current_rate < target: self._viol[slot] += 1
else:
v = self._viol[slot] - 1
self._viol[slot] = v if v > 0 else 0
if self._viol[slot] > grace:
self._blocked[slot] = self._last[slot] + target + penalty
rate_blocked = True
else: self._last[slot] = now
else: rate_blocked = True
return rate_blocked
# Caller must hold the table lock
def cull(self, now, ttl, max_scan=1024, max_cull=None):
if max_cull is None: max_cull = max_scan
horizon = now - ttl
culled = 0
scanned = 0
n = len(self._keys)
while scanned < max_scan and culled < max_cull and n > 0:
if self._cursor >= n: self._cursor = 0
slot = self._cursor
if self._last[slot] < horizon and self._blocked[slot] <= now:
last_i = n - 1
stale_key = self._keys[slot]
if slot != last_i:
moved_key = self._keys[last_i]
self._keys[slot] = moved_key
self._index[moved_key] = slot
self._last[slot] = self._last[last_i]
self._blocked[slot] = self._blocked[last_i]
self._viol[slot] = self._viol[last_i]
self._ts_len[slot] = self._ts_len[last_i]
self._ts_head[slot] = self._ts_head[last_i]
f = last_i*self.max_ts; t = slot*self.max_ts
self._ts[t:t+self.max_ts] = self._ts[f:f+self.max_ts]
self._keys.pop()
self._last.pop(); self._blocked.pop(); self._viol.pop()
self._ts_len.pop(); self._ts_head.pop()
del self._ts[len(self._ts)-self.max_ts:] # drop tail block of removed slot
del self._index[stale_key]
n -= 1
culled += 1
# Do not advance cursor, re-examine the slot a moved entry landed in
else:
self._cursor = slot + 1
scanned += 1
if self._cursor >= n: self._cursor = 0
return culled
# Hold table lock, or accept a possibly inconsistent snapshot
def items(self):
for slot, key in enumerate(self._keys):
n = self._ts_len[slot]
head = self._ts_head[slot]
base = slot*self.max_ts
if n < self.max_ts: ts = list(self._ts[base:base+n])
else: ts = list(self._ts[base+head:base+self.max_ts]) + list(self._ts[base:base+head])
yield key, { "last": self._last[slot], "rate_violations": self._viol[slot],
"blocked_until": self._blocked[slot], "timestamps": ts }
class PacketHashlist:
# Bucketed byte-blob packet filter hashlist, with the primary
# design goal of memory coherence and density. All operations
# are safe to call without holding a lock.
def __init__(self, hash_width=32, bucket_count=16384):
assert bucket_count > 0 and bucket_count & (bucket_count-1) == 0, "Bucket count must be a power of two"
self.width = hash_width
self._nb = bucket_count
self._mask = bucket_count - 1
self._cur = [bytearray() for _ in range(bucket_count)]
self._prv = [bytearray() for _ in range(bucket_count)]
self._count = 0
def __len__(self):
return self._count
def __contains__(self, packet_hash):
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
return packet_hash in self._cur[bucket] or packet_hash in self._prv[bucket]
def add(self, packet_hash):
assert len(packet_hash) == self.width, "Packet hash width mismatch"
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
if packet_hash not in self._cur[bucket]:
self._cur[bucket].extend(packet_hash)
self._count += 1
def discard(self, packet_hash):
w = self.width
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
for generation in (self._cur, self._prv):
entries = generation[bucket]
pos = entries.find(packet_hash)
while pos != -1:
if pos % w == 0:
del entries[pos:pos+w]
if generation is self._cur: self._count -= 1
pos = entries.find(packet_hash, pos)
else:
pos = entries.find(packet_hash, pos+1)
def swap(self):
self._prv = self._cur
self._cur = [bytearray() for _ in range(self._nb)]
self._count = 0
# Current-generation buckets for persist operations
def buckets(self): return tuple(self._cur)
class KnownDestinationsStore:
# Write-behind SQLite storage backend for known destinations.
# The actual in-memory table is used directly for all transport
# operations; this serves purely as the store, and no read operations
# utilize the database directly.
def __init__(self, db_path, hash_width=16, read_only=False):
import sqlite3
self.sqlite3 = sqlite3
self.db_path = db_path
self.hash_width = hash_width
self.read_only = read_only
self._dirty_upsert = set()
self._dirty_delete = set()
self._ratchet_upserts = {}
self._ratchet_deletes = set()
self._flush_lock = threading.Lock()
if read_only: self._conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False, isolation_level=None)
else:
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None)
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA synchronous=NORMAL")
self._conn.execute("PRAGMA journal_size_limit=8388608")
self.__create_schema()
def close(self):
if self._conn != None:
self._conn.close()
self._conn = None
def __create_schema(self):
self._conn.execute("""CREATE TABLE IF NOT EXISTS known_destinations (
destination_hash BLOB PRIMARY KEY,
last_heard REAL NOT NULL,
packet_hash BLOB NOT NULL,
public_key BLOB NOT NULL,
app_data BLOB,
used REAL NOT NULL DEFAULT 0,
stamp BLOB,
stamp_value INTEGER,
identity_hash BLOB NOT NULL)""")
self._conn.execute("""CREATE TABLE IF NOT EXISTS ratchets (
destination_hash BLOB PRIMARY KEY,
ratchet BLOB NOT NULL,
received REAL NOT NULL)""")
self._conn.execute("PRAGMA user_version = 1")
self._conn.commit()
def __entry_row(self, destination_hash, entry):
stamp = entry[5] if len(entry) > 5 else None
stamp_value = entry[6] if len(entry) > 6 else 0
identity_hash = hashlib.sha256(entry[2]).digest()[:self.hash_width]
return (destination_hash, entry[0], entry[1], entry[2], entry[3], entry[4], stamp, stamp_value, identity_hash)
# Caller must hold Identity.known_destinations_lock
def mark_dirty(self, destination_hash):
if self.read_only: return
self._dirty_delete.discard(destination_hash)
self._dirty_upsert.add(destination_hash)
# Caller must hold Identity.known_destinations_lock
def mark_deleted(self, destination_hash):
if self.read_only: return
self._dirty_upsert.discard(destination_hash)
self._dirty_delete.add(destination_hash)
# Caller must hold Identity.known_destinations_lock
def take_pending(self):
if self.read_only: return set(), set(), {}, set()
upserts = self._dirty_upsert
deletes = self._dirty_delete
ratchet_upserts = self._ratchet_upserts
ratchet_deletes = self._ratchet_deletes
self._dirty_upsert = set()
self._dirty_delete = set()
self._ratchet_upserts = {}
self._ratchet_deletes = set()
return upserts, deletes, ratchet_upserts, ratchet_deletes
# May be called without any locks held
def apply(self, upserts, deletes, ratchet_upserts, ratchet_deletes, known_destinations):
if self.read_only: return 0
if len(upserts) == 0 and len(deletes) == 0 and len(ratchet_upserts) == 0 and len(ratchet_deletes) == 0: return 0
with self._flush_lock:
rows = []
removables = []
ratchet_rows = []
ratchet_removables = []
for destination_hash in upserts:
entry = known_destinations.get(destination_hash)
if entry != None: rows.append(self.__entry_row(destination_hash, entry))
for destination_hash in deletes:
if destination_hash not in known_destinations: removables.append((destination_hash,))
for destination_hash, ratchet_entry in ratchet_upserts.items():
# Ratchets of purged destinations must be removed as well
if destination_hash in known_destinations: ratchet_rows.append((destination_hash, *ratchet_entry))
else: ratchet_removables.append((destination_hash,))
for destination_hash in ratchet_deletes:
if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,))
try:
self._conn.execute("BEGIN")
for i in range(0, len(rows), 4096): self._conn.executemany("INSERT INTO known_destinations VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, identity_hash=excluded.identity_hash", rows[i:i+4096])
for i in range(0, len(removables), 4096): self._conn.executemany("DELETE FROM known_destinations WHERE destination_hash=?", removables[i:i+4096])
for i in range(0, len(ratchet_rows), 4096): self._conn.executemany("INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received", ratchet_rows[i:i+4096])
for i in range(0, len(ratchet_removables), 4096): self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", ratchet_removables[i:i+4096])
self._conn.commit()
except Exception:
# Re-mark everything as dirty, so the changes are retried
# on the next flush, then propagate.
self._dirty_upsert |= {row[0] for row in rows}
self._dirty_delete |= {row[0] for row in removables}
self._ratchet_upserts |= {row[0]: row[1:] for row in ratchet_rows}
self._ratchet_deletes |= {row[0] for row in ratchet_removables}
self._conn.rollback()
raise
return len(rows) + len(removables) + len(ratchet_rows) + len(ratchet_removables)
def load_all(self):
cursor = self._conn.execute("SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value FROM known_destinations")
while True:
rows = cursor.fetchmany(4096)
if not rows: break
for row in rows: yield row[0], [row[1], row[2], row[3], row[4], row[5], row[6], row[7]]
# Caller must hold Identity.known_destinations_lock.
def remember_ratchet(self, destination_hash, ratchet, now):
if self.read_only: return False
pending = self._ratchet_upserts.get(destination_hash)
if pending != None:
if pending[0] == ratchet:
self._ratchet_deletes.discard(destination_hash)
return False
else:
row = self._conn.execute("SELECT ratchet FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
if row != None and row[0] == ratchet:
# Discard any pending delete mark queued
# before this ratchet arrived
self._ratchet_deletes.discard(destination_hash)
return False
self._ratchet_upserts[destination_hash] = (ratchet, now)
self._ratchet_deletes.discard(destination_hash)
return True
def fetch_ratchet(self, destination_hash):
pending = self._ratchet_upserts.get(destination_hash)
if pending != None: return pending[0], pending[1]
if destination_hash in self._ratchet_deletes: return None
row = self._conn.execute("SELECT ratchet, received FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
if row != None: return row[0], row[1]
return None
# Caller must hold Identity.known_destinations_lock
def mark_ratchet_deleted(self, destination_hash):
if self.read_only: return
self._ratchet_upserts.pop(destination_hash, None)
self._ratchet_deletes.add(destination_hash)
def expire_ratchets(self, cutoff):
if self.read_only: return []
with self._flush_lock:
# Drop expired entries in pending as well
expired = set(row[0] for row in self._conn.execute("SELECT destination_hash FROM ratchets WHERE received<?", (cutoff,)))
for destination_hash, entry in list(self._ratchet_upserts.items()):
if entry[1] < cutoff:
expired.add(destination_hash)
self._ratchet_upserts.pop(destination_hash, None)
expired.difference_update(self._ratchet_deletes)
if len(expired) > 0:
self._conn.execute("BEGIN")
for i in range(0, len(expired), 4096):
self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", [(h,) for h in list(expired)[i:i+4096]])
self._conn.commit()
return list(expired)
def load_ratchets(self):
cursor = self._conn.execute("SELECT destination_hash, ratchet, received FROM ratchets")
while True:
rows = cursor.fetchmany(4096)
if not rows: break
for row in rows: yield row[0], row[1], row[2]
def import_legacy_ratchets(self, dir_path, expiry_cutoff, known_destinations, ratchet_width=32):
import RNS
from .vendor import umsgpack
imported = 0
for filename in os.listdir(dir_path):
file_path = os.path.join(dir_path, filename)
try:
if not os.path.isfile(file_path): continue
with open(file_path, "rb") as rf:
ratchet_data = umsgpack.unpackb(rf.read())
destination_hash = bytes.fromhex(filename)
ratchet = ratchet_data["ratchet"]; received = ratchet_data["received"]
if len(ratchet) != ratchet_width: raise ValueError("Invalid ratchet width")
if received < expiry_cutoff: raise ValueError("Ratchet expired")
if destination_hash not in known_destinations: raise ValueError("Destination unknown")
self._ratchet_upserts[destination_hash] = (ratchet, received)
imported += 1
os.unlink(file_path)
except Exception as e:
RNS.log(f"Could not import ratchet for {filename}: {e}", RNS.LOG_WARNING)
try: os.unlink(file_path)
except: pass
pending = self._ratchet_upserts
self._ratchet_upserts = {}
self.apply(set(), set(), pending, set(), known_destinations)
return imported
def import_legacy(self, path):
from .vendor import umsgpack
with open(path, "rb") as file: legacy = umsgpack.load(file)
rows = []
now = time.time()
for destination_hash, entry in legacy.items():
if len(entry) < 5: continue
entry = list(entry)
while len(entry) < 7: entry.append(None if len(entry) == 5 else 0)
if entry[0] == None or entry[0] <= 0: entry[0] = now
rows.append(self.__entry_row(destination_hash, entry))
self._conn.execute("BEGIN")
for i in range(0, len(rows), 4096): self._conn.executemany("INSERT OR REPLACE INTO known_destinations VALUES (?,?,?,?,?,?,?,?,?)", rows[i:i+4096])
self._conn.commit()
return len(rows)
class AnnounceCache:
def __init__(self, db_path):
import sqlite3
self.db_path = db_path
self._pending = {}
self._flush_lock = threading.Lock()
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None)
self._conn.execute("PRAGMA journal_mode=WAL")
self._conn.execute("PRAGMA synchronous=NORMAL")
self._conn.execute("PRAGMA journal_size_limit=8388608")
self._conn.execute("PRAGMA auto_vacuum=INCREMENTAL")
self.__create_schema()
def __create_schema(self):
self._conn.execute("""CREATE TABLE IF NOT EXISTS cached_announces (
packet_hash BLOB PRIMARY KEY,
received REAL NOT NULL,
raw BLOB NOT NULL,
interface_reference TEXT)""")
self._conn.execute("PRAGMA user_version = 1")
def store(self, packet_hash, raw, interface_reference):
self._pending[packet_hash] = (time.time(), raw, interface_reference)
def pending_count(self):
return len(self._pending)
def fetch(self, packet_hash):
pending = self._pending.get(packet_hash)
if pending != None: return pending[1], pending[2]
row = self._conn.execute("SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=?", (packet_hash,)).fetchone()
if row != None: return row[0], row[1]
return None
def flush(self):
with self._flush_lock: return self.__apply_pending()
def __apply_pending(self):
pending = self._pending
self._pending = {}
if len(pending) == 0: return 0
try:
self._conn.execute("BEGIN")
items = list(pending.items())
for i in range(0, len(items), 4096):
self._conn.executemany("INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference",
[(packet_hash, *entry) for packet_hash, entry in items[i:i+4096]])
self._conn.commit()
except Exception:
# Re-mark everything as pending, so the changes
# are retried on the next flush, then propagate.
self._pending.update(pending)
self._conn.rollback()
raise
return len(pending)
def clean(self, active_hashes):
with self._flush_lock:
self.__apply_pending()
try:
self._conn.execute("BEGIN")
self._conn.execute("DROP TABLE IF EXISTS temp.active_announces")
self._conn.execute("CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY)")
active_list = list(active_hashes)
for i in range(0, len(active_list), 4096):
self._conn.executemany("INSERT OR IGNORE INTO active_announces VALUES (?)", [(h,) for h in active_list[i:i+4096]])
changes_before = self._conn.total_changes
self._conn.execute("DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces)")
removed = self._conn.total_changes - changes_before
self._conn.commit()
except Exception:
self._conn.rollback()
raise
if removed > 0: self._conn.execute("PRAGMA incremental_vacuum")
return removed
def import_legacy(self, dir_path):
from .vendor import umsgpack
imported = 0
for filename in os.listdir(dir_path):
file_path = os.path.join(dir_path, filename)
try:
if not os.path.isfile(file_path): continue
with open(file_path, "rb") as file:
cached_data = umsgpack.unpackb(file.read())
packet_hash = bytes.fromhex(filename)
interface_reference = cached_data[1] if len(cached_data) > 1 else None
self.store(packet_hash, cached_data[0], interface_reference)
imported += 1
os.unlink(file_path)
except Exception:
try: os.unlink(file_path)
except Exception: pass
self.flush()
return imported
def close(self):
if self._conn != None:
self._conn.close()
self._conn = None

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@ -42,6 +42,7 @@ from queue import Full, Empty
from threading import Lock, Condition
from .vendor import umsgpack as umsgpack
from RNS.Interfaces.BackboneInterface import BackboneInterface
from RNS.Storage import AnnounceRateTable, PacketHashlist, AnnounceCache
class InboundQueues:
__slots__ = ("_cond", "_queues", "_sizes", "_dropped", "_hw_mark")
@ -123,7 +124,6 @@ class Transport:
PATHFINDER_R = 1 # Retransmit retries
PATHFINDER_G = 5 # Retry grace period
PATHFINDER_RW = 0.5 # Random window for announce rebroadcast
PATHFINDER_E = 60*60*24*7 # Path expiration of one week
AP_PATH_TIME = 60*60*24 # Path expiration of one day for Access Point paths
ROAMING_PATH_TIME = 60*60*6 # Path expiration of 6 hours for Roaming paths
@ -152,10 +152,14 @@ class Transport:
LINK_TIMEOUT = RNS.Link.STALE_TIME * 1.25
ALLOW_LINK_PATH_REBALANCE = True
REVERSE_TIMEOUT = 8*60 # Reverse table entries are removed after 8 minutes
DESTINATION_TIMEOUT = 60*60*24*7 # Destination table entries are removed if unused for one week
UNUSED_DESTINATION_LINGER = 6*60 # Linger time for pathless and never used destinations
PATH_TIMEOUT = 60*60*24*7 # Path table entry retainment after last cryptographically verifiable liveness event
UNUSED_PATH_LINGER = 60*60*1.75 # Linger time for never used paths
UNUSED_PATH_LINGER_LEAF = 60*5 # Linger time for never used paths on non-transport instances
UNUSED_DESTINATION_LINGER = 60*6 # Linger time for pathless and never used destinations
PATH_POOL_UNUSED_TARGET = 2048 # Max unused path table pool target, unused destinations are culled to reach it
TUNNEL_TIMEOUT = 60*60*8 # Tunnel table entries are removed if unused for eight hours
TUNNEL_PATH_TIMEOUT = 60*60*8 # Tunnel path table entries are removed if unused for eight hours
RATE_TABLE_RETAIN = 60*60 # Default announce rate table retainment, increased dynamically
MAX_RECEIPTS = 1024 # Maximum number of receipts to keep track of
MAX_RATE_TIMESTAMPS = 16 # Maximum number of announce timestamps to keep per destination
PERSIST_RANDOM_BLOBS = 32 # Maximum number of random blobs per destination to persist to disk
@ -169,9 +173,8 @@ class Transport:
active_links = [] # Links that are active
pending_links_map = {} # Link ID hash map of pending links
active_links_map = {} # Link ID hash map of active links
packet_hashlist = set() # A list of packet hashes for duplicate detection
packet_hashlist_prev = set()
receipts = [] # Receipts of all outgoing packets for proof processing
packet_hashlist = PacketHashlist(hash_width=RNS.Identity.HASHLENGTH//8)
# Notes on memory usage: 1 megabyte of memory can store approximately
# 55.100 path table entries or approximately 22.300 link table entries.
@ -183,11 +186,11 @@ class Transport:
held_announces = {} # A table containing temporarily held announce-table entries
announce_handlers = [] # A table storing externally registered announce handlers
tunnels = {} # A table storing tunnels to other transport instances
announce_rate_table = {} # A table for keeping track of announce rates
path_requests = {} # A table for storing path request timestamps
inflight_path_requests = {} # A table for tracking in-flight path request
path_states = {} # A table for keeping track of path states
blackholed_identities = {} # A table for keeping track of blackholed identities
announce_rate_table = AnnounceRateTable(max_timestamps=MAX_RATE_TIMESTAMPS)
discovery_path_requests = {} # A table for keeping track of path requests on behalf of other nodes
discovery_pr_tags = set() # A table for keeping track of tagged path requests
@ -260,20 +263,36 @@ class Transport:
known_destinations_interval = 5*60
tables_last_culled = 0.0
tables_cull_interval = 5.0
rate_table_last_culled = 0.0
rate_table_cull_interval = 5.0
rate_table_cull_chunk = 1024
pool_size_last_checked = 0.0
pool_size_check_interval = 5*60
kd_flush_interval = 10.0
announce_cache = None
ac_flush_interval = 15.0
ac_last_flush = 0.0
announce_cache_max_pending = 10000
kd_last_flush = 0.0
interface_last_jobs = 0.0
interface_jobs_interval = 5.0
last_mgmt_announce = 0
mgmt_announce_interval = 2*60*60
blackhole_last_checked = 0
blackhole_check_interval = 60
unused_path_linger = None
interface_announcer = None
discovery_handler = None
blackhole_updater = None
traffic_rxb = 0
traffic_txb = 0
announce_rxc = 0
announce_txc = 0
announce_rxb = 0
announce_txb = 0
pr_rxc = 0
pr_txc = 0
pr_rxb = 0
pr_txb = 0
speed_rx = 0
@ -299,6 +318,7 @@ class Transport:
_identity = None
_should_run = True
_unused_path_count_estimate = 0
@staticmethod
def internal_identity(): return Transport._identity
@ -336,6 +356,12 @@ class Transport:
if RNS.Reticulum.local_hops_delta(): Transport.local_hops_delta = (ord(os.urandom(1))%6)+2
if not RNS.Reticulum.transport_enabled() or Transport.owner.is_connected_to_shared_instance:
Transport.unused_path_linger = Transport.UNUSED_PATH_LINGER_LEAF
else:
Transport.unused_path_linger = Transport.UNUSED_PATH_LINGER
RNS.log(f"Unused path linger time is {RNS.prettytime(Transport.unused_path_linger)}", RNS.LOG_DEBUG)
packet_hashlist_path = RNS.Reticulum.storagepath+"/packet_hashlist.raw"
if RNS.Reticulum.transport_enabled() and not Transport.owner.is_connected_to_shared_instance:
if os.path.isfile(packet_hashlist_path):
@ -393,6 +419,9 @@ class Transport:
# Defer cleaning known destinations
Transport.destinations_last_cleaned = time.time()
# Defer path table unused pool cull
Transport.pool_size_last_checked = time.time()
# Defer sending management announces for 15 seconds
Transport.last_mgmt_announce = time.time() - Transport.mgmt_announce_interval + 15
@ -401,6 +430,8 @@ class Transport:
threading.Thread(target=Transport.count_traffic_loop, daemon=True).start()
# Load transport-related data
if not Transport.owner.is_connected_to_shared_instance: Transport.init_announce_cache()
if RNS.Reticulum.transport_enabled():
path_table_path = RNS.Reticulum.storagepath+"/destination_table"
tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels"
@ -412,6 +443,7 @@ class Transport:
serialised_destinations = umsgpack.unpackb(file.read())
file.close()
now = time.time()
for serialised_entry in serialised_destinations:
destination_hash = serialised_entry[0]
@ -423,6 +455,7 @@ class Transport:
random_blobs = serialised_entry[5]
receiving_interface = Transport.find_interface_from_hash(serialised_entry[6])
announce_packet = Transport.get_cached_packet(serialised_entry[7], packet_type="announce")
used = serialised_entry[8] if len(serialised_entry) > 8 else now
blackholed = False
if len(Transport.blackholed_identities) > 0:
@ -438,7 +471,10 @@ class Transport:
# increased hop-count.
announce_packet.hops += 1
with Transport.path_table_lock:
Transport.path_table[destination_hash] = [timestamp, received_from, hops, expires, random_blobs, receiving_interface, announce_packet.packet_hash]
Transport.path_table[destination_hash] = [timestamp, received_from, hops, expires,
random_blobs, receiving_interface,
announce_packet.packet_hash, used]
RNS.log("Loaded path table entry for "+RNS.prettyhexrep(destination_hash)+" from storage", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
else:
RNS.log("Could not reconstruct path table entry from storage for "+RNS.prettyhexrep(destination_hash), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
@ -466,6 +502,7 @@ class Transport:
serialised_tunnels = umsgpack.unpackb(file.read())
file.close()
now = time.time()
for serialised_tunnel in serialised_tunnels:
tunnel_id = serialised_tunnel[IDX_TT_TUNNEL_ID]
interface_hash = serialised_tunnel[IDX_TT_IF]
@ -482,6 +519,7 @@ class Transport:
random_blobs = list(set(serialised_entry[5]))
receiving_interface = Transport.find_interface_from_hash(serialised_entry[6])
announce_packet = Transport.get_cached_packet(serialised_entry[7], packet_type="announce")
used = serialised_entry[8] if len(serialised_entry) > 8 else now
if announce_packet != None:
if not announce_packet.unpack(): continue
@ -491,7 +529,7 @@ class Transport:
# increased hop-count.
announce_packet.hops += 1
tunnel_path = [timestamp, received_from, hops, path_expires, random_blobs, receiving_interface, announce_packet.packet_hash]
tunnel_path = [timestamp, received_from, hops, path_expires, random_blobs, receiving_interface, announce_packet.packet_hash, used]
tunnel_paths[destination_hash] = tunnel_path
if len(tunnel_paths) > 0:
@ -543,6 +581,9 @@ class Transport:
with Transport.interfaces_lock:
if not interface in Transport.interfaces:
Transport.interfaces.append(interface)
if interface.announce_rate_target and interface.announce_rate_target > Transport.RATE_TABLE_RETAIN:
Transport.RATE_TABLE_RETAIN = interface.announce_rate_target
RNS.log(f"Increased announce rate table retainment time to {RNS.prettytime(Transport.RATE_TABLE_RETAIN)}", RNS.LOG_DEBUG)
@staticmethod
def remove_interface(interface):
@ -828,10 +869,9 @@ class Transport:
Transport.announces_last_checked = time.time()
# Cull the packet hashlist if it has reached its max size
# Swap the packet hashlist generations if it has reached its max size
if len(Transport.packet_hashlist) > Transport.hashlist_maxsize//2:
Transport.packet_hashlist_prev = Transport.packet_hashlist
Transport.packet_hashlist = set()
Transport.packet_hashlist.swap()
# Cull invalidated path requests
if time.time() > Transport.pending_prs_last_checked+Transport.pending_prs_check_interval:
@ -852,6 +892,24 @@ class Transport:
Transport.discovery_pr_tags_prev = Transport.discovery_pr_tags
Transport.discovery_pr_tags = set()
# Incrementally cull stale announce rate table entries
if time.time() > Transport.rate_table_last_culled+Transport.rate_table_cull_interval:
with Transport.announce_rate_table_lock:
Transport.announce_rate_table.cull(time.time(), Transport.RATE_TABLE_RETAIN,
Transport.rate_table_cull_chunk, Transport.rate_table_cull_chunk)
Transport.rate_table_last_culled = time.time()
# Flush pending known destinations to storage
if RNS.Identity.known_destinations_store != None and time.time() > Transport.kd_last_flush+Transport.kd_flush_interval:
Transport.kd_last_flush = time.time()
RNS.Identity.flush_known_destinations()
# Flush pending cached announces to storage
if Transport.announce_cache != None and (time.time() > Transport.ac_last_flush+Transport.ac_flush_interval or Transport.announce_cache.pending_count() > Transport.announce_cache_max_pending):
Transport.ac_last_flush = time.time()
Transport.announce_cache.flush()
if time.time() > Transport.tables_last_culled + Transport.tables_cull_interval:
# Remove unneeded path state entries
stale_path_states = []
@ -957,25 +1015,42 @@ class Transport:
# Cull the path table
stale_paths = []
with Transport.path_table_lock:
now = time.time()
check_unused = now > Transport.pool_size_last_checked+Transport.pool_size_check_interval
unused_paths = []
for destination_hash in Transport.path_table:
destination_entry = Transport.path_table[destination_hash]
attached_interface = destination_entry[IDX_PT_RVCD_IF]
path_entry = Transport.path_table[destination_hash]
attached_interface = path_entry[IDX_PT_RVCD_IF]
path_timestamp = path_entry[IDX_PT_TIMESTAMP]
path_expiry = path_entry[IDX_PT_EXPIRES]
path_used = path_entry[IDX_PT_USED]
if attached_interface != None and hasattr(attached_interface, "mode") and attached_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT:
destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.AP_PATH_TIME
elif attached_interface != None and hasattr(attached_interface, "mode") and attached_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING:
destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.ROAMING_PATH_TIME
else:
destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.DESTINATION_TIMEOUT
if check_unused and not path_used: unused_paths.append((path_expiry, destination_hash))
if time.time() > destination_expiry:
if now > path_expiry:
stale_paths.append(destination_hash)
should_collect = True
RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
elif not attached_interface in Transport.interfaces:
stale_paths.append(destination_hash)
should_collect = True
RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
elif path_used == 0 and now > path_timestamp + Transport.unused_path_linger:
stale_paths.append(destination_hash)
should_collect = True
RNS.log(f"Removing never used path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
if len(unused_paths) > Transport.PATH_POOL_UNUSED_TARGET:
cull_count = len(unused_paths) - Transport.PATH_POOL_UNUSED_TARGET
RNS.log(f"Culling {cull_count} never used path table entries to reach pool size target {Transport.PATH_POOL_UNUSED_TARGET}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
unused_paths.sort()
for cull_entry in unused_paths[-cull_count:]:
stale_paths.append(cull_entry[1])
if check_unused:
Transport._unused_path_count_estimate = len(unused_paths)
Transport.pool_size_last_checked = time.time()
# Cull the pending path requests table
stale_path_requests = []
@ -1387,6 +1462,7 @@ class Transport:
return False
outbound_interface = path_entry[IDX_PT_RVCD_IF]
path_entry[IDX_PT_USED] = outbound_time
# If there's more than one hop to the destination, and we know
# a path, we insert the packet into transport by adding the next
@ -1403,7 +1479,6 @@ class Transport:
new_raw += packet.raw[2:]
packet_sent(packet)
Transport.transmit(outbound_interface, new_raw)
path_entry[IDX_PT_TIMESTAMP] = time.time()
sent = True
# In the special case where we are connected to a local shared
@ -1423,7 +1498,6 @@ class Transport:
new_raw += packet.raw[2:]
packet_sent(packet)
Transport.transmit(outbound_interface, new_raw)
path_entry[IDX_PT_TIMESTAMP] = time.time()
sent = True
# If none of the above applies, we know the destination is
@ -1597,8 +1671,8 @@ class Transport:
else: raw = Transport.mangle_hops(packet.raw, Transport.local_hops_delta, transport_insert=packet.header_type==RNS.Packet.HEADER_1)
Transport.transmit(interface, raw)
if packet.packet_type == RNS.Packet.ANNOUNCE: interface.sent_announce(size=len(raw))
if packet.destination.type == RNS.Destination.PLAIN and packet.is_outbound_pr: interface.sent_path_request(size=len(raw))
if packet.packet_type == RNS.Packet.ANNOUNCE: interface.sent_announce(size=len(raw)); Transport.announce_txc += 1
if packet.destination.type == RNS.Destination.PLAIN and packet.is_outbound_pr: interface.sent_path_request(size=len(raw)); Transport.pr_txc += 1
packet_sent(packet)
sent = True
@ -1665,7 +1739,7 @@ class Transport:
if packet.receiving_interface: packet.receiving_interface.protocol_violation("Announce packet with GROUP type")
return False
if not packet.packet_hash in Transport.packet_hashlist and not packet.packet_hash in Transport.packet_hashlist_prev: return True
if not packet.packet_hash in Transport.packet_hashlist: return True
else:
if packet.packet_type == RNS.Packet.ANNOUNCE:
if packet.destination_type == RNS.Destination.SINGLE:
@ -1809,8 +1883,9 @@ class Transport:
return packet.receiving_interface.protocol_violation(f"Invalid announce signature for {RNS.prettyhexrep(packet.destination_hash)}") if packet.receiving_interface else None
elif packet.receiving_interface != None: packet.receiving_interface.received_announce(size=len(packet.raw))
announced_destination_known = packet.destination_hash in Transport.path_table
Transport.announce_rxc += 1
announced_destination_known = packet.destination_hash in Transport.path_table
if not announced_destination_known:
# This is an unknown destination, and we'll apply
# potential ingress limiting. Already known
@ -1856,6 +1931,7 @@ class Transport:
return
if packet.receiving_interface: packet.receiving_interface.received_path_request(size=len(raw))
Transport.pr_rxc += 1
if interface.should_ingress_limit_pr():
traffic_class = Transport.TC_INGRESS_LIMITED
@ -2110,7 +2186,7 @@ class Transport:
if Transport.local_hops_delta != 0 and from_local_client and not to_local_client: new_raw = Transport.mangle_hops(new_raw, Transport.local_hops_delta)
Transport.transmit(outbound_interface, new_raw)
path_entry[IDX_PT_TIMESTAMP] = time.time()
path_entry[IDX_PT_USED] = time.time()
else:
# TODO: There should probably be some kind of REJECT
@ -2159,7 +2235,11 @@ class Transport:
new_raw += struct.pack("!B", packet.hops if not from_local_client or instance_local_link or Transport.local_hops_delta == 0 else Transport.local_hops_delta)
new_raw += packet.raw[2:]
Transport.transmit(outbound_interface, new_raw)
Transport.link_table[packet.destination_hash][IDX_LT_TIMESTAMP] = time.time()
active_time = time.time()
Transport.link_table[packet.destination_hash][IDX_LT_TIMESTAMP] = active_time
link_path_entry = Transport.path_table.get(Transport.link_table[packet.destination_hash][IDX_LT_DSTHASH])
if link_path_entry: link_path_entry[IDX_PT_USED] = active_time
else: RNS.log(f"No-outbound return on link packet from {packet.receiving_interface}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
@ -2213,27 +2293,30 @@ class Transport:
if local_and_hops_condition:
announce_emitted = Transport.announce_emitted(packet)
path_used = 0
random_blobs = []
random_blob = packet.data[RNS.Identity.KEYSIZE//8+RNS.Identity.NAME_HASH_LENGTH//8:RNS.Identity.KEYSIZE//8+RNS.Identity.NAME_HASH_LENGTH//8+10]
if not random_blob: return packet.receiving_interface.protocol_violation("No random blob in data") if packet.receiving_interface else None
with Transport.inbound_announce_lock:
announced_destination_known = packet.destination_hash in Transport.path_table
existing_path_entry = Transport.path_table.get(packet.destination_hash)
announced_destination_known = existing_path_entry != None
if not announced_destination_known:
# If this destination is unknown in our table
# we should add it
should_add = True
else:
random_blobs = Transport.path_table[packet.destination_hash][IDX_PT_RANDBLOBS]
current_gravity = Transport.path_table[packet.destination_hash][IDX_PT_RVCD_IF].gravity
random_blobs = existing_path_entry[IDX_PT_RANDBLOBS]
current_gravity = existing_path_entry[IDX_PT_RVCD_IF].gravity
path_used = existing_path_entry[IDX_PT_USED]
announce_gravity = packet.receiving_interface.gravity if packet.receiving_interface != None else None
# If we already have a path to the announced destination,
# but a more recently emitted announce arrives with a hop
# count equal to or less than the existing path, we will
# update our tables.
if packet.hops <= Transport.path_table[packet.destination_hash][IDX_PT_HOPS]:
if packet.hops <= existing_path_entry[IDX_PT_HOPS]:
path_timebase = Transport.timebase_from_random_blobs(random_blobs)
if not random_blob in random_blobs and announce_emitted > path_timebase:
Transport.mark_path_unknown_state(packet.destination_hash)
@ -2255,7 +2338,7 @@ class Transport:
# ignore it, unless the path is expired, or
# the emission timestamp is more recent.
now = time.time()
path_expires = Transport.path_table[packet.destination_hash][IDX_PT_EXPIRES]
path_expires = existing_path_entry[IDX_PT_EXPIRES]
path_announce_emitted = 0
for path_random_blob in random_blobs:
@ -2302,33 +2385,10 @@ class Transport:
rate_blocked = False
if packet.context != RNS.Packet.PATH_RESPONSE and packet.receiving_interface.announce_rate_target != None:
with Transport.announce_rate_table_lock:
if not packet.destination_hash in Transport.announce_rate_table:
rate_entry = { "last": now, "rate_violations": 0, "blocked_until": 0, "timestamps": [now]}
Transport.announce_rate_table[packet.destination_hash] = rate_entry
else:
rate_entry = Transport.announce_rate_table[packet.destination_hash]
rate_entry["timestamps"].append(now)
while len(rate_entry["timestamps"]) > Transport.MAX_RATE_TIMESTAMPS:
rate_entry["timestamps"].pop(0)
current_rate = now - rate_entry["last"]
if now > rate_entry["blocked_until"]:
if current_rate < packet.receiving_interface.announce_rate_target: rate_entry["rate_violations"] += 1
else: rate_entry["rate_violations"] = max(0, rate_entry["rate_violations"]-1)
if rate_entry["rate_violations"] > packet.receiving_interface.announce_rate_grace:
rate_target = packet.receiving_interface.announce_rate_target
rate_penalty = packet.receiving_interface.announce_rate_penalty
rate_entry["blocked_until"] = rate_entry["last"] + rate_target + rate_penalty
rate_blocked = True
else:
rate_entry["last"] = now
else: rate_blocked = True
rate_blocked = Transport.announce_rate_table.check_and_update(packet.destination_hash, now,
packet.receiving_interface.announce_rate_target,
packet.receiving_interface.announce_rate_grace,
packet.receiving_interface.announce_rate_penalty)
retries = 0
announce_hops = packet.hops
local_rebroadcasts = 0
@ -2336,13 +2396,7 @@ class Transport:
attached_interface = None
retransmit_timeout = now + (RNS.rand() * Transport.PATHFINDER_RW)
if hasattr(packet.receiving_interface, "mode") and packet.receiving_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT:
expires = now + Transport.AP_PATH_TIME
elif hasattr(packet.receiving_interface, "mode") and packet.receiving_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING:
expires = now + Transport.ROAMING_PATH_TIME
else:
expires = now + Transport.PATHFINDER_E
expires = now + Transport.path_timeout_from_interface(packet.receiving_interface)
if not random_blob in random_blobs:
random_blobs.append(random_blob)
@ -2455,8 +2509,21 @@ class Transport:
new_announce.send()
if not Transport.owner.is_connected_to_shared_instance: Transport.cache(packet, force_cache=True, packet_type="announce")
path_table_entry = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet.packet_hash]
with Transport.path_table_lock: Transport.path_table[packet.destination_hash] = path_table_entry
if rate_blocked:
# Rate-limited announces do not update liveness
# metrics, only the essential routing data.
with Transport.path_table_lock:
rate_limited_path_entry = Transport.path_table.get(packet.destination_hash)
if rate_limited_path_entry != None:
rate_limited_path_entry[IDX_PT_NEXT_HOP] = received_from
rate_limited_path_entry[IDX_PT_HOPS] = announce_hops
rate_limited_path_entry[IDX_PT_RANDBLOBS] = random_blobs
rate_limited_path_entry[IDX_PT_RVCD_IF] = packet.receiving_interface
rate_limited_path_entry[IDX_PT_PACKET] = packet.packet_hash
else:
path_table_entry = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet.packet_hash, path_used]
with Transport.path_table_lock: Transport.path_table[packet.destination_hash] = path_table_entry
Transport.mark_path_unknown_state(packet.destination_hash)
RNS.log("Destination "+RNS.prettyhexrep(packet.destination_hash)+" is now "+str(announce_hops)+" hops away via "+RNS.prettyhexrep(received_from)+" on "+str(packet.receiving_interface), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
if packet.destination_hash in Transport.path_requests:
@ -2464,13 +2531,13 @@ class Transport:
# If the receiving interface is a tunnel, we add the
# announce to the tunnels table
if hasattr(packet.receiving_interface, "tunnel_id") and packet.receiving_interface.tunnel_id != None:
if not rate_blocked and hasattr(packet.receiving_interface, "tunnel_id") and packet.receiving_interface.tunnel_id != None:
with Transport.tunnels_lock:
if not packet.receiving_interface.tunnel_id in Transport.tunnels: RNS.log(f"Tunnel ID for {packet.receiving_interface} was not found in tunnel table", RNS.LOG_WARNING)
else:
tunnel_entry = Transport.tunnels[packet.receiving_interface.tunnel_id]
paths = tunnel_entry[IDX_TT_PATHS]
paths[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, None, packet.packet_hash]
paths[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, None, packet.packet_hash, path_used]
expires = time.time() + Transport.TUNNEL_TIMEOUT
tunnel_entry[IDX_TT_EXPIRES] = expires
RNS.log("Path to "+RNS.prettyhexrep(packet.destination_hash)+" associated with tunnel "+RNS.prettyhexrep(packet.receiving_interface.tunnel_id), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
@ -2590,8 +2657,7 @@ class Transport:
# to another path, we remove this packet hash from
# the filter hashlist so the link can receive the
# packet when it finally arrives over another path.
while packet.packet_hash in Transport.packet_hashlist: Transport.packet_hashlist.remove(packet.packet_hash)
while packet.packet_hash in Transport.packet_hashlist_prev: Transport.packet_hashlist_prev.remove(packet.packet_hash)
Transport.packet_hashlist.discard(packet.packet_hash)
else:
destination = Transport.destinations_map.get(packet.destination_hash)
if destination and destination.type == packet.destination_type:
@ -2632,6 +2698,7 @@ class Transport:
link_entry[IDX_LT_REM_HOPS] = packet.hops
path_entry = Transport.path_table.get(link_destination)
if path_entry: path_entry[IDX_PT_HOPS] = packet.hops
Transport.mark_path_responsive(link_destination)
elif not link_entry[IDX_LT_VALIDATED]: RNS.log(f"Aborting link request proof path re-balancing for {RNS.prettyhexrep(link_destination)} on link {RNS.prettyhexrep(packet.destination_hash)} due to invalid signature", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
else: pass
@ -2662,6 +2729,7 @@ class Transport:
Transport.transmit(link_entry[IDX_LT_RCVD_IF], new_raw)
if not Transport.owner.is_connected_to_shared_instance:
RNS.Identity._used_destination_data(link_entry[IDX_LT_DSTHASH])
Transport.mark_path_responsive(link_entry[IDX_LT_DSTHASH])
else:
RNS.log("Invalid link request proof in transport for link "+RNS.prettyhexrep(packet.destination_hash)+", dropping proof.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
@ -2706,6 +2774,7 @@ class Transport:
if path_entry:
path_entry[IDX_PT_HOPS] = packet.hops
RNS.log(f"Path table re-balanced for {RNS.prettyhexrep(link.destination.hash)}", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
Transport.mark_path_responsive(link.destination.hash)
else: RNS.log(f"Aborting path re-balancing at link terminus for {RNS.prettyhexrep(link.destination.hash)} on link {link} due to invalid signature", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
except Exception as e: RNS.log("Error while validating link request proof for path re-balancing at link terminus. The contained exception was: "+str(e), REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
@ -2837,13 +2906,15 @@ class Transport:
deprecated_paths = []
with Transport.tunnels_lock:
for destination_hash, path_entry in paths.items():
path_timestamp = path_entry[0]
received_from = path_entry[1]
announce_hops = path_entry[2]
expires = path_entry[3]
random_blobs = list(set(path_entry[4]))
receiving_interface = interface
packet_hash = path_entry[6]
new_entry = [time.time(), received_from, announce_hops, expires, random_blobs, receiving_interface, packet_hash]
used = path_entry[7]
new_entry = [path_timestamp, received_from, announce_hops, expires, random_blobs, receiving_interface, packet_hash, used]
should_add = False
with Transport.path_table_lock:
@ -2991,6 +3062,21 @@ class Transport:
return None
@staticmethod
def init_announce_cache():
try:
Transport.announce_cache = AnnounceCache(os.path.join(RNS.Reticulum.cachepath, "announces.db"))
legacy_dir = os.path.join(RNS.Reticulum.cachepath, "announces")
if os.path.isdir(legacy_dir) and len(os.listdir(legacy_dir)) > 0:
st = time.time()
imported = Transport.announce_cache.import_legacy(legacy_dir)
RNS.log(f"Imported {imported} cached announces from legacy file storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE)
except Exception as e:
RNS.log("Could not initialise announce cache database, falling back to file-based caching. The contained exception was: "+str(e), RNS.LOG_ERROR)
Transport.announce_cache = None
@staticmethod
def should_cache(packet):
# TODO: Rework the caching system. It's currently
@ -3024,10 +3110,17 @@ class Transport:
@staticmethod
def clean_announce_cache():
st = time.time()
with Transport.path_table_lock: active_path_hashes = set([Transport.path_table[dst_hash][IDX_PT_PACKET] for dst_hash in Transport.path_table])
with Transport.tunnels_lock: tunnel_path_hashes = set([path_dict[dst_hash][IDX_PT_PACKET] for path_dict in [Transport.tunnels[tunnel_id][IDX_TT_PATHS] for tunnel_id in Transport.tunnels] for dst_hash in path_dict])
active_path_hashes |= tunnel_path_hashes
if Transport.announce_cache != None:
removed = Transport.announce_cache.clean(active_path_hashes)
if removed > 0: RNS.log(f"Removed {removed} cached announces in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return
target_path = os.path.join(RNS.Reticulum.cachepath, "announces")
cached_announce_hashes = os.listdir(target_path)
with Transport.path_table_lock: active_path_hashes = list(set([Transport.path_table[dst_hash][IDX_PT_PACKET] for dst_hash in Transport.path_table]))
with Transport.tunnels_lock: tunnel_path_hashes = list(set([path_dict[dst_hash][IDX_PT_PACKET] for path_dict in [Transport.tunnels[tunnel_id][IDX_TT_PATHS] for tunnel_id in Transport.tunnels] for dst_hash in path_dict]))
removed = 0; total = 0
for packet_hash in cached_announce_hashes:
remove = False
@ -3053,22 +3146,40 @@ class Transport:
def cache(packet, force_cache=False, packet_type=None):
if force_cache or RNS.Transport.should_cache(packet):
try:
packet_hash = RNS.hexrep(packet.get_hash(), delimit=False)
interface_reference = None
if packet.receiving_interface != None: interface_reference = str(packet.receiving_interface)
if packet_type == "announce": cachepath = os.path.join(RNS.Reticulum.cachepath, "announces", packet_hash)
else: cachepath = os.path.join(RNS.Reticulum.cachepath, packet_hash)
if packet_type == "announce":
if Transport.announce_cache != None:
Transport.announce_cache.store(packet.get_hash(), packet.raw, interface_reference)
return
cachepath = os.path.join(RNS.Reticulum.cachepath, "announces", RNS.hexrep(packet.get_hash(), delimit=False))
else:
cachepath = os.path.join(RNS.Reticulum.cachepath, RNS.hexrep(packet.get_hash(), delimit=False))
file = open(cachepath, "wb")
file.write(umsgpack.packb([packet.raw, interface_reference]))
file.close()
except Exception as e:
RNS.log("Error writing packet to cache. The contained exception was: "+str(e), RNS.LOG_ERROR)
except Exception as e: RNS.log("Error writing packet to cache. The contained exception was: "+str(e), RNS.LOG_ERROR)
@staticmethod
def get_cached_packet(packet_hash, packet_type=None):
try:
if packet_type == "announce" and Transport.announce_cache != None:
cached = Transport.announce_cache.fetch(packet_hash)
if cached == None: return None
packet = RNS.Packet(None, cached[0])
interface_reference = cached[1]
for interface in Transport.interfaces:
if str(interface) == interface_reference:
packet.receiving_interface = interface
return packet
packet_hash = RNS.hexrep(packet_hash, delimit=False)
if packet_type == "announce": path = os.path.join(RNS.Reticulum.cachepath, "announces", packet_hash)
else: path = os.path.join(RNS.Reticulum.cachepath, packet_hash)
@ -3207,6 +3318,22 @@ class Transport:
if not Transport.lowest_interface_bitrate: return 0
return 2*(RNS.Reticulum.MTU*8/max(Transport.lowest_interface_bitrate, RNS.Reticulum.MINIMUM_BITRATE)) + RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT
@staticmethod
def path_count():
return len(Transport.path_table)
@staticmethod
def unused_path_count():
with Transport.path_table_lock:
return sum(1 for e in Transport.path_table.values() if not e[IDX_PT_USED])
@staticmethod
def path_pool_counts():
with Transport.path_table_lock:
total = len(Transport.path_table)
unused = sum(1 for e in Transport.path_table.values() if not e[IDX_PT_USED])
return total, unused
@staticmethod
def link_count():
return len(Transport.link_table)
@ -3215,16 +3342,46 @@ class Transport:
def active_link_count():
return sum(1 for e in (True for entry in Transport.link_table if entry[IDX_LT_VALIDATED]))
@staticmethod
def link_distribution():
link_destinations = {}
# TODO: Finish implementation
return link_destinations
# try:
# for link_id in Transport.link_table:
# link_entry = Transport.link_table.get(link_id)
# if link_entry:
# link_destination = link_entry[IDX_LT_DSTHASH]
# active = link_entry[IDX_LT_VALIDATED]
# if not link_destination in link_destinations:
# link_destinations[link_destination] = {}
# else:
# pass
# except Exception as e: RNS.log(f"Error while getting link distribution stats: {e}", RNS.LOG_DEBUG)
@staticmethod
def expire_path(destination_hash):
with Transport.path_table_lock:
if destination_hash in Transport.path_table:
Transport.path_table[destination_hash][IDX_PT_TIMESTAMP] = 0
Transport.path_table[destination_hash][IDX_PT_EXPIRES] = 0
Transport.tables_last_culled = 0
return True
else: return False
@staticmethod
def path_timeout_from_interface(interface):
if interface != None and hasattr(interface, "mode"):
if interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: return Transport.AP_PATH_TIME
elif interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING: return Transport.ROAMING_PATH_TIME
return Transport.PATH_TIMEOUT
@staticmethod
def mark_path_unresponsive(destination_hash):
if destination_hash in Transport.path_table:
@ -3235,11 +3392,22 @@ class Transport:
@staticmethod
def mark_path_responsive(destination_hash):
if destination_hash in Transport.path_table:
with Transport.path_states_lock: Transport.path_states[destination_hash] = Transport.STATE_RESPONSIVE
return True
else: return False
# Marking a path live must only occur on events that
# cryptographically prove recent destination liveness.
now = time.time()
with Transport.path_table_lock:
path_entry = Transport.path_table.get(destination_hash)
if path_entry != None:
path_entry[IDX_PT_TIMESTAMP] = now
path_entry[IDX_PT_EXPIRES] = now + Transport.path_timeout_from_interface(path_entry[IDX_PT_RVCD_IF])
path_entry[IDX_PT_USED] = now
with Transport.path_states_lock:
Transport.path_states[destination_hash] = Transport.STATE_RESPONSIVE
return True
else: return False
@staticmethod
def mark_path_unknown_state(destination_hash):
@ -3767,14 +3935,14 @@ class Transport:
packet_hashlist_path = RNS.Reticulum.storagepath+"/packet_hashlist.raw"
with open(packet_hashlist_path, "wb") as file:
for packet_hash in Transport.packet_hashlist.copy():
for bucket in Transport.packet_hashlist.buckets():
if background:
if time.time() - round_started_at > yield_threshold:
# Low priority, yield thread
round_started_at = time.time()
time.sleep(0.001)
file.write(packet_hash)
file.write(bucket)
RNS.log(f"Saved packet hashlist in {RNS.prettyshorttime(time.time()-save_start)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
@ -3839,6 +4007,7 @@ class Transport:
expires = de[IDX_PT_EXPIRES]
random_blobs = de[IDX_PT_RANDBLOBS]
packet_hash = de[IDX_PT_PACKET]
used = de[IDX_PT_USED]
interface_hash = interface.get_hash()
serialised_entry = [ destination_hash,
@ -3848,7 +4017,8 @@ class Transport:
expires,
random_blobs,
interface_hash,
packet_hash ]
packet_hash,
used ]
serialised_destinations.append(serialised_entry)
@ -3922,6 +4092,7 @@ class Transport:
path_expires = de[3]
random_blobs = de[4][-Transport.PERSIST_RANDOM_BLOBS:]
packet_hash = de[6]
used = de[7]
serialised_entry = [
destination_hash,
@ -3931,7 +4102,8 @@ class Transport:
path_expires,
random_blobs,
interface_hash,
packet_hash
packet_hash,
used,
]
serialised_paths.append(serialised_entry)
@ -3979,6 +4151,7 @@ class Transport:
Transport.void_queues()
if not Transport.owner.is_connected_to_shared_instance:
Transport.persist_data()
if Transport.announce_cache != None: Transport.announce_cache.flush()
@staticmethod
def blackhole_identity(identity_hash, until=None, reason=None):
@ -4126,6 +4299,7 @@ IDX_PT_EXPIRES = 3
IDX_PT_RANDBLOBS = 4
IDX_PT_RVCD_IF = 5
IDX_PT_PACKET = 6
IDX_PT_USED = 7
# Transport.reverse_table entry indices
IDX_RT_RCVD_IF = 0

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
@ -102,7 +102,7 @@ def parse_hash(input_str):
def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity, timeout, drop_queues,
drop_via, max_hops, remote=None, management_identity=None, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT,
blackholed=False, blackhole=False, unblackhole=False, blackhole_duration=None, blackhole_reason=None,
remote_blackhole_list=False, remote_blackhole_list_filter=None, no_output=False, json=False):
remote_blackhole_list=False, remote_blackhole_list_filter=None, usage=False, no_output=False, json=False):
global remote_link, reticulum
reticulum = RNS.Reticulum(configdir = configdir, loglevel = 3+verbosity)
@ -281,13 +281,22 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity,
exit()
else:
now = time.time()
if usage:
try: table = sorted(table, key=lambda e: e["used"], reverse=True)
except Exception as e: pass
for path in table:
if destination_hash == None or destination_hash == path["hash"]:
displayed += 1
exp_str = RNS.timestamp_str(path["expires"])
if path["hops"] == 1: m_str = " "
else: m_str = "s"
print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"]))
if not "used" in path: u_str = ", no usage info"
else:
if usage: u_str = f", last used {RNS.prettytime(now-path['used'])} ago" if path["used"] else ", never used"
else: u_str = ""
print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"])+u_str)
if destination_hash != None and displayed == 0:
print("No path known")
@ -484,6 +493,7 @@ def main():
parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
parser.add_argument("--version", action="version", version="rnpath {version}".format(version=__version__))
parser.add_argument("-t", "--table", action="store_true", help="show all known paths", default=False)
parser.add_argument("-u", "--usage", action="store_true", help="show path usage time", default=False)
parser.add_argument("-m", "--max", action="store", metavar="hops", type=int, help="maximum hops to filter path table by", default=None)
parser.add_argument("-r", "--rates", action="store_true", help="show announce rate info", default=False)
parser.add_argument("-d", "--drop", action="store_true", help="remove the path to a destination", default=False)
@ -518,7 +528,7 @@ def main():
verbosity = args.verbose, timeout = args.w, drop_queues = args.drop_announces, drop_via = args.drop_via, max_hops = args.max,
remote=args.R, management_identity=args.i, remote_timeout=args.W, blackholed=args.blackholed, blackhole=args.blackhole,
unblackhole=args.unblackhole, blackhole_duration=args.duration, blackhole_reason=args.reason, remote_blackhole_list=args.blackholed_list,
remote_blackhole_list_filter=args.list_filter, json=args.json)
remote_blackhole_list_filter=args.list_filter, usage=args.usage, json=args.json)
sys.exit(0)

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -2,7 +2,7 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -1,6 +1,6 @@
# Reticulum License
#
# Copyright (c) 2016-2025 Mark Qvist
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal

View file

@ -304,9 +304,7 @@ def build_scenarios(instance, interface, interface_b, requested=None):
salt=b"bench", context=None)
now = time.time()
with RNS.Transport.path_table_lock:
RNS.Transport.path_table[dst_h] = [
now, next_hop, 3, now + 3600, [], interface, bytes(32)
]
RNS.Transport.path_table[dst_h] = [now, next_hop, 3, now + 3600, [], interface, bytes(32), now]
frames = [_transit_single_frame(interface, dst_h,
_payload(i, size, 0x57))
for i in range(total)]
@ -327,9 +325,7 @@ def build_scenarios(instance, interface, interface_b, requested=None):
salt=b"bench", context=None)
now = time.time()
with RNS.Transport.path_table_lock:
RNS.Transport.path_table[dst_h] = [
now, next_hop, 1, now + 3600, [], interface, bytes(32)
]
RNS.Transport.path_table[dst_h] = [now, next_hop, 1, now + 3600, [], interface, bytes(32), now]
frames = [_transit_single_frame(interface, dst_h,
_payload(i, 135, 0x46))
for i in range(total)]
@ -450,9 +446,7 @@ def build_scenarios(instance, interface, interface_b, requested=None):
salt=b"bench", context=None)
now = time.time()
with RNS.Transport.path_table_lock:
RNS.Transport.path_table[dst_h] = [
now, next_hop, 3, now + 3600, [], interface, bytes(32)
]
RNS.Transport.path_table[dst_h] = [now, next_hop, 3, now + 3600, [], interface, bytes(32), now]
remote_id = RNS.Identity(create_keys=False)
remote_id.load_public_key(os.urandom(RNS.Identity.KEYSIZE//8))
outbound_destination = RNS.Destination(remote_id, RNS.Destination.OUT,