diff --git a/Changelog.md b/Changelog.md index 5e9770a5..ae976152 100644 --- a/Changelog.md +++ b/Changelog.md @@ -1,3 +1,21 @@ +### RNS 1.6.0 + +*Pending*. + +**Changes** +- Added path usage tracking +- Added automatic path table culling based on traffic and usage +- Added unused path pool monitoring and handling +- Added managed storage backend using SQLite +- Added fallback to legacy file-based storage if SQLite is unavailable +- Added managed storage for known destinations and ratchets +- Added managed storage for cached announces +- Added more comprehensive metrics to status output structure +- Optimized announce rate table implementation +- Optimized packet filter hashlist implementation +- Fixed `RNodeInterface` phy stats regression on Android +- Fixed `rngit` default config path typo for `~/.config/rngit` case + ### 2026-10-05: RNS 1.5.7 This maintenance release fixes a bug in the dataplane egress control handling. diff --git a/RNS/Buffer.py b/RNS/Buffer.py index 807b081a..8117fefe 100644 --- a/RNS/Buffer.py +++ b/RNS/Buffer.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Channel.py b/RNS/Channel.py index 799f002e..29c44019 100644 --- a/RNS/Channel.py +++ b/RNS/Channel.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Destination.py b/RNS/Destination.py index c5a65a2d..24c5021a 100755 --- a/RNS/Destination.py +++ b/RNS/Destination.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Identity.py b/RNS/Identity.py index 2f5e4a1e..c863d58c 100644 --- a/RNS/Identity.py +++ b/RNS/Identity.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -40,6 +40,7 @@ from .vendor import umsgpack as umsgpack from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey from RNS.Cryptography import Token +from RNS.Storage import KnownDestinationsStore class Identity: @@ -92,6 +93,7 @@ class Identity: # Storage known_destinations = {} + known_destinations_store = None known_ratchets = {} ratchet_persist_lock = threading.Lock() @@ -104,7 +106,7 @@ class Identity: else: with Identity.known_destinations_lock: if not destination_hash in Identity.known_destinations: - Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0] + Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0, None, 0] else: entry = Identity.known_destinations[destination_hash] entry[0] = time.time() @@ -112,6 +114,8 @@ class Identity: entry[2] = public_key entry[3] = app_data + Identity._kd_mark_dirty(destination_hash) + @staticmethod def recall(target_hash, from_identity_hash=False, _no_use=False): """ @@ -177,7 +181,12 @@ class Identity: def save_known_destinations(background=False, recombine=False): if recombine: RNS.log(f"Recombining known destinations from disk cache on persist is deprecated, argument ignored", RNS.LOG_WARNING) if RNS.Transport.owner.is_connected_to_shared_instance: return - + + if Identity.known_destinations_store != None: Identity.flush_known_destinations() + else: Identity.save_known_destinations_legacy(background=background) + + @staticmethod + def save_known_destinations_legacy(background=False): try: if hasattr(Identity, "saving_known_destinations"): wait_interval = 0.2 @@ -214,6 +223,72 @@ class Identity: @staticmethod def load_known_destinations(): + if RNS.Reticulum.get_instance().is_connected_to_shared_instance: + try: + store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True) + count = 0 + with Identity.known_destinations_lock: + for destination_hash, entry in store.load_all(): + Identity.known_destinations[destination_hash] = entry + count += 1 + + store.close() + RNS.log(f"Loaded {count} known destinations from shared instance database", RNS.LOG_VERBOSE) + + except Exception as e: + with Identity.known_destinations_lock: Identity.known_destinations = {} + RNS.log("Could not load known destinations from shared instance database, continuing with an empty table. The contained exception was: "+str(e), RNS.LOG_ERROR) + + return + + store = None + try: store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8) + except Exception as e: + RNS.log("Could not initialise known destinations database, falling back to legacy snapshot persistence. The contained exception was: "+str(e), RNS.LOG_ERROR) + Identity.known_destinations_store = None + + if store != None: + Identity.known_destinations_store = store + + legacy_path = RNS.Reticulum.storagepath+"/known_destinations" + if os.path.isfile(legacy_path): + try: + imported = store.import_legacy(legacy_path) + os.rename(legacy_path, legacy_path+".imported") + RNS.log(f"Imported {imported} known destinations from legacy storage into the known destinations database", RNS.LOG_VERBOSE) + + except Exception as e: + RNS.log("Could not import legacy known destinations file into the known destinations database. The contained exception was: "+str(e), RNS.LOG_ERROR) + RNS.trace_exception(e) + + st = time.time() + try: + count = 0 + with Identity.known_destinations_lock: + for destination_hash, entry in store.load_all(): + Identity.known_destinations[destination_hash] = entry + count += 1 + + RNS.log(f"Loaded {count} known destinations from storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE) + + except Exception as e: + RNS.log("Error loading known destinations from storage, continuing with an empty table", RNS.LOG_ERROR) + RNS.trace_exception(e) + + ratchetdir = RNS.Reticulum.storagepath+"/ratchets" + if os.path.isdir(ratchetdir) and len(os.listdir(ratchetdir)) > 0: + try: + imported = store.import_legacy_ratchets(ratchetdir, time.time()-Identity.RATCHET_EXPIRY, Identity.known_destinations) + RNS.log(f"Imported {imported} ratchets from legacy file storage", RNS.LOG_VERBOSE) + + except Exception as e: + RNS.log(f"Could not import legacy ratchet files into the known destinations database. The contained exception was: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + else: Identity.load_known_destinations_legacy() + + @staticmethod + def load_known_destinations_legacy(): if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"): st = time.time() try: @@ -244,6 +319,7 @@ class Identity: if destination_hash in Identity.known_destinations: if not Identity.known_destinations[destination_hash][4] < 0: Identity.known_destinations[destination_hash][4] = time.time() + Identity._kd_mark_dirty(destination_hash) return True return False @@ -253,6 +329,7 @@ class Identity: with Identity.known_destinations_lock: if destination_hash in Identity.known_destinations: Identity.known_destinations[destination_hash][4] = -1 + Identity._kd_mark_dirty(destination_hash) return True return False @@ -262,6 +339,7 @@ class Identity: with Identity.known_destinations_lock: if destination_hash in Identity.known_destinations: Identity.known_destinations[destination_hash][4] = time.time() + Identity._kd_mark_dirty(destination_hash) return True return False @@ -327,7 +405,7 @@ class Identity: if not is_retained and not has_path: if not was_used and now - last_announce > RNS.Transport.UNUSED_DESTINATION_LINGER: stale.append(destination_hash) - elif unused_for > RNS.Transport.DESTINATION_TIMEOUT*1.25: stale.append(destination_hash) + elif unused_for > RNS.Transport.PATH_TIMEOUT*1.25: stale.append(destination_hash) except Exception as e: RNS.log(f"Faulty entry for {RNS.prettyhexrep(destination_hash)} while cleaning known destinations: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None @@ -336,13 +414,23 @@ class Identity: with Identity.known_destinations_lock: if destination_hash in Identity.known_destinations: Identity.known_destinations.pop(destination_hash) + Identity._kd_mark_deleted(destination_hash) + if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash) removed += 1 - try: - hexhash = RNS.hexrep(destination_hash, delimit=False) - ratchet_path = f"{ratchetdir}/{hexhash}" - if os.path.isfile(ratchet_path): os.unlink(ratchet_path) - except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING) + if not Identity.known_destinations_store: + try: + hexhash = RNS.hexrep(destination_hash, delimit=False) + ratchet_path = f"{ratchetdir}/{hexhash}" + if os.path.isfile(ratchet_path): os.unlink(ratchet_path) + except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING) + + if Identity.known_destinations_store != None: + expired_ratchets = Identity.known_destinations_store.expire_ratchets(time.time()-Identity.RATCHET_EXPIRY) + if len(expired_ratchets) > 0: + for destination_hash in expired_ratchets: + with Identity.known_destinations_lock: + if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash) RNS.log(f"Cleaned known destinations in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None RNS.log(f"Total: {total}, stale: {len(stale)}, removed: {removed}, no path: {no_path}, never used: {never_used}, with path: {total-no_path}, used: {total-never_used}, retained: {retained}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None @@ -409,33 +497,40 @@ class Identity: @staticmethod def _remember_ratchet(destination_hash, ratchet): try: - if destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet: - ratchet_exists = True + # Don't re-insert existing ratchet + if Identity.known_ratchets.get(destination_hash) == ratchet: return + + store = Identity.known_destinations_store + if store != None: + # Persist to store, only update in-memory cache if this + # destination already exists there. + store.remember_ratchet(destination_hash, ratchet, time.time()) + if destination_hash in Identity.known_ratchets: + Identity.known_ratchets[destination_hash] = ratchet + + # Legacy store: Use file system + # Shared instance client: Just keep in memory cache else: - ratchet_exists = False + ratchet_exists = destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet + if not ratchet_exists: + RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None + Identity.known_ratchets[destination_hash] = ratchet + if not RNS.Transport.owner.is_connected_to_shared_instance: + def persist_job(): + with Identity.ratchet_persist_lock: + hexhash = RNS.hexrep(destination_hash, delimit=False) + ratchet_data = {"ratchet": ratchet, "received": time.time()} - if not ratchet_exists: - RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None - Identity.known_ratchets[destination_hash] = ratchet - if not RNS.Transport.owner.is_connected_to_shared_instance: - def persist_job(): - with Identity.ratchet_persist_lock: - hexhash = RNS.hexrep(destination_hash, delimit=False) - ratchet_data = {"ratchet": ratchet, "received": time.time()} + ratchetdir = RNS.Reticulum.storagepath+"/ratchets" - ratchetdir = RNS.Reticulum.storagepath+"/ratchets" - - if not os.path.isdir(ratchetdir): - os.makedirs(ratchetdir) + if not os.path.isdir(ratchetdir): os.makedirs(ratchetdir) - outpath = f"{ratchetdir}/{hexhash}.out" - finalpath = f"{ratchetdir}/{hexhash}" - with open(outpath, "wb") as ratchet_file: - ratchet_file.write(umsgpack.packb(ratchet_data)) - os.replace(outpath, finalpath) + outpath = f"{ratchetdir}/{hexhash}.out" + finalpath = f"{ratchetdir}/{hexhash}" + with open(outpath, "wb") as ratchet_file: ratchet_file.write(umsgpack.packb(ratchet_data)) + os.replace(outpath, finalpath) - - threading.Thread(target=persist_job, daemon=True).start() + threading.Thread(target=persist_job, daemon=True).start() except Exception as e: RNS.log(f"Could not persist ratchet for {RNS.prettyhexrep(destination_hash)} to storage.", RNS.LOG_ERROR) @@ -483,7 +578,43 @@ class Identity: @staticmethod def get_ratchet(destination_hash): - if not destination_hash in Identity.known_ratchets: + if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash] + + store = Identity.known_destinations_store + if store != None: + ratchet_data = store.fetch_ratchet(destination_hash) + if ratchet_data != None: + ratchet, received = ratchet_data + if time.time() < received+Identity.RATCHET_EXPIRY: + Identity.known_ratchets[destination_hash] = ratchet + return ratchet + + else: return None + + else: + RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + return None + + elif RNS.Reticulum.get_instance().is_connected_to_shared_instance: + try: + # TODO: Ideally, refactor this to an RPC method instead + store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True) + ratchet_data = store.fetch_ratchet(destination_hash) + store.close() + + if ratchet_data != None: + ratchet, received = ratchet_data + if time.time() < received+Identity.RATCHET_EXPIRY: + Identity.known_ratchets[destination_hash] = ratchet + return ratchet + + except Exception as e: RNS.log(f"Could not fetch ratchet for {RNS.prettyhexrep(destination_hash)} from shared instance database. The contained exception was: {e}", RNS.LOG_ERROR) + + return None + + else: + if not os.path.isfile(RNS.Reticulum.storagepath+"/ratchets/"+RNS.hexrep(destination_hash, delimit=False)): return None + ratchetdir = RNS.Reticulum.storagepath+"/ratchets" hexhash = RNS.hexrep(destination_hash, delimit=False) ratchet_path = f"{ratchetdir}/{hexhash}" @@ -493,16 +624,14 @@ class Identity: ratchet_data = umsgpack.unpackb(ratchet_file.read()) if time.time() < ratchet_data["received"]+Identity.RATCHET_EXPIRY and len(ratchet_data["ratchet"]) == Identity.RATCHETSIZE//8: Identity.known_ratchets[destination_hash] = ratchet_data["ratchet"] - else: - return None - + else: return None + except Exception as e: RNS.log(f"An error occurred while loading ratchet data for {RNS.prettyhexrep(destination_hash)} from storage.", RNS.LOG_ERROR) RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR) return None - if destination_hash in Identity.known_ratchets: - return Identity.known_ratchets[destination_hash] + if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash] else: RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None return None @@ -614,12 +743,35 @@ class Identity: @staticmethod def persist_data(background=False): if not RNS.Transport.owner.is_connected_to_shared_instance: - Identity.save_known_destinations(background=background) + if Identity.known_destinations_store != None: Identity.flush_known_destinations() + else: Identity.save_known_destinations_legacy(background=background) @staticmethod - def exit_handler(): - Identity.persist_data() + def flush_known_destinations(): + store = Identity.known_destinations_store + if store == None: return False + with Identity.known_destinations_lock: upserts, deletes, ratchet_upserts, ratchet_deletes = store.take_pending() + + try: return store.apply(upserts, deletes, ratchet_upserts, ratchet_deletes, Identity.known_destinations) + except Exception as e: + RNS.log(f"Could not flush known destinations to storage. The contained exception was: {e}", RNS.LOG_ERROR) + return False + + @staticmethod + def _kd_mark_dirty(destination_hash): + store = Identity.known_destinations_store + if store != None: store.mark_dirty(destination_hash) + + @staticmethod + def _kd_mark_deleted(destination_hash): + store = Identity.known_destinations_store + if store != None: + store.mark_deleted(destination_hash) + store.mark_ratchet_deleted(destination_hash) + + @staticmethod + def exit_handler(): Identity.persist_data() @staticmethod def from_bytes(prv_bytes): diff --git a/RNS/Link.py b/RNS/Link.py index 8d5079da..bb0c3c8a 100644 --- a/RNS/Link.py +++ b/RNS/Link.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -425,6 +425,7 @@ class Link: self.activated_at = time.time() self.last_proof = self.activated_at RNS.Transport.activate_link(self) + RNS.Transport.mark_path_responsive(self.destination.hash) RNS.log(f"Link {self} established with {self.destination}, RTT is {RNS.prettyshorttime(self.rtt)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0: diff --git a/RNS/Packet.py b/RNS/Packet.py index 991819bc..31ddf7fb 100755 --- a/RNS/Packet.py +++ b/RNS/Packet.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Resolver.py b/RNS/Resolver.py index 9f0b1a52..61b9c619 100644 --- a/RNS/Resolver.py +++ b/RNS/Resolver.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Resource.py b/RNS/Resource.py index c7466af6..f5db9fb3 100644 --- a/RNS/Resource.py +++ b/RNS/Resource.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Reticulum.py b/RNS/Reticulum.py index 48f362d1..54b67664 100755 --- a/RNS/Reticulum.py +++ b/RNS/Reticulum.py @@ -1350,7 +1350,7 @@ class Reticulum: filepath = self.cachepath + "/" + filename mtime = os.path.getmtime(filepath) age = now - mtime - if age > RNS.Transport.DESTINATION_TIMEOUT: os.unlink(filepath) + if age > RNS.Transport.PATH_TIMEOUT: os.unlink(filepath) if background: time.sleep(0.001) except Exception as e: @@ -1387,6 +1387,7 @@ class Reticulum: if path == "lowest_interface_bitrate": self.rpc_return(conn, self.get_lowest_interface_bitrate()) if path == "medium_path_timeout": self.rpc_return(conn, self.get_medium_path_timeout()) if path == "link_count": self.rpc_return(conn, self.get_link_count()) + if path == "link_distribution": self.rpc_return(conn, self.get_link_distribution()) if path == "active_link_count": self.rpc_return(conn, self.get_active_link_count()) if path == "packet_rssi": self.rpc_return(conn, self.get_packet_rssi(call["packet_hash"])) if path == "packet_snr": self.rpc_return(conn, self.get_packet_snr(call["packet_hash"])) @@ -1684,6 +1685,7 @@ class Reticulum: pql = RNS.Transport.INBOUND_PR_QUEUE_LENGTH ilql = RNS.Transport.INBOUND_IL_QUEUE_LENGTH tql = dql+aql+pql+ilql + ppcs = RNS.Transport.path_pool_counts() stats = {} stats["interfaces"] = interfaces @@ -1705,6 +1707,12 @@ class Reticulum: stats["ptxf"] = RNS.Transport.pr_freq_tx stats["rxpps"] = RNS.Transport.rx_pps stats["txpps"] = RNS.Transport.tx_pps + stats["rxpkts"] = RNS.Transport.rx_packets + stats["txpkts"] = RNS.Transport.tx_packets + stats["arxpkts"] = RNS.Transport.announce_rxc + stats["atxpkts"] = RNS.Transport.announce_txc + stats["prxpkts"] = RNS.Transport.pr_rxc + stats["ptxpkts"] = RNS.Transport.pr_txc stats["rxqt"] = qsnapshot[0] stats["rxqd"] = qsnapshot[1][0] stats["rxqa"] = qsnapshot[1][1] @@ -1721,6 +1729,12 @@ class Reticulum: stats["pqpressure"] = qsnapshot[1][2]/pql if qsnapshot[1][2] else 0 stats["ilqpressure"] = qsnapshot[1][3]/ilql if qsnapshot[1][3] else 0 stats["txq"] = None + stats["tlinks"] = RNS.Transport.link_count() + stats["alinks"] = RNS.Transport.active_link_count() + stats["plinks"] = stats["tlinks"]-stats["alinks"] + stats["tpcount"] = ppcs[0] + stats["unupcount"] = ppcs[1] + stats["actpcount"] = stats["tpcount"] - stats["unupcount"] if Reticulum.transport_enabled(): stats["transport_id"] = RNS.Transport.identity.hash @@ -1753,6 +1767,7 @@ class Reticulum: entry = { "hash": dst_hash, "timestamp": RNS.Transport.path_table[dst_hash][0], "via": RNS.Transport.path_table[dst_hash][1], "hops": path_hops, "expires": RNS.Transport.path_table[dst_hash][3], + "used": RNS.Transport.path_table[dst_hash][7], "interface": str(RNS.Transport.path_table[dst_hash][5]) } path_table.append(entry) @@ -1767,15 +1782,10 @@ class Reticulum: else: rate_table = [] - for dst_hash in RNS.Transport.announce_rate_table: - entry = { - "hash": dst_hash, - "last": RNS.Transport.announce_rate_table[dst_hash]["last"], - "rate_violations": RNS.Transport.announce_rate_table[dst_hash]["rate_violations"], - "blocked_until": RNS.Transport.announce_rate_table[dst_hash]["blocked_until"], - "timestamps": RNS.Transport.announce_rate_table[dst_hash]["timestamps"], - } - rate_table.append(entry) + with RNS.Transport.announce_rate_table_lock: + for dst_hash, entry in RNS.Transport.announce_rate_table.items(): + entry["hash"] = dst_hash + rate_table.append(entry) return rate_table @@ -1925,6 +1935,16 @@ class Reticulum: else: return RNS.Transport.active_link_count() + def get_link_distribution(self): + if self.is_connected_to_shared_instance: + rpc_connection = self.get_rpc_client() + rpc_connection.send_bytes(mp.packb({"get": "link_distribution"})) + response = mp.unpackb(rpc_connection.recv_bytes()) + return response + + else: + return RNS.Transport.link_distribution() + def get_packet_rssi(self, packet_hash): if self.is_connected_to_shared_instance: rpc_connection = self.get_rpc_client() diff --git a/RNS/Storage.py b/RNS/Storage.py new file mode 100644 index 00000000..ff7631cb --- /dev/null +++ b/RNS/Storage.py @@ -0,0 +1,580 @@ +# Reticulum License +# +# Copyright (c) 2016-2026 Mark Qvist +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# - The Software shall not be used in any kind of system which includes amongst +# its functions the ability to purposefully do harm to human beings. +# +# - The Software shall not be used, directly or indirectly, in the creation of +# an artificial intelligence, machine learning or language model training +# dataset, including but not limited to any use that contributes to the +# training or development of such a model or algorithm. +# +# - The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +from array import array +import hashlib +import threading +import os +import time + +class AnnounceRateTable: + # Columnar store for announce rate tracking. Layout: + # _index : dict[destination_hash -> slot] + # _keys : list[slot] -> destination_hash (for swap-remove bookkeeping) + # _last : array('d') last non-violating announce time + # _blocked: array('d') blocked_until + # _viol : array('i') rate_violations + # _ts : array('d') flat ring of max_timestamps doubles per slot + # _ts_len : array('B') valid timestamp count per slot + # _ts_head: array('B') ring write cursor per slot + + def __init__(self, max_timestamps=16): + assert max_timestamps <= 255, "Timestamp ring size must fit in a single byte" + self.max_ts = max_timestamps + self._index = {} + self._keys = [] + self._last = array('d') + self._blocked = array('d') + self._viol = array('i') + self._ts = array('d') + self._ts_len = array('B') + self._ts_head = array('B') + self._cursor = 0 + + def __len__(self): return len(self._keys) + def __contains__(self, destination_hash): return destination_hash in self._index + + # Perform the announce-rate check-and-update for one announce. + # Caller must hold the announce_rate_table_lock. + def check_and_update(self, destination_hash, now, target, grace, penalty): + slot = self._index.get(destination_hash) + + if slot is None: + slot = len(self._keys) + self._index[destination_hash] = slot + self._keys.append(destination_hash) + self._last.append(now) + self._blocked.append(0.0) + self._viol.append(0) + self._ts.extend([0.0]*self.max_ts) + self._ts[slot*self.max_ts] = now + self._ts_len.append(1) + self._ts_head.append(1 % self.max_ts) + return False + + base = slot*self.max_ts + head = self._ts_head[slot] + self._ts[base+head] = now + self._ts_head[slot] = (head+1) % self.max_ts + if self._ts_len[slot] < self.max_ts: self._ts_len[slot] += 1 + + current_rate = now - self._last[slot] + rate_blocked = False + + if now > self._blocked[slot]: + if current_rate < target: self._viol[slot] += 1 + else: + v = self._viol[slot] - 1 + self._viol[slot] = v if v > 0 else 0 + + if self._viol[slot] > grace: + self._blocked[slot] = self._last[slot] + target + penalty + rate_blocked = True + else: self._last[slot] = now + + else: rate_blocked = True + + return rate_blocked + + # Caller must hold the table lock + def cull(self, now, ttl, max_scan=1024, max_cull=None): + if max_cull is None: max_cull = max_scan + horizon = now - ttl + culled = 0 + scanned = 0 + n = len(self._keys) + + while scanned < max_scan and culled < max_cull and n > 0: + if self._cursor >= n: self._cursor = 0 + slot = self._cursor + + if self._last[slot] < horizon and self._blocked[slot] <= now: + last_i = n - 1 + stale_key = self._keys[slot] + + if slot != last_i: + moved_key = self._keys[last_i] + self._keys[slot] = moved_key + self._index[moved_key] = slot + self._last[slot] = self._last[last_i] + self._blocked[slot] = self._blocked[last_i] + self._viol[slot] = self._viol[last_i] + self._ts_len[slot] = self._ts_len[last_i] + self._ts_head[slot] = self._ts_head[last_i] + + f = last_i*self.max_ts; t = slot*self.max_ts + self._ts[t:t+self.max_ts] = self._ts[f:f+self.max_ts] + + self._keys.pop() + self._last.pop(); self._blocked.pop(); self._viol.pop() + self._ts_len.pop(); self._ts_head.pop() + del self._ts[len(self._ts)-self.max_ts:] # drop tail block of removed slot + del self._index[stale_key] + + n -= 1 + culled += 1 + # Do not advance cursor, re-examine the slot a moved entry landed in + + else: + self._cursor = slot + 1 + scanned += 1 + + if self._cursor >= n: self._cursor = 0 + + return culled + + # Hold table lock, or accept a possibly inconsistent snapshot + def items(self): + for slot, key in enumerate(self._keys): + n = self._ts_len[slot] + head = self._ts_head[slot] + base = slot*self.max_ts + if n < self.max_ts: ts = list(self._ts[base:base+n]) + else: ts = list(self._ts[base+head:base+self.max_ts]) + list(self._ts[base:base+head]) + yield key, { "last": self._last[slot], "rate_violations": self._viol[slot], + "blocked_until": self._blocked[slot], "timestamps": ts } + +class PacketHashlist: + # Bucketed byte-blob packet filter hashlist, with the primary + # design goal of memory coherence and density. All operations + # are safe to call without holding a lock. + + def __init__(self, hash_width=32, bucket_count=16384): + assert bucket_count > 0 and bucket_count & (bucket_count-1) == 0, "Bucket count must be a power of two" + self.width = hash_width + self._nb = bucket_count + self._mask = bucket_count - 1 + self._cur = [bytearray() for _ in range(bucket_count)] + self._prv = [bytearray() for _ in range(bucket_count)] + self._count = 0 + + def __len__(self): + return self._count + + def __contains__(self, packet_hash): + bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask + return packet_hash in self._cur[bucket] or packet_hash in self._prv[bucket] + + def add(self, packet_hash): + assert len(packet_hash) == self.width, "Packet hash width mismatch" + bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask + if packet_hash not in self._cur[bucket]: + self._cur[bucket].extend(packet_hash) + self._count += 1 + + def discard(self, packet_hash): + w = self.width + bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask + for generation in (self._cur, self._prv): + entries = generation[bucket] + pos = entries.find(packet_hash) + while pos != -1: + if pos % w == 0: + del entries[pos:pos+w] + if generation is self._cur: self._count -= 1 + pos = entries.find(packet_hash, pos) + else: + pos = entries.find(packet_hash, pos+1) + + def swap(self): + self._prv = self._cur + self._cur = [bytearray() for _ in range(self._nb)] + self._count = 0 + + # Current-generation buckets for persist operations + def buckets(self): return tuple(self._cur) + +class KnownDestinationsStore: + # Write-behind SQLite storage backend for known destinations. + # The actual in-memory table is used directly for all transport + # operations; this serves purely as the store, and no read operations + # utilize the database directly. + + def __init__(self, db_path, hash_width=16, read_only=False): + import sqlite3 + self.sqlite3 = sqlite3 + self.db_path = db_path + self.hash_width = hash_width + self.read_only = read_only + self._dirty_upsert = set() + self._dirty_delete = set() + self._ratchet_upserts = {} + self._ratchet_deletes = set() + self._flush_lock = threading.Lock() + + if read_only: self._conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False, isolation_level=None) + else: + self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None) + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._conn.execute("PRAGMA journal_size_limit=8388608") + self.__create_schema() + + def close(self): + if self._conn != None: + self._conn.close() + self._conn = None + + def __create_schema(self): + self._conn.execute("""CREATE TABLE IF NOT EXISTS known_destinations ( + destination_hash BLOB PRIMARY KEY, + last_heard REAL NOT NULL, + packet_hash BLOB NOT NULL, + public_key BLOB NOT NULL, + app_data BLOB, + used REAL NOT NULL DEFAULT 0, + stamp BLOB, + stamp_value INTEGER, + identity_hash BLOB NOT NULL)""") + + self._conn.execute("""CREATE TABLE IF NOT EXISTS ratchets ( + destination_hash BLOB PRIMARY KEY, + ratchet BLOB NOT NULL, + received REAL NOT NULL)""") + + self._conn.execute("PRAGMA user_version = 1") + self._conn.commit() + + def __entry_row(self, destination_hash, entry): + stamp = entry[5] if len(entry) > 5 else None + stamp_value = entry[6] if len(entry) > 6 else 0 + identity_hash = hashlib.sha256(entry[2]).digest()[:self.hash_width] + return (destination_hash, entry[0], entry[1], entry[2], entry[3], entry[4], stamp, stamp_value, identity_hash) + + # Caller must hold Identity.known_destinations_lock + def mark_dirty(self, destination_hash): + if self.read_only: return + self._dirty_delete.discard(destination_hash) + self._dirty_upsert.add(destination_hash) + + # Caller must hold Identity.known_destinations_lock + def mark_deleted(self, destination_hash): + if self.read_only: return + self._dirty_upsert.discard(destination_hash) + self._dirty_delete.add(destination_hash) + + # Caller must hold Identity.known_destinations_lock + def take_pending(self): + if self.read_only: return set(), set(), {}, set() + upserts = self._dirty_upsert + deletes = self._dirty_delete + ratchet_upserts = self._ratchet_upserts + ratchet_deletes = self._ratchet_deletes + self._dirty_upsert = set() + self._dirty_delete = set() + self._ratchet_upserts = {} + self._ratchet_deletes = set() + return upserts, deletes, ratchet_upserts, ratchet_deletes + + # May be called without any locks held + def apply(self, upserts, deletes, ratchet_upserts, ratchet_deletes, known_destinations): + if self.read_only: return 0 + if len(upserts) == 0 and len(deletes) == 0 and len(ratchet_upserts) == 0 and len(ratchet_deletes) == 0: return 0 + + with self._flush_lock: + rows = [] + removables = [] + ratchet_rows = [] + ratchet_removables = [] + + for destination_hash in upserts: + entry = known_destinations.get(destination_hash) + if entry != None: rows.append(self.__entry_row(destination_hash, entry)) + + for destination_hash in deletes: + if destination_hash not in known_destinations: removables.append((destination_hash,)) + + for destination_hash, ratchet_entry in ratchet_upserts.items(): + # Ratchets of purged destinations must be removed as well + if destination_hash in known_destinations: ratchet_rows.append((destination_hash, *ratchet_entry)) + else: ratchet_removables.append((destination_hash,)) + + for destination_hash in ratchet_deletes: + if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,)) + + try: + self._conn.execute("BEGIN") + for i in range(0, len(rows), 4096): self._conn.executemany("INSERT INTO known_destinations VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, identity_hash=excluded.identity_hash", rows[i:i+4096]) + for i in range(0, len(removables), 4096): self._conn.executemany("DELETE FROM known_destinations WHERE destination_hash=?", removables[i:i+4096]) + for i in range(0, len(ratchet_rows), 4096): self._conn.executemany("INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received", ratchet_rows[i:i+4096]) + for i in range(0, len(ratchet_removables), 4096): self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", ratchet_removables[i:i+4096]) + self._conn.commit() + + except Exception: + # Re-mark everything as dirty, so the changes are retried + # on the next flush, then propagate. + self._dirty_upsert |= {row[0] for row in rows} + self._dirty_delete |= {row[0] for row in removables} + self._ratchet_upserts |= {row[0]: row[1:] for row in ratchet_rows} + self._ratchet_deletes |= {row[0] for row in ratchet_removables} + self._conn.rollback() + raise + + return len(rows) + len(removables) + len(ratchet_rows) + len(ratchet_removables) + + def load_all(self): + cursor = self._conn.execute("SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value FROM known_destinations") + while True: + rows = cursor.fetchmany(4096) + if not rows: break + for row in rows: yield row[0], [row[1], row[2], row[3], row[4], row[5], row[6], row[7]] + + # Caller must hold Identity.known_destinations_lock. + def remember_ratchet(self, destination_hash, ratchet, now): + if self.read_only: return False + + pending = self._ratchet_upserts.get(destination_hash) + if pending != None: + if pending[0] == ratchet: + self._ratchet_deletes.discard(destination_hash) + return False + + else: + row = self._conn.execute("SELECT ratchet FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone() + if row != None and row[0] == ratchet: + # Discard any pending delete mark queued + # before this ratchet arrived + self._ratchet_deletes.discard(destination_hash) + return False + + self._ratchet_upserts[destination_hash] = (ratchet, now) + self._ratchet_deletes.discard(destination_hash) + return True + + def fetch_ratchet(self, destination_hash): + pending = self._ratchet_upserts.get(destination_hash) + if pending != None: return pending[0], pending[1] + if destination_hash in self._ratchet_deletes: return None + + row = self._conn.execute("SELECT ratchet, received FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone() + if row != None: return row[0], row[1] + + return None + + # Caller must hold Identity.known_destinations_lock + def mark_ratchet_deleted(self, destination_hash): + if self.read_only: return + self._ratchet_upserts.pop(destination_hash, None) + self._ratchet_deletes.add(destination_hash) + + def expire_ratchets(self, cutoff): + if self.read_only: return [] + + with self._flush_lock: + # Drop expired entries in pending as well + expired = set(row[0] for row in self._conn.execute("SELECT destination_hash FROM ratchets WHERE received 0: + self._conn.execute("BEGIN") + for i in range(0, len(expired), 4096): + self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", [(h,) for h in list(expired)[i:i+4096]]) + self._conn.commit() + + return list(expired) + + def load_ratchets(self): + cursor = self._conn.execute("SELECT destination_hash, ratchet, received FROM ratchets") + while True: + rows = cursor.fetchmany(4096) + if not rows: break + for row in rows: yield row[0], row[1], row[2] + + def import_legacy_ratchets(self, dir_path, expiry_cutoff, known_destinations, ratchet_width=32): + import RNS + from .vendor import umsgpack + imported = 0 + for filename in os.listdir(dir_path): + file_path = os.path.join(dir_path, filename) + try: + if not os.path.isfile(file_path): continue + with open(file_path, "rb") as rf: + ratchet_data = umsgpack.unpackb(rf.read()) + + destination_hash = bytes.fromhex(filename) + ratchet = ratchet_data["ratchet"]; received = ratchet_data["received"] + if len(ratchet) != ratchet_width: raise ValueError("Invalid ratchet width") + if received < expiry_cutoff: raise ValueError("Ratchet expired") + if destination_hash not in known_destinations: raise ValueError("Destination unknown") + + self._ratchet_upserts[destination_hash] = (ratchet, received) + imported += 1 + os.unlink(file_path) + + except Exception as e: + RNS.log(f"Could not import ratchet for {filename}: {e}", RNS.LOG_WARNING) + try: os.unlink(file_path) + except: pass + + pending = self._ratchet_upserts + self._ratchet_upserts = {} + self.apply(set(), set(), pending, set(), known_destinations) + + return imported + + def import_legacy(self, path): + from .vendor import umsgpack + with open(path, "rb") as file: legacy = umsgpack.load(file) + + rows = [] + now = time.time() + for destination_hash, entry in legacy.items(): + if len(entry) < 5: continue + entry = list(entry) + while len(entry) < 7: entry.append(None if len(entry) == 5 else 0) + if entry[0] == None or entry[0] <= 0: entry[0] = now + rows.append(self.__entry_row(destination_hash, entry)) + + self._conn.execute("BEGIN") + for i in range(0, len(rows), 4096): self._conn.executemany("INSERT OR REPLACE INTO known_destinations VALUES (?,?,?,?,?,?,?,?,?)", rows[i:i+4096]) + self._conn.commit() + + return len(rows) + +class AnnounceCache: + + def __init__(self, db_path): + import sqlite3 + self.db_path = db_path + self._pending = {} + self._flush_lock = threading.Lock() + self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None) + + self._conn.execute("PRAGMA journal_mode=WAL") + self._conn.execute("PRAGMA synchronous=NORMAL") + self._conn.execute("PRAGMA journal_size_limit=8388608") + self._conn.execute("PRAGMA auto_vacuum=INCREMENTAL") + self.__create_schema() + + def __create_schema(self): + self._conn.execute("""CREATE TABLE IF NOT EXISTS cached_announces ( + packet_hash BLOB PRIMARY KEY, + received REAL NOT NULL, + raw BLOB NOT NULL, + interface_reference TEXT)""") + + self._conn.execute("PRAGMA user_version = 1") + + def store(self, packet_hash, raw, interface_reference): + self._pending[packet_hash] = (time.time(), raw, interface_reference) + + def pending_count(self): + return len(self._pending) + + def fetch(self, packet_hash): + pending = self._pending.get(packet_hash) + if pending != None: return pending[1], pending[2] + + row = self._conn.execute("SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=?", (packet_hash,)).fetchone() + if row != None: return row[0], row[1] + + return None + + def flush(self): + with self._flush_lock: return self.__apply_pending() + + def __apply_pending(self): + pending = self._pending + self._pending = {} + if len(pending) == 0: return 0 + + try: + self._conn.execute("BEGIN") + items = list(pending.items()) + for i in range(0, len(items), 4096): + self._conn.executemany("INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference", + [(packet_hash, *entry) for packet_hash, entry in items[i:i+4096]]) + self._conn.commit() + + except Exception: + # Re-mark everything as pending, so the changes + # are retried on the next flush, then propagate. + self._pending.update(pending) + self._conn.rollback() + raise + + return len(pending) + + def clean(self, active_hashes): + with self._flush_lock: + self.__apply_pending() + + try: + self._conn.execute("BEGIN") + self._conn.execute("DROP TABLE IF EXISTS temp.active_announces") + self._conn.execute("CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY)") + active_list = list(active_hashes) + for i in range(0, len(active_list), 4096): + self._conn.executemany("INSERT OR IGNORE INTO active_announces VALUES (?)", [(h,) for h in active_list[i:i+4096]]) + + changes_before = self._conn.total_changes + self._conn.execute("DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces)") + removed = self._conn.total_changes - changes_before + self._conn.commit() + + except Exception: + self._conn.rollback() + raise + + if removed > 0: self._conn.execute("PRAGMA incremental_vacuum") + + return removed + + def import_legacy(self, dir_path): + from .vendor import umsgpack + imported = 0 + for filename in os.listdir(dir_path): + file_path = os.path.join(dir_path, filename) + try: + if not os.path.isfile(file_path): continue + with open(file_path, "rb") as file: + cached_data = umsgpack.unpackb(file.read()) + + packet_hash = bytes.fromhex(filename) + interface_reference = cached_data[1] if len(cached_data) > 1 else None + self.store(packet_hash, cached_data[0], interface_reference) + imported += 1 + os.unlink(file_path) + + except Exception: + try: os.unlink(file_path) + except Exception: pass + + self.flush() + return imported + + def close(self): + if self._conn != None: + self._conn.close() + self._conn = None diff --git a/RNS/Transport.py b/RNS/Transport.py index 2a46569f..4cbddba4 100755 --- a/RNS/Transport.py +++ b/RNS/Transport.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -42,6 +42,7 @@ from queue import Full, Empty from threading import Lock, Condition from .vendor import umsgpack as umsgpack from RNS.Interfaces.BackboneInterface import BackboneInterface +from RNS.Storage import AnnounceRateTable, PacketHashlist, AnnounceCache class InboundQueues: __slots__ = ("_cond", "_queues", "_sizes", "_dropped", "_hw_mark") @@ -123,7 +124,6 @@ class Transport: PATHFINDER_R = 1 # Retransmit retries PATHFINDER_G = 5 # Retry grace period PATHFINDER_RW = 0.5 # Random window for announce rebroadcast - PATHFINDER_E = 60*60*24*7 # Path expiration of one week AP_PATH_TIME = 60*60*24 # Path expiration of one day for Access Point paths ROAMING_PATH_TIME = 60*60*6 # Path expiration of 6 hours for Roaming paths @@ -152,10 +152,14 @@ class Transport: LINK_TIMEOUT = RNS.Link.STALE_TIME * 1.25 ALLOW_LINK_PATH_REBALANCE = True REVERSE_TIMEOUT = 8*60 # Reverse table entries are removed after 8 minutes - DESTINATION_TIMEOUT = 60*60*24*7 # Destination table entries are removed if unused for one week - UNUSED_DESTINATION_LINGER = 6*60 # Linger time for pathless and never used destinations + PATH_TIMEOUT = 60*60*24*7 # Path table entry retainment after last cryptographically verifiable liveness event + UNUSED_PATH_LINGER = 60*60*1.75 # Linger time for never used paths + UNUSED_PATH_LINGER_LEAF = 60*5 # Linger time for never used paths on non-transport instances + UNUSED_DESTINATION_LINGER = 60*6 # Linger time for pathless and never used destinations + PATH_POOL_UNUSED_TARGET = 2048 # Max unused path table pool target, unused destinations are culled to reach it TUNNEL_TIMEOUT = 60*60*8 # Tunnel table entries are removed if unused for eight hours TUNNEL_PATH_TIMEOUT = 60*60*8 # Tunnel path table entries are removed if unused for eight hours + RATE_TABLE_RETAIN = 60*60 # Default announce rate table retainment, increased dynamically MAX_RECEIPTS = 1024 # Maximum number of receipts to keep track of MAX_RATE_TIMESTAMPS = 16 # Maximum number of announce timestamps to keep per destination PERSIST_RANDOM_BLOBS = 32 # Maximum number of random blobs per destination to persist to disk @@ -169,9 +173,8 @@ class Transport: active_links = [] # Links that are active pending_links_map = {} # Link ID hash map of pending links active_links_map = {} # Link ID hash map of active links - packet_hashlist = set() # A list of packet hashes for duplicate detection - packet_hashlist_prev = set() receipts = [] # Receipts of all outgoing packets for proof processing + packet_hashlist = PacketHashlist(hash_width=RNS.Identity.HASHLENGTH//8) # Notes on memory usage: 1 megabyte of memory can store approximately # 55.100 path table entries or approximately 22.300 link table entries. @@ -183,11 +186,11 @@ class Transport: held_announces = {} # A table containing temporarily held announce-table entries announce_handlers = [] # A table storing externally registered announce handlers tunnels = {} # A table storing tunnels to other transport instances - announce_rate_table = {} # A table for keeping track of announce rates path_requests = {} # A table for storing path request timestamps inflight_path_requests = {} # A table for tracking in-flight path request path_states = {} # A table for keeping track of path states blackholed_identities = {} # A table for keeping track of blackholed identities + announce_rate_table = AnnounceRateTable(max_timestamps=MAX_RATE_TIMESTAMPS) discovery_path_requests = {} # A table for keeping track of path requests on behalf of other nodes discovery_pr_tags = set() # A table for keeping track of tagged path requests @@ -260,20 +263,36 @@ class Transport: known_destinations_interval = 5*60 tables_last_culled = 0.0 tables_cull_interval = 5.0 + rate_table_last_culled = 0.0 + rate_table_cull_interval = 5.0 + rate_table_cull_chunk = 1024 + pool_size_last_checked = 0.0 + pool_size_check_interval = 5*60 + kd_flush_interval = 10.0 + announce_cache = None + ac_flush_interval = 15.0 + ac_last_flush = 0.0 + announce_cache_max_pending = 10000 + kd_last_flush = 0.0 interface_last_jobs = 0.0 interface_jobs_interval = 5.0 last_mgmt_announce = 0 mgmt_announce_interval = 2*60*60 blackhole_last_checked = 0 blackhole_check_interval = 60 + unused_path_linger = None interface_announcer = None discovery_handler = None blackhole_updater = None traffic_rxb = 0 traffic_txb = 0 + announce_rxc = 0 + announce_txc = 0 announce_rxb = 0 announce_txb = 0 + pr_rxc = 0 + pr_txc = 0 pr_rxb = 0 pr_txb = 0 speed_rx = 0 @@ -299,6 +318,7 @@ class Transport: _identity = None _should_run = True + _unused_path_count_estimate = 0 @staticmethod def internal_identity(): return Transport._identity @@ -336,6 +356,12 @@ class Transport: if RNS.Reticulum.local_hops_delta(): Transport.local_hops_delta = (ord(os.urandom(1))%6)+2 + if not RNS.Reticulum.transport_enabled() or Transport.owner.is_connected_to_shared_instance: + Transport.unused_path_linger = Transport.UNUSED_PATH_LINGER_LEAF + else: + Transport.unused_path_linger = Transport.UNUSED_PATH_LINGER + RNS.log(f"Unused path linger time is {RNS.prettytime(Transport.unused_path_linger)}", RNS.LOG_DEBUG) + packet_hashlist_path = RNS.Reticulum.storagepath+"/packet_hashlist.raw" if RNS.Reticulum.transport_enabled() and not Transport.owner.is_connected_to_shared_instance: if os.path.isfile(packet_hashlist_path): @@ -393,6 +419,9 @@ class Transport: # Defer cleaning known destinations Transport.destinations_last_cleaned = time.time() + # Defer path table unused pool cull + Transport.pool_size_last_checked = time.time() + # Defer sending management announces for 15 seconds Transport.last_mgmt_announce = time.time() - Transport.mgmt_announce_interval + 15 @@ -401,6 +430,8 @@ class Transport: threading.Thread(target=Transport.count_traffic_loop, daemon=True).start() # Load transport-related data + if not Transport.owner.is_connected_to_shared_instance: Transport.init_announce_cache() + if RNS.Reticulum.transport_enabled(): path_table_path = RNS.Reticulum.storagepath+"/destination_table" tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels" @@ -412,6 +443,7 @@ class Transport: serialised_destinations = umsgpack.unpackb(file.read()) file.close() + now = time.time() for serialised_entry in serialised_destinations: destination_hash = serialised_entry[0] @@ -423,6 +455,7 @@ class Transport: random_blobs = serialised_entry[5] receiving_interface = Transport.find_interface_from_hash(serialised_entry[6]) announce_packet = Transport.get_cached_packet(serialised_entry[7], packet_type="announce") + used = serialised_entry[8] if len(serialised_entry) > 8 else now blackholed = False if len(Transport.blackholed_identities) > 0: @@ -438,7 +471,10 @@ class Transport: # increased hop-count. announce_packet.hops += 1 with Transport.path_table_lock: - Transport.path_table[destination_hash] = [timestamp, received_from, hops, expires, random_blobs, receiving_interface, announce_packet.packet_hash] + Transport.path_table[destination_hash] = [timestamp, received_from, hops, expires, + random_blobs, receiving_interface, + announce_packet.packet_hash, used] + RNS.log("Loaded path table entry for "+RNS.prettyhexrep(destination_hash)+" from storage", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None else: RNS.log("Could not reconstruct path table entry from storage for "+RNS.prettyhexrep(destination_hash), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None @@ -466,6 +502,7 @@ class Transport: serialised_tunnels = umsgpack.unpackb(file.read()) file.close() + now = time.time() for serialised_tunnel in serialised_tunnels: tunnel_id = serialised_tunnel[IDX_TT_TUNNEL_ID] interface_hash = serialised_tunnel[IDX_TT_IF] @@ -482,6 +519,7 @@ class Transport: random_blobs = list(set(serialised_entry[5])) receiving_interface = Transport.find_interface_from_hash(serialised_entry[6]) announce_packet = Transport.get_cached_packet(serialised_entry[7], packet_type="announce") + used = serialised_entry[8] if len(serialised_entry) > 8 else now if announce_packet != None: if not announce_packet.unpack(): continue @@ -491,7 +529,7 @@ class Transport: # increased hop-count. announce_packet.hops += 1 - tunnel_path = [timestamp, received_from, hops, path_expires, random_blobs, receiving_interface, announce_packet.packet_hash] + tunnel_path = [timestamp, received_from, hops, path_expires, random_blobs, receiving_interface, announce_packet.packet_hash, used] tunnel_paths[destination_hash] = tunnel_path if len(tunnel_paths) > 0: @@ -543,6 +581,9 @@ class Transport: with Transport.interfaces_lock: if not interface in Transport.interfaces: Transport.interfaces.append(interface) + if interface.announce_rate_target and interface.announce_rate_target > Transport.RATE_TABLE_RETAIN: + Transport.RATE_TABLE_RETAIN = interface.announce_rate_target + RNS.log(f"Increased announce rate table retainment time to {RNS.prettytime(Transport.RATE_TABLE_RETAIN)}", RNS.LOG_DEBUG) @staticmethod def remove_interface(interface): @@ -828,10 +869,9 @@ class Transport: Transport.announces_last_checked = time.time() - # Cull the packet hashlist if it has reached its max size + # Swap the packet hashlist generations if it has reached its max size if len(Transport.packet_hashlist) > Transport.hashlist_maxsize//2: - Transport.packet_hashlist_prev = Transport.packet_hashlist - Transport.packet_hashlist = set() + Transport.packet_hashlist.swap() # Cull invalidated path requests if time.time() > Transport.pending_prs_last_checked+Transport.pending_prs_check_interval: @@ -852,6 +892,24 @@ class Transport: Transport.discovery_pr_tags_prev = Transport.discovery_pr_tags Transport.discovery_pr_tags = set() + # Incrementally cull stale announce rate table entries + if time.time() > Transport.rate_table_last_culled+Transport.rate_table_cull_interval: + with Transport.announce_rate_table_lock: + Transport.announce_rate_table.cull(time.time(), Transport.RATE_TABLE_RETAIN, + Transport.rate_table_cull_chunk, Transport.rate_table_cull_chunk) + + Transport.rate_table_last_culled = time.time() + + # Flush pending known destinations to storage + if RNS.Identity.known_destinations_store != None and time.time() > Transport.kd_last_flush+Transport.kd_flush_interval: + Transport.kd_last_flush = time.time() + RNS.Identity.flush_known_destinations() + + # Flush pending cached announces to storage + if Transport.announce_cache != None and (time.time() > Transport.ac_last_flush+Transport.ac_flush_interval or Transport.announce_cache.pending_count() > Transport.announce_cache_max_pending): + Transport.ac_last_flush = time.time() + Transport.announce_cache.flush() + if time.time() > Transport.tables_last_culled + Transport.tables_cull_interval: # Remove unneeded path state entries stale_path_states = [] @@ -957,25 +1015,42 @@ class Transport: # Cull the path table stale_paths = [] with Transport.path_table_lock: + now = time.time() + check_unused = now > Transport.pool_size_last_checked+Transport.pool_size_check_interval + unused_paths = [] + for destination_hash in Transport.path_table: - destination_entry = Transport.path_table[destination_hash] - attached_interface = destination_entry[IDX_PT_RVCD_IF] + path_entry = Transport.path_table[destination_hash] + attached_interface = path_entry[IDX_PT_RVCD_IF] + path_timestamp = path_entry[IDX_PT_TIMESTAMP] + path_expiry = path_entry[IDX_PT_EXPIRES] + path_used = path_entry[IDX_PT_USED] - if attached_interface != None and hasattr(attached_interface, "mode") and attached_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: - destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.AP_PATH_TIME - elif attached_interface != None and hasattr(attached_interface, "mode") and attached_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING: - destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.ROAMING_PATH_TIME - else: - destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.DESTINATION_TIMEOUT + if check_unused and not path_used: unused_paths.append((path_expiry, destination_hash)) - if time.time() > destination_expiry: + if now > path_expiry: stale_paths.append(destination_hash) should_collect = True - RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None + RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None elif not attached_interface in Transport.interfaces: stale_paths.append(destination_hash) should_collect = True - RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None + RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None + elif path_used == 0 and now > path_timestamp + Transport.unused_path_linger: + stale_paths.append(destination_hash) + should_collect = True + RNS.log(f"Removing never used path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None + + if len(unused_paths) > Transport.PATH_POOL_UNUSED_TARGET: + cull_count = len(unused_paths) - Transport.PATH_POOL_UNUSED_TARGET + RNS.log(f"Culling {cull_count} never used path table entries to reach pool size target {Transport.PATH_POOL_UNUSED_TARGET}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + unused_paths.sort() + for cull_entry in unused_paths[-cull_count:]: + stale_paths.append(cull_entry[1]) + + if check_unused: + Transport._unused_path_count_estimate = len(unused_paths) + Transport.pool_size_last_checked = time.time() # Cull the pending path requests table stale_path_requests = [] @@ -1387,6 +1462,7 @@ class Transport: return False outbound_interface = path_entry[IDX_PT_RVCD_IF] + path_entry[IDX_PT_USED] = outbound_time # If there's more than one hop to the destination, and we know # a path, we insert the packet into transport by adding the next @@ -1403,7 +1479,6 @@ class Transport: new_raw += packet.raw[2:] packet_sent(packet) Transport.transmit(outbound_interface, new_raw) - path_entry[IDX_PT_TIMESTAMP] = time.time() sent = True # In the special case where we are connected to a local shared @@ -1423,7 +1498,6 @@ class Transport: new_raw += packet.raw[2:] packet_sent(packet) Transport.transmit(outbound_interface, new_raw) - path_entry[IDX_PT_TIMESTAMP] = time.time() sent = True # If none of the above applies, we know the destination is @@ -1597,8 +1671,8 @@ class Transport: else: raw = Transport.mangle_hops(packet.raw, Transport.local_hops_delta, transport_insert=packet.header_type==RNS.Packet.HEADER_1) Transport.transmit(interface, raw) - if packet.packet_type == RNS.Packet.ANNOUNCE: interface.sent_announce(size=len(raw)) - if packet.destination.type == RNS.Destination.PLAIN and packet.is_outbound_pr: interface.sent_path_request(size=len(raw)) + if packet.packet_type == RNS.Packet.ANNOUNCE: interface.sent_announce(size=len(raw)); Transport.announce_txc += 1 + if packet.destination.type == RNS.Destination.PLAIN and packet.is_outbound_pr: interface.sent_path_request(size=len(raw)); Transport.pr_txc += 1 packet_sent(packet) sent = True @@ -1665,7 +1739,7 @@ class Transport: if packet.receiving_interface: packet.receiving_interface.protocol_violation("Announce packet with GROUP type") return False - if not packet.packet_hash in Transport.packet_hashlist and not packet.packet_hash in Transport.packet_hashlist_prev: return True + if not packet.packet_hash in Transport.packet_hashlist: return True else: if packet.packet_type == RNS.Packet.ANNOUNCE: if packet.destination_type == RNS.Destination.SINGLE: @@ -1809,8 +1883,9 @@ class Transport: return packet.receiving_interface.protocol_violation(f"Invalid announce signature for {RNS.prettyhexrep(packet.destination_hash)}") if packet.receiving_interface else None elif packet.receiving_interface != None: packet.receiving_interface.received_announce(size=len(packet.raw)) - announced_destination_known = packet.destination_hash in Transport.path_table + Transport.announce_rxc += 1 + announced_destination_known = packet.destination_hash in Transport.path_table if not announced_destination_known: # This is an unknown destination, and we'll apply # potential ingress limiting. Already known @@ -1856,6 +1931,7 @@ class Transport: return if packet.receiving_interface: packet.receiving_interface.received_path_request(size=len(raw)) + Transport.pr_rxc += 1 if interface.should_ingress_limit_pr(): traffic_class = Transport.TC_INGRESS_LIMITED @@ -2110,7 +2186,7 @@ class Transport: if Transport.local_hops_delta != 0 and from_local_client and not to_local_client: new_raw = Transport.mangle_hops(new_raw, Transport.local_hops_delta) Transport.transmit(outbound_interface, new_raw) - path_entry[IDX_PT_TIMESTAMP] = time.time() + path_entry[IDX_PT_USED] = time.time() else: # TODO: There should probably be some kind of REJECT @@ -2159,7 +2235,11 @@ class Transport: new_raw += struct.pack("!B", packet.hops if not from_local_client or instance_local_link or Transport.local_hops_delta == 0 else Transport.local_hops_delta) new_raw += packet.raw[2:] Transport.transmit(outbound_interface, new_raw) - Transport.link_table[packet.destination_hash][IDX_LT_TIMESTAMP] = time.time() + + active_time = time.time() + Transport.link_table[packet.destination_hash][IDX_LT_TIMESTAMP] = active_time + link_path_entry = Transport.path_table.get(Transport.link_table[packet.destination_hash][IDX_LT_DSTHASH]) + if link_path_entry: link_path_entry[IDX_PT_USED] = active_time else: RNS.log(f"No-outbound return on link packet from {packet.receiving_interface}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None @@ -2213,27 +2293,30 @@ class Transport: if local_and_hops_condition: announce_emitted = Transport.announce_emitted(packet) + path_used = 0 random_blobs = [] random_blob = packet.data[RNS.Identity.KEYSIZE//8+RNS.Identity.NAME_HASH_LENGTH//8:RNS.Identity.KEYSIZE//8+RNS.Identity.NAME_HASH_LENGTH//8+10] if not random_blob: return packet.receiving_interface.protocol_violation("No random blob in data") if packet.receiving_interface else None with Transport.inbound_announce_lock: - announced_destination_known = packet.destination_hash in Transport.path_table + existing_path_entry = Transport.path_table.get(packet.destination_hash) + announced_destination_known = existing_path_entry != None if not announced_destination_known: # If this destination is unknown in our table # we should add it should_add = True else: - random_blobs = Transport.path_table[packet.destination_hash][IDX_PT_RANDBLOBS] - current_gravity = Transport.path_table[packet.destination_hash][IDX_PT_RVCD_IF].gravity + random_blobs = existing_path_entry[IDX_PT_RANDBLOBS] + current_gravity = existing_path_entry[IDX_PT_RVCD_IF].gravity + path_used = existing_path_entry[IDX_PT_USED] announce_gravity = packet.receiving_interface.gravity if packet.receiving_interface != None else None # If we already have a path to the announced destination, # but a more recently emitted announce arrives with a hop # count equal to or less than the existing path, we will # update our tables. - if packet.hops <= Transport.path_table[packet.destination_hash][IDX_PT_HOPS]: + if packet.hops <= existing_path_entry[IDX_PT_HOPS]: path_timebase = Transport.timebase_from_random_blobs(random_blobs) if not random_blob in random_blobs and announce_emitted > path_timebase: Transport.mark_path_unknown_state(packet.destination_hash) @@ -2255,7 +2338,7 @@ class Transport: # ignore it, unless the path is expired, or # the emission timestamp is more recent. now = time.time() - path_expires = Transport.path_table[packet.destination_hash][IDX_PT_EXPIRES] + path_expires = existing_path_entry[IDX_PT_EXPIRES] path_announce_emitted = 0 for path_random_blob in random_blobs: @@ -2302,33 +2385,10 @@ class Transport: rate_blocked = False if packet.context != RNS.Packet.PATH_RESPONSE and packet.receiving_interface.announce_rate_target != None: with Transport.announce_rate_table_lock: - if not packet.destination_hash in Transport.announce_rate_table: - rate_entry = { "last": now, "rate_violations": 0, "blocked_until": 0, "timestamps": [now]} - Transport.announce_rate_table[packet.destination_hash] = rate_entry - - else: - rate_entry = Transport.announce_rate_table[packet.destination_hash] - rate_entry["timestamps"].append(now) - - while len(rate_entry["timestamps"]) > Transport.MAX_RATE_TIMESTAMPS: - rate_entry["timestamps"].pop(0) - - current_rate = now - rate_entry["last"] - - if now > rate_entry["blocked_until"]: - if current_rate < packet.receiving_interface.announce_rate_target: rate_entry["rate_violations"] += 1 - else: rate_entry["rate_violations"] = max(0, rate_entry["rate_violations"]-1) - - if rate_entry["rate_violations"] > packet.receiving_interface.announce_rate_grace: - rate_target = packet.receiving_interface.announce_rate_target - rate_penalty = packet.receiving_interface.announce_rate_penalty - rate_entry["blocked_until"] = rate_entry["last"] + rate_target + rate_penalty - rate_blocked = True - else: - rate_entry["last"] = now - - else: rate_blocked = True - + rate_blocked = Transport.announce_rate_table.check_and_update(packet.destination_hash, now, + packet.receiving_interface.announce_rate_target, + packet.receiving_interface.announce_rate_grace, + packet.receiving_interface.announce_rate_penalty) retries = 0 announce_hops = packet.hops local_rebroadcasts = 0 @@ -2336,13 +2396,7 @@ class Transport: attached_interface = None retransmit_timeout = now + (RNS.rand() * Transport.PATHFINDER_RW) - - if hasattr(packet.receiving_interface, "mode") and packet.receiving_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: - expires = now + Transport.AP_PATH_TIME - elif hasattr(packet.receiving_interface, "mode") and packet.receiving_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING: - expires = now + Transport.ROAMING_PATH_TIME - else: - expires = now + Transport.PATHFINDER_E + expires = now + Transport.path_timeout_from_interface(packet.receiving_interface) if not random_blob in random_blobs: random_blobs.append(random_blob) @@ -2455,8 +2509,21 @@ class Transport: new_announce.send() if not Transport.owner.is_connected_to_shared_instance: Transport.cache(packet, force_cache=True, packet_type="announce") - path_table_entry = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet.packet_hash] - with Transport.path_table_lock: Transport.path_table[packet.destination_hash] = path_table_entry + if rate_blocked: + # Rate-limited announces do not update liveness + # metrics, only the essential routing data. + with Transport.path_table_lock: + rate_limited_path_entry = Transport.path_table.get(packet.destination_hash) + if rate_limited_path_entry != None: + rate_limited_path_entry[IDX_PT_NEXT_HOP] = received_from + rate_limited_path_entry[IDX_PT_HOPS] = announce_hops + rate_limited_path_entry[IDX_PT_RANDBLOBS] = random_blobs + rate_limited_path_entry[IDX_PT_RVCD_IF] = packet.receiving_interface + rate_limited_path_entry[IDX_PT_PACKET] = packet.packet_hash + else: + path_table_entry = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet.packet_hash, path_used] + with Transport.path_table_lock: Transport.path_table[packet.destination_hash] = path_table_entry + Transport.mark_path_unknown_state(packet.destination_hash) RNS.log("Destination "+RNS.prettyhexrep(packet.destination_hash)+" is now "+str(announce_hops)+" hops away via "+RNS.prettyhexrep(received_from)+" on "+str(packet.receiving_interface), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None if packet.destination_hash in Transport.path_requests: @@ -2464,13 +2531,13 @@ class Transport: # If the receiving interface is a tunnel, we add the # announce to the tunnels table - if hasattr(packet.receiving_interface, "tunnel_id") and packet.receiving_interface.tunnel_id != None: + if not rate_blocked and hasattr(packet.receiving_interface, "tunnel_id") and packet.receiving_interface.tunnel_id != None: with Transport.tunnels_lock: if not packet.receiving_interface.tunnel_id in Transport.tunnels: RNS.log(f"Tunnel ID for {packet.receiving_interface} was not found in tunnel table", RNS.LOG_WARNING) else: tunnel_entry = Transport.tunnels[packet.receiving_interface.tunnel_id] paths = tunnel_entry[IDX_TT_PATHS] - paths[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, None, packet.packet_hash] + paths[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, None, packet.packet_hash, path_used] expires = time.time() + Transport.TUNNEL_TIMEOUT tunnel_entry[IDX_TT_EXPIRES] = expires RNS.log("Path to "+RNS.prettyhexrep(packet.destination_hash)+" associated with tunnel "+RNS.prettyhexrep(packet.receiving_interface.tunnel_id), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None @@ -2590,8 +2657,7 @@ class Transport: # to another path, we remove this packet hash from # the filter hashlist so the link can receive the # packet when it finally arrives over another path. - while packet.packet_hash in Transport.packet_hashlist: Transport.packet_hashlist.remove(packet.packet_hash) - while packet.packet_hash in Transport.packet_hashlist_prev: Transport.packet_hashlist_prev.remove(packet.packet_hash) + Transport.packet_hashlist.discard(packet.packet_hash) else: destination = Transport.destinations_map.get(packet.destination_hash) if destination and destination.type == packet.destination_type: @@ -2632,6 +2698,7 @@ class Transport: link_entry[IDX_LT_REM_HOPS] = packet.hops path_entry = Transport.path_table.get(link_destination) if path_entry: path_entry[IDX_PT_HOPS] = packet.hops + Transport.mark_path_responsive(link_destination) elif not link_entry[IDX_LT_VALIDATED]: RNS.log(f"Aborting link request proof path re-balancing for {RNS.prettyhexrep(link_destination)} on link {RNS.prettyhexrep(packet.destination_hash)} due to invalid signature", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None else: pass @@ -2662,6 +2729,7 @@ class Transport: Transport.transmit(link_entry[IDX_LT_RCVD_IF], new_raw) if not Transport.owner.is_connected_to_shared_instance: RNS.Identity._used_destination_data(link_entry[IDX_LT_DSTHASH]) + Transport.mark_path_responsive(link_entry[IDX_LT_DSTHASH]) else: RNS.log("Invalid link request proof in transport for link "+RNS.prettyhexrep(packet.destination_hash)+", dropping proof.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None @@ -2706,6 +2774,7 @@ class Transport: if path_entry: path_entry[IDX_PT_HOPS] = packet.hops RNS.log(f"Path table re-balanced for {RNS.prettyhexrep(link.destination.hash)}", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None + Transport.mark_path_responsive(link.destination.hash) else: RNS.log(f"Aborting path re-balancing at link terminus for {RNS.prettyhexrep(link.destination.hash)} on link {link} due to invalid signature", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None except Exception as e: RNS.log("Error while validating link request proof for path re-balancing at link terminus. The contained exception was: "+str(e), REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None @@ -2837,13 +2906,15 @@ class Transport: deprecated_paths = [] with Transport.tunnels_lock: for destination_hash, path_entry in paths.items(): + path_timestamp = path_entry[0] received_from = path_entry[1] announce_hops = path_entry[2] expires = path_entry[3] random_blobs = list(set(path_entry[4])) receiving_interface = interface packet_hash = path_entry[6] - new_entry = [time.time(), received_from, announce_hops, expires, random_blobs, receiving_interface, packet_hash] + used = path_entry[7] + new_entry = [path_timestamp, received_from, announce_hops, expires, random_blobs, receiving_interface, packet_hash, used] should_add = False with Transport.path_table_lock: @@ -2991,6 +3062,21 @@ class Transport: return None + @staticmethod + def init_announce_cache(): + try: + Transport.announce_cache = AnnounceCache(os.path.join(RNS.Reticulum.cachepath, "announces.db")) + + legacy_dir = os.path.join(RNS.Reticulum.cachepath, "announces") + if os.path.isdir(legacy_dir) and len(os.listdir(legacy_dir)) > 0: + st = time.time() + imported = Transport.announce_cache.import_legacy(legacy_dir) + RNS.log(f"Imported {imported} cached announces from legacy file storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE) + + except Exception as e: + RNS.log("Could not initialise announce cache database, falling back to file-based caching. The contained exception was: "+str(e), RNS.LOG_ERROR) + Transport.announce_cache = None + @staticmethod def should_cache(packet): # TODO: Rework the caching system. It's currently @@ -3024,10 +3110,17 @@ class Transport: @staticmethod def clean_announce_cache(): st = time.time() + with Transport.path_table_lock: active_path_hashes = set([Transport.path_table[dst_hash][IDX_PT_PACKET] for dst_hash in Transport.path_table]) + with Transport.tunnels_lock: tunnel_path_hashes = set([path_dict[dst_hash][IDX_PT_PACKET] for path_dict in [Transport.tunnels[tunnel_id][IDX_TT_PATHS] for tunnel_id in Transport.tunnels] for dst_hash in path_dict]) + active_path_hashes |= tunnel_path_hashes + + if Transport.announce_cache != None: + removed = Transport.announce_cache.clean(active_path_hashes) + if removed > 0: RNS.log(f"Removed {removed} cached announces in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + return + target_path = os.path.join(RNS.Reticulum.cachepath, "announces") cached_announce_hashes = os.listdir(target_path) - with Transport.path_table_lock: active_path_hashes = list(set([Transport.path_table[dst_hash][IDX_PT_PACKET] for dst_hash in Transport.path_table])) - with Transport.tunnels_lock: tunnel_path_hashes = list(set([path_dict[dst_hash][IDX_PT_PACKET] for path_dict in [Transport.tunnels[tunnel_id][IDX_TT_PATHS] for tunnel_id in Transport.tunnels] for dst_hash in path_dict])) removed = 0; total = 0 for packet_hash in cached_announce_hashes: remove = False @@ -3053,22 +3146,40 @@ class Transport: def cache(packet, force_cache=False, packet_type=None): if force_cache or RNS.Transport.should_cache(packet): try: - packet_hash = RNS.hexrep(packet.get_hash(), delimit=False) interface_reference = None if packet.receiving_interface != None: interface_reference = str(packet.receiving_interface) - if packet_type == "announce": cachepath = os.path.join(RNS.Reticulum.cachepath, "announces", packet_hash) - else: cachepath = os.path.join(RNS.Reticulum.cachepath, packet_hash) + + if packet_type == "announce": + if Transport.announce_cache != None: + Transport.announce_cache.store(packet.get_hash(), packet.raw, interface_reference) + return + + cachepath = os.path.join(RNS.Reticulum.cachepath, "announces", RNS.hexrep(packet.get_hash(), delimit=False)) + else: + cachepath = os.path.join(RNS.Reticulum.cachepath, RNS.hexrep(packet.get_hash(), delimit=False)) file = open(cachepath, "wb") file.write(umsgpack.packb([packet.raw, interface_reference])) file.close() - except Exception as e: - RNS.log("Error writing packet to cache. The contained exception was: "+str(e), RNS.LOG_ERROR) + except Exception as e: RNS.log("Error writing packet to cache. The contained exception was: "+str(e), RNS.LOG_ERROR) @staticmethod def get_cached_packet(packet_hash, packet_type=None): try: + if packet_type == "announce" and Transport.announce_cache != None: + cached = Transport.announce_cache.fetch(packet_hash) + if cached == None: return None + + packet = RNS.Packet(None, cached[0]) + interface_reference = cached[1] + + for interface in Transport.interfaces: + if str(interface) == interface_reference: + packet.receiving_interface = interface + + return packet + packet_hash = RNS.hexrep(packet_hash, delimit=False) if packet_type == "announce": path = os.path.join(RNS.Reticulum.cachepath, "announces", packet_hash) else: path = os.path.join(RNS.Reticulum.cachepath, packet_hash) @@ -3207,6 +3318,22 @@ class Transport: if not Transport.lowest_interface_bitrate: return 0 return 2*(RNS.Reticulum.MTU*8/max(Transport.lowest_interface_bitrate, RNS.Reticulum.MINIMUM_BITRATE)) + RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT + @staticmethod + def path_count(): + return len(Transport.path_table) + + @staticmethod + def unused_path_count(): + with Transport.path_table_lock: + return sum(1 for e in Transport.path_table.values() if not e[IDX_PT_USED]) + + @staticmethod + def path_pool_counts(): + with Transport.path_table_lock: + total = len(Transport.path_table) + unused = sum(1 for e in Transport.path_table.values() if not e[IDX_PT_USED]) + return total, unused + @staticmethod def link_count(): return len(Transport.link_table) @@ -3215,16 +3342,46 @@ class Transport: def active_link_count(): return sum(1 for e in (True for entry in Transport.link_table if entry[IDX_LT_VALIDATED])) + @staticmethod + def link_distribution(): + link_destinations = {} + + # TODO: Finish implementation + return link_destinations + + # try: + # for link_id in Transport.link_table: + # link_entry = Transport.link_table.get(link_id) + # if link_entry: + # link_destination = link_entry[IDX_LT_DSTHASH] + # active = link_entry[IDX_LT_VALIDATED] + + # if not link_destination in link_destinations: + # link_destinations[link_destination] = {} + + # else: + # pass + + # except Exception as e: RNS.log(f"Error while getting link distribution stats: {e}", RNS.LOG_DEBUG) + @staticmethod def expire_path(destination_hash): with Transport.path_table_lock: if destination_hash in Transport.path_table: Transport.path_table[destination_hash][IDX_PT_TIMESTAMP] = 0 + Transport.path_table[destination_hash][IDX_PT_EXPIRES] = 0 Transport.tables_last_culled = 0 return True else: return False + @staticmethod + def path_timeout_from_interface(interface): + if interface != None and hasattr(interface, "mode"): + if interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: return Transport.AP_PATH_TIME + elif interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING: return Transport.ROAMING_PATH_TIME + return Transport.PATH_TIMEOUT + @staticmethod def mark_path_unresponsive(destination_hash): if destination_hash in Transport.path_table: @@ -3235,11 +3392,22 @@ class Transport: @staticmethod def mark_path_responsive(destination_hash): - if destination_hash in Transport.path_table: - with Transport.path_states_lock: Transport.path_states[destination_hash] = Transport.STATE_RESPONSIVE - return True - - else: return False + # Marking a path live must only occur on events that + # cryptographically prove recent destination liveness. + now = time.time() + with Transport.path_table_lock: + path_entry = Transport.path_table.get(destination_hash) + if path_entry != None: + path_entry[IDX_PT_TIMESTAMP] = now + path_entry[IDX_PT_EXPIRES] = now + Transport.path_timeout_from_interface(path_entry[IDX_PT_RVCD_IF]) + path_entry[IDX_PT_USED] = now + + with Transport.path_states_lock: + Transport.path_states[destination_hash] = Transport.STATE_RESPONSIVE + + return True + + else: return False @staticmethod def mark_path_unknown_state(destination_hash): @@ -3767,14 +3935,14 @@ class Transport: packet_hashlist_path = RNS.Reticulum.storagepath+"/packet_hashlist.raw" with open(packet_hashlist_path, "wb") as file: - for packet_hash in Transport.packet_hashlist.copy(): + for bucket in Transport.packet_hashlist.buckets(): if background: if time.time() - round_started_at > yield_threshold: # Low priority, yield thread round_started_at = time.time() time.sleep(0.001) - file.write(packet_hash) + file.write(bucket) RNS.log(f"Saved packet hashlist in {RNS.prettyshorttime(time.time()-save_start)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None @@ -3839,6 +4007,7 @@ class Transport: expires = de[IDX_PT_EXPIRES] random_blobs = de[IDX_PT_RANDBLOBS] packet_hash = de[IDX_PT_PACKET] + used = de[IDX_PT_USED] interface_hash = interface.get_hash() serialised_entry = [ destination_hash, @@ -3848,7 +4017,8 @@ class Transport: expires, random_blobs, interface_hash, - packet_hash ] + packet_hash, + used ] serialised_destinations.append(serialised_entry) @@ -3922,6 +4092,7 @@ class Transport: path_expires = de[3] random_blobs = de[4][-Transport.PERSIST_RANDOM_BLOBS:] packet_hash = de[6] + used = de[7] serialised_entry = [ destination_hash, @@ -3931,7 +4102,8 @@ class Transport: path_expires, random_blobs, interface_hash, - packet_hash + packet_hash, + used, ] serialised_paths.append(serialised_entry) @@ -3979,6 +4151,7 @@ class Transport: Transport.void_queues() if not Transport.owner.is_connected_to_shared_instance: Transport.persist_data() + if Transport.announce_cache != None: Transport.announce_cache.flush() @staticmethod def blackhole_identity(identity_hash, until=None, reason=None): @@ -4126,6 +4299,7 @@ IDX_PT_EXPIRES = 3 IDX_PT_RANDBLOBS = 4 IDX_PT_RVCD_IF = 5 IDX_PT_PACKET = 6 +IDX_PT_USED = 7 # Transport.reverse_table entry indices IDX_RT_RCVD_IF = 0 diff --git a/RNS/Utilities/__init__.py b/RNS/Utilities/__init__.py index 454b31ff..1f227878 100644 --- a/RNS/Utilities/__init__.py +++ b/RNS/Utilities/__init__.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rncp.py b/RNS/Utilities/rncp.py index 151a5723..e6f004ae 100644 --- a/RNS/Utilities/rncp.py +++ b/RNS/Utilities/rncp.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnid.py b/RNS/Utilities/rnid.py index 3731fd8e..18bfecf5 100644 --- a/RNS/Utilities/rnid.py +++ b/RNS/Utilities/rnid.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnir.py b/RNS/Utilities/rnir.py index 2d1002fb..57e761d0 100644 --- a/RNS/Utilities/rnir.py +++ b/RNS/Utilities/rnir.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnodeconf.py b/RNS/Utilities/rnodeconf.py index c799a38f..0ff83487 100755 --- a/RNS/Utilities/rnodeconf.py +++ b/RNS/Utilities/rnodeconf.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnpath.py b/RNS/Utilities/rnpath.py index 3f65a691..2d0404ec 100644 --- a/RNS/Utilities/rnpath.py +++ b/RNS/Utilities/rnpath.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal @@ -102,7 +102,7 @@ def parse_hash(input_str): def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity, timeout, drop_queues, drop_via, max_hops, remote=None, management_identity=None, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT, blackholed=False, blackhole=False, unblackhole=False, blackhole_duration=None, blackhole_reason=None, - remote_blackhole_list=False, remote_blackhole_list_filter=None, no_output=False, json=False): + remote_blackhole_list=False, remote_blackhole_list_filter=None, usage=False, no_output=False, json=False): global remote_link, reticulum reticulum = RNS.Reticulum(configdir = configdir, loglevel = 3+verbosity) @@ -281,13 +281,22 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity, exit() else: + now = time.time() + if usage: + try: table = sorted(table, key=lambda e: e["used"], reverse=True) + except Exception as e: pass + for path in table: if destination_hash == None or destination_hash == path["hash"]: displayed += 1 exp_str = RNS.timestamp_str(path["expires"]) if path["hops"] == 1: m_str = " " else: m_str = "s" - print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"])) + if not "used" in path: u_str = ", no usage info" + else: + if usage: u_str = f", last used {RNS.prettytime(now-path['used'])} ago" if path["used"] else ", never used" + else: u_str = "" + print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"])+u_str) if destination_hash != None and displayed == 0: print("No path known") @@ -484,6 +493,7 @@ def main(): parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str) parser.add_argument("--version", action="version", version="rnpath {version}".format(version=__version__)) parser.add_argument("-t", "--table", action="store_true", help="show all known paths", default=False) + parser.add_argument("-u", "--usage", action="store_true", help="show path usage time", default=False) parser.add_argument("-m", "--max", action="store", metavar="hops", type=int, help="maximum hops to filter path table by", default=None) parser.add_argument("-r", "--rates", action="store_true", help="show announce rate info", default=False) parser.add_argument("-d", "--drop", action="store_true", help="remove the path to a destination", default=False) @@ -518,7 +528,7 @@ def main(): verbosity = args.verbose, timeout = args.w, drop_queues = args.drop_announces, drop_via = args.drop_via, max_hops = args.max, remote=args.R, management_identity=args.i, remote_timeout=args.W, blackholed=args.blackholed, blackhole=args.blackhole, unblackhole=args.unblackhole, blackhole_duration=args.duration, blackhole_reason=args.reason, remote_blackhole_list=args.blackholed_list, - remote_blackhole_list_filter=args.list_filter, json=args.json) + remote_blackhole_list_filter=args.list_filter, usage=args.usage, json=args.json) sys.exit(0) diff --git a/RNS/Utilities/rnpkg.py b/RNS/Utilities/rnpkg.py index 15e54ceb..ecdab3f4 100644 --- a/RNS/Utilities/rnpkg.py +++ b/RNS/Utilities/rnpkg.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnprobe.py b/RNS/Utilities/rnprobe.py index 40603805..f2188082 100644 --- a/RNS/Utilities/rnprobe.py +++ b/RNS/Utilities/rnprobe.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnsd.py b/RNS/Utilities/rnsd.py index 4e4e5ea3..24a3e35a 100755 --- a/RNS/Utilities/rnsd.py +++ b/RNS/Utilities/rnsd.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnstatus.py b/RNS/Utilities/rnstatus.py index c69b62bb..65e7e1d3 100644 --- a/RNS/Utilities/rnstatus.py +++ b/RNS/Utilities/rnstatus.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/Utilities/rnx.py b/RNS/Utilities/rnx.py index 4bfcf83b..4623c595 100644 --- a/RNS/Utilities/rnx.py +++ b/RNS/Utilities/rnx.py @@ -2,7 +2,7 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/RNS/__init__.py b/RNS/__init__.py index 81f9b75d..d20c579e 100755 --- a/RNS/__init__.py +++ b/RNS/__init__.py @@ -1,6 +1,6 @@ # Reticulum License # -# Copyright (c) 2016-2025 Mark Qvist +# Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal diff --git a/tests/throughput.py b/tests/throughput.py index 2ec6757a..550b3bc2 100644 --- a/tests/throughput.py +++ b/tests/throughput.py @@ -304,9 +304,7 @@ def build_scenarios(instance, interface, interface_b, requested=None): salt=b"bench", context=None) now = time.time() with RNS.Transport.path_table_lock: - RNS.Transport.path_table[dst_h] = [ - now, next_hop, 3, now + 3600, [], interface, bytes(32) - ] + RNS.Transport.path_table[dst_h] = [now, next_hop, 3, now + 3600, [], interface, bytes(32), now] frames = [_transit_single_frame(interface, dst_h, _payload(i, size, 0x57)) for i in range(total)] @@ -327,9 +325,7 @@ def build_scenarios(instance, interface, interface_b, requested=None): salt=b"bench", context=None) now = time.time() with RNS.Transport.path_table_lock: - RNS.Transport.path_table[dst_h] = [ - now, next_hop, 1, now + 3600, [], interface, bytes(32) - ] + RNS.Transport.path_table[dst_h] = [now, next_hop, 1, now + 3600, [], interface, bytes(32), now] frames = [_transit_single_frame(interface, dst_h, _payload(i, 135, 0x46)) for i in range(total)] @@ -450,9 +446,7 @@ def build_scenarios(instance, interface, interface_b, requested=None): salt=b"bench", context=None) now = time.time() with RNS.Transport.path_table_lock: - RNS.Transport.path_table[dst_h] = [ - now, next_hop, 3, now + 3600, [], interface, bytes(32) - ] + RNS.Transport.path_table[dst_h] = [now, next_hop, 3, now + 3600, [], interface, bytes(32), now] remote_id = RNS.Identity(create_keys=False) remote_id.load_public_key(os.urandom(RNS.Identity.KEYSIZE//8)) outbound_destination = RNS.Destination(remote_id, RNS.Destination.OUT,