mirror of
https://github.com/markqvist/Reticulum.git
synced 2026-11-19 14:36:34 -05:00
Merged staging
This commit is contained in:
parent
f3ebec9cb1
commit
e24d207312
25 changed files with 1123 additions and 174 deletions
18
Changelog.md
18
Changelog.md
|
|
@ -1,3 +1,21 @@
|
|||
### RNS 1.6.0
|
||||
|
||||
*Pending*.
|
||||
|
||||
**Changes**
|
||||
- Added path usage tracking
|
||||
- Added automatic path table culling based on traffic and usage
|
||||
- Added unused path pool monitoring and handling
|
||||
- Added managed storage backend using SQLite
|
||||
- Added fallback to legacy file-based storage if SQLite is unavailable
|
||||
- Added managed storage for known destinations and ratchets
|
||||
- Added managed storage for cached announces
|
||||
- Added more comprehensive metrics to status output structure
|
||||
- Optimized announce rate table implementation
|
||||
- Optimized packet filter hashlist implementation
|
||||
- Fixed `RNodeInterface` phy stats regression on Android
|
||||
- Fixed `rngit` default config path typo for `~/.config/rngit` case
|
||||
|
||||
### 2026-10-05: RNS 1.5.7
|
||||
|
||||
This maintenance release fixes a bug in the dataplane egress control handling.
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
232
RNS/Identity.py
232
RNS/Identity.py
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
@ -40,6 +40,7 @@ from .vendor import umsgpack as umsgpack
|
|||
|
||||
from RNS.Cryptography import X25519PrivateKey, X25519PublicKey, Ed25519PrivateKey, Ed25519PublicKey
|
||||
from RNS.Cryptography import Token
|
||||
from RNS.Storage import KnownDestinationsStore
|
||||
|
||||
|
||||
class Identity:
|
||||
|
|
@ -92,6 +93,7 @@ class Identity:
|
|||
|
||||
# Storage
|
||||
known_destinations = {}
|
||||
known_destinations_store = None
|
||||
known_ratchets = {}
|
||||
|
||||
ratchet_persist_lock = threading.Lock()
|
||||
|
|
@ -104,7 +106,7 @@ class Identity:
|
|||
else:
|
||||
with Identity.known_destinations_lock:
|
||||
if not destination_hash in Identity.known_destinations:
|
||||
Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0]
|
||||
Identity.known_destinations[destination_hash] = [time.time(), packet_hash, public_key, app_data, 0, None, 0]
|
||||
else:
|
||||
entry = Identity.known_destinations[destination_hash]
|
||||
entry[0] = time.time()
|
||||
|
|
@ -112,6 +114,8 @@ class Identity:
|
|||
entry[2] = public_key
|
||||
entry[3] = app_data
|
||||
|
||||
Identity._kd_mark_dirty(destination_hash)
|
||||
|
||||
@staticmethod
|
||||
def recall(target_hash, from_identity_hash=False, _no_use=False):
|
||||
"""
|
||||
|
|
@ -177,7 +181,12 @@ class Identity:
|
|||
def save_known_destinations(background=False, recombine=False):
|
||||
if recombine: RNS.log(f"Recombining known destinations from disk cache on persist is deprecated, argument ignored", RNS.LOG_WARNING)
|
||||
if RNS.Transport.owner.is_connected_to_shared_instance: return
|
||||
|
||||
|
||||
if Identity.known_destinations_store != None: Identity.flush_known_destinations()
|
||||
else: Identity.save_known_destinations_legacy(background=background)
|
||||
|
||||
@staticmethod
|
||||
def save_known_destinations_legacy(background=False):
|
||||
try:
|
||||
if hasattr(Identity, "saving_known_destinations"):
|
||||
wait_interval = 0.2
|
||||
|
|
@ -214,6 +223,72 @@ class Identity:
|
|||
|
||||
@staticmethod
|
||||
def load_known_destinations():
|
||||
if RNS.Reticulum.get_instance().is_connected_to_shared_instance:
|
||||
try:
|
||||
store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True)
|
||||
count = 0
|
||||
with Identity.known_destinations_lock:
|
||||
for destination_hash, entry in store.load_all():
|
||||
Identity.known_destinations[destination_hash] = entry
|
||||
count += 1
|
||||
|
||||
store.close()
|
||||
RNS.log(f"Loaded {count} known destinations from shared instance database", RNS.LOG_VERBOSE)
|
||||
|
||||
except Exception as e:
|
||||
with Identity.known_destinations_lock: Identity.known_destinations = {}
|
||||
RNS.log("Could not load known destinations from shared instance database, continuing with an empty table. The contained exception was: "+str(e), RNS.LOG_ERROR)
|
||||
|
||||
return
|
||||
|
||||
store = None
|
||||
try: store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8)
|
||||
except Exception as e:
|
||||
RNS.log("Could not initialise known destinations database, falling back to legacy snapshot persistence. The contained exception was: "+str(e), RNS.LOG_ERROR)
|
||||
Identity.known_destinations_store = None
|
||||
|
||||
if store != None:
|
||||
Identity.known_destinations_store = store
|
||||
|
||||
legacy_path = RNS.Reticulum.storagepath+"/known_destinations"
|
||||
if os.path.isfile(legacy_path):
|
||||
try:
|
||||
imported = store.import_legacy(legacy_path)
|
||||
os.rename(legacy_path, legacy_path+".imported")
|
||||
RNS.log(f"Imported {imported} known destinations from legacy storage into the known destinations database", RNS.LOG_VERBOSE)
|
||||
|
||||
except Exception as e:
|
||||
RNS.log("Could not import legacy known destinations file into the known destinations database. The contained exception was: "+str(e), RNS.LOG_ERROR)
|
||||
RNS.trace_exception(e)
|
||||
|
||||
st = time.time()
|
||||
try:
|
||||
count = 0
|
||||
with Identity.known_destinations_lock:
|
||||
for destination_hash, entry in store.load_all():
|
||||
Identity.known_destinations[destination_hash] = entry
|
||||
count += 1
|
||||
|
||||
RNS.log(f"Loaded {count} known destinations from storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE)
|
||||
|
||||
except Exception as e:
|
||||
RNS.log("Error loading known destinations from storage, continuing with an empty table", RNS.LOG_ERROR)
|
||||
RNS.trace_exception(e)
|
||||
|
||||
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
|
||||
if os.path.isdir(ratchetdir) and len(os.listdir(ratchetdir)) > 0:
|
||||
try:
|
||||
imported = store.import_legacy_ratchets(ratchetdir, time.time()-Identity.RATCHET_EXPIRY, Identity.known_destinations)
|
||||
RNS.log(f"Imported {imported} ratchets from legacy file storage", RNS.LOG_VERBOSE)
|
||||
|
||||
except Exception as e:
|
||||
RNS.log(f"Could not import legacy ratchet files into the known destinations database. The contained exception was: {e}", RNS.LOG_ERROR)
|
||||
RNS.trace_exception(e)
|
||||
|
||||
else: Identity.load_known_destinations_legacy()
|
||||
|
||||
@staticmethod
|
||||
def load_known_destinations_legacy():
|
||||
if os.path.isfile(RNS.Reticulum.storagepath+"/known_destinations"):
|
||||
st = time.time()
|
||||
try:
|
||||
|
|
@ -244,6 +319,7 @@ class Identity:
|
|||
if destination_hash in Identity.known_destinations:
|
||||
if not Identity.known_destinations[destination_hash][4] < 0:
|
||||
Identity.known_destinations[destination_hash][4] = time.time()
|
||||
Identity._kd_mark_dirty(destination_hash)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -253,6 +329,7 @@ class Identity:
|
|||
with Identity.known_destinations_lock:
|
||||
if destination_hash in Identity.known_destinations:
|
||||
Identity.known_destinations[destination_hash][4] = -1
|
||||
Identity._kd_mark_dirty(destination_hash)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -262,6 +339,7 @@ class Identity:
|
|||
with Identity.known_destinations_lock:
|
||||
if destination_hash in Identity.known_destinations:
|
||||
Identity.known_destinations[destination_hash][4] = time.time()
|
||||
Identity._kd_mark_dirty(destination_hash)
|
||||
return True
|
||||
|
||||
return False
|
||||
|
|
@ -327,7 +405,7 @@ class Identity:
|
|||
|
||||
if not is_retained and not has_path:
|
||||
if not was_used and now - last_announce > RNS.Transport.UNUSED_DESTINATION_LINGER: stale.append(destination_hash)
|
||||
elif unused_for > RNS.Transport.DESTINATION_TIMEOUT*1.25: stale.append(destination_hash)
|
||||
elif unused_for > RNS.Transport.PATH_TIMEOUT*1.25: stale.append(destination_hash)
|
||||
|
||||
except Exception as e: RNS.log(f"Faulty entry for {RNS.prettyhexrep(destination_hash)} while cleaning known destinations: {e}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
|
||||
|
|
@ -336,13 +414,23 @@ class Identity:
|
|||
with Identity.known_destinations_lock:
|
||||
if destination_hash in Identity.known_destinations:
|
||||
Identity.known_destinations.pop(destination_hash)
|
||||
Identity._kd_mark_deleted(destination_hash)
|
||||
if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash)
|
||||
removed += 1
|
||||
|
||||
try:
|
||||
hexhash = RNS.hexrep(destination_hash, delimit=False)
|
||||
ratchet_path = f"{ratchetdir}/{hexhash}"
|
||||
if os.path.isfile(ratchet_path): os.unlink(ratchet_path)
|
||||
except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
|
||||
if not Identity.known_destinations_store:
|
||||
try:
|
||||
hexhash = RNS.hexrep(destination_hash, delimit=False)
|
||||
ratchet_path = f"{ratchetdir}/{hexhash}"
|
||||
if os.path.isfile(ratchet_path): os.unlink(ratchet_path)
|
||||
except Exception as e: RNS.log(f"Could not clean stale ratchets for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
|
||||
|
||||
if Identity.known_destinations_store != None:
|
||||
expired_ratchets = Identity.known_destinations_store.expire_ratchets(time.time()-Identity.RATCHET_EXPIRY)
|
||||
if len(expired_ratchets) > 0:
|
||||
for destination_hash in expired_ratchets:
|
||||
with Identity.known_destinations_lock:
|
||||
if destination_hash in Identity.known_ratchets: Identity.known_ratchets.pop(destination_hash)
|
||||
|
||||
RNS.log(f"Cleaned known destinations in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
RNS.log(f"Total: {total}, stale: {len(stale)}, removed: {removed}, no path: {no_path}, never used: {never_used}, with path: {total-no_path}, used: {total-never_used}, retained: {retained}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
|
|
@ -409,33 +497,40 @@ class Identity:
|
|||
@staticmethod
|
||||
def _remember_ratchet(destination_hash, ratchet):
|
||||
try:
|
||||
if destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet:
|
||||
ratchet_exists = True
|
||||
# Don't re-insert existing ratchet
|
||||
if Identity.known_ratchets.get(destination_hash) == ratchet: return
|
||||
|
||||
store = Identity.known_destinations_store
|
||||
if store != None:
|
||||
# Persist to store, only update in-memory cache if this
|
||||
# destination already exists there.
|
||||
store.remember_ratchet(destination_hash, ratchet, time.time())
|
||||
if destination_hash in Identity.known_ratchets:
|
||||
Identity.known_ratchets[destination_hash] = ratchet
|
||||
|
||||
# Legacy store: Use file system
|
||||
# Shared instance client: Just keep in memory cache
|
||||
else:
|
||||
ratchet_exists = False
|
||||
ratchet_exists = destination_hash in Identity.known_ratchets and Identity.known_ratchets[destination_hash] == ratchet
|
||||
if not ratchet_exists:
|
||||
RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
|
||||
Identity.known_ratchets[destination_hash] = ratchet
|
||||
if not RNS.Transport.owner.is_connected_to_shared_instance:
|
||||
def persist_job():
|
||||
with Identity.ratchet_persist_lock:
|
||||
hexhash = RNS.hexrep(destination_hash, delimit=False)
|
||||
ratchet_data = {"ratchet": ratchet, "received": time.time()}
|
||||
|
||||
if not ratchet_exists:
|
||||
RNS.log(f"Remembering ratchet {RNS.prettyhexrep(Identity._get_ratchet_id(ratchet))} for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
|
||||
Identity.known_ratchets[destination_hash] = ratchet
|
||||
if not RNS.Transport.owner.is_connected_to_shared_instance:
|
||||
def persist_job():
|
||||
with Identity.ratchet_persist_lock:
|
||||
hexhash = RNS.hexrep(destination_hash, delimit=False)
|
||||
ratchet_data = {"ratchet": ratchet, "received": time.time()}
|
||||
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
|
||||
|
||||
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
|
||||
|
||||
if not os.path.isdir(ratchetdir):
|
||||
os.makedirs(ratchetdir)
|
||||
if not os.path.isdir(ratchetdir): os.makedirs(ratchetdir)
|
||||
|
||||
outpath = f"{ratchetdir}/{hexhash}.out"
|
||||
finalpath = f"{ratchetdir}/{hexhash}"
|
||||
with open(outpath, "wb") as ratchet_file:
|
||||
ratchet_file.write(umsgpack.packb(ratchet_data))
|
||||
os.replace(outpath, finalpath)
|
||||
outpath = f"{ratchetdir}/{hexhash}.out"
|
||||
finalpath = f"{ratchetdir}/{hexhash}"
|
||||
with open(outpath, "wb") as ratchet_file: ratchet_file.write(umsgpack.packb(ratchet_data))
|
||||
os.replace(outpath, finalpath)
|
||||
|
||||
|
||||
threading.Thread(target=persist_job, daemon=True).start()
|
||||
threading.Thread(target=persist_job, daemon=True).start()
|
||||
|
||||
except Exception as e:
|
||||
RNS.log(f"Could not persist ratchet for {RNS.prettyhexrep(destination_hash)} to storage.", RNS.LOG_ERROR)
|
||||
|
|
@ -483,7 +578,43 @@ class Identity:
|
|||
|
||||
@staticmethod
|
||||
def get_ratchet(destination_hash):
|
||||
if not destination_hash in Identity.known_ratchets:
|
||||
if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash]
|
||||
|
||||
store = Identity.known_destinations_store
|
||||
if store != None:
|
||||
ratchet_data = store.fetch_ratchet(destination_hash)
|
||||
if ratchet_data != None:
|
||||
ratchet, received = ratchet_data
|
||||
if time.time() < received+Identity.RATCHET_EXPIRY:
|
||||
Identity.known_ratchets[destination_hash] = ratchet
|
||||
return ratchet
|
||||
|
||||
else: return None
|
||||
|
||||
else:
|
||||
RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
return None
|
||||
|
||||
elif RNS.Reticulum.get_instance().is_connected_to_shared_instance:
|
||||
try:
|
||||
# TODO: Ideally, refactor this to an RPC method instead
|
||||
store = KnownDestinationsStore(RNS.Reticulum.storagepath+"/known_destinations.db", hash_width=RNS.Reticulum.TRUNCATED_HASHLENGTH//8, read_only=True)
|
||||
ratchet_data = store.fetch_ratchet(destination_hash)
|
||||
store.close()
|
||||
|
||||
if ratchet_data != None:
|
||||
ratchet, received = ratchet_data
|
||||
if time.time() < received+Identity.RATCHET_EXPIRY:
|
||||
Identity.known_ratchets[destination_hash] = ratchet
|
||||
return ratchet
|
||||
|
||||
except Exception as e: RNS.log(f"Could not fetch ratchet for {RNS.prettyhexrep(destination_hash)} from shared instance database. The contained exception was: {e}", RNS.LOG_ERROR)
|
||||
|
||||
return None
|
||||
|
||||
else:
|
||||
if not os.path.isfile(RNS.Reticulum.storagepath+"/ratchets/"+RNS.hexrep(destination_hash, delimit=False)): return None
|
||||
|
||||
ratchetdir = RNS.Reticulum.storagepath+"/ratchets"
|
||||
hexhash = RNS.hexrep(destination_hash, delimit=False)
|
||||
ratchet_path = f"{ratchetdir}/{hexhash}"
|
||||
|
|
@ -493,16 +624,14 @@ class Identity:
|
|||
ratchet_data = umsgpack.unpackb(ratchet_file.read())
|
||||
if time.time() < ratchet_data["received"]+Identity.RATCHET_EXPIRY and len(ratchet_data["ratchet"]) == Identity.RATCHETSIZE//8:
|
||||
Identity.known_ratchets[destination_hash] = ratchet_data["ratchet"]
|
||||
else:
|
||||
return None
|
||||
|
||||
else: return None
|
||||
|
||||
except Exception as e:
|
||||
RNS.log(f"An error occurred while loading ratchet data for {RNS.prettyhexrep(destination_hash)} from storage.", RNS.LOG_ERROR)
|
||||
RNS.log(f"The contained exception was: {e}", RNS.LOG_ERROR)
|
||||
return None
|
||||
|
||||
if destination_hash in Identity.known_ratchets:
|
||||
return Identity.known_ratchets[destination_hash]
|
||||
if destination_hash in Identity.known_ratchets: return Identity.known_ratchets[destination_hash]
|
||||
else:
|
||||
RNS.log(f"Could not load ratchet for {RNS.prettyhexrep(destination_hash)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
return None
|
||||
|
|
@ -614,12 +743,35 @@ class Identity:
|
|||
@staticmethod
|
||||
def persist_data(background=False):
|
||||
if not RNS.Transport.owner.is_connected_to_shared_instance:
|
||||
Identity.save_known_destinations(background=background)
|
||||
if Identity.known_destinations_store != None: Identity.flush_known_destinations()
|
||||
else: Identity.save_known_destinations_legacy(background=background)
|
||||
|
||||
@staticmethod
|
||||
def exit_handler():
|
||||
Identity.persist_data()
|
||||
def flush_known_destinations():
|
||||
store = Identity.known_destinations_store
|
||||
if store == None: return False
|
||||
|
||||
with Identity.known_destinations_lock: upserts, deletes, ratchet_upserts, ratchet_deletes = store.take_pending()
|
||||
|
||||
try: return store.apply(upserts, deletes, ratchet_upserts, ratchet_deletes, Identity.known_destinations)
|
||||
except Exception as e:
|
||||
RNS.log(f"Could not flush known destinations to storage. The contained exception was: {e}", RNS.LOG_ERROR)
|
||||
return False
|
||||
|
||||
@staticmethod
|
||||
def _kd_mark_dirty(destination_hash):
|
||||
store = Identity.known_destinations_store
|
||||
if store != None: store.mark_dirty(destination_hash)
|
||||
|
||||
@staticmethod
|
||||
def _kd_mark_deleted(destination_hash):
|
||||
store = Identity.known_destinations_store
|
||||
if store != None:
|
||||
store.mark_deleted(destination_hash)
|
||||
store.mark_ratchet_deleted(destination_hash)
|
||||
|
||||
@staticmethod
|
||||
def exit_handler(): Identity.persist_data()
|
||||
|
||||
@staticmethod
|
||||
def from_bytes(prv_bytes):
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
@ -425,6 +425,7 @@ class Link:
|
|||
self.activated_at = time.time()
|
||||
self.last_proof = self.activated_at
|
||||
RNS.Transport.activate_link(self)
|
||||
RNS.Transport.mark_path_responsive(self.destination.hash)
|
||||
RNS.log(f"Link {self} established with {self.destination}, RTT is {RNS.prettyshorttime(self.rtt)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
|
||||
if self.rtt != None and self.establishment_cost != None and self.rtt > 0 and self.establishment_cost > 0:
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -1350,7 +1350,7 @@ class Reticulum:
|
|||
filepath = self.cachepath + "/" + filename
|
||||
mtime = os.path.getmtime(filepath)
|
||||
age = now - mtime
|
||||
if age > RNS.Transport.DESTINATION_TIMEOUT: os.unlink(filepath)
|
||||
if age > RNS.Transport.PATH_TIMEOUT: os.unlink(filepath)
|
||||
if background: time.sleep(0.001)
|
||||
|
||||
except Exception as e:
|
||||
|
|
@ -1387,6 +1387,7 @@ class Reticulum:
|
|||
if path == "lowest_interface_bitrate": self.rpc_return(conn, self.get_lowest_interface_bitrate())
|
||||
if path == "medium_path_timeout": self.rpc_return(conn, self.get_medium_path_timeout())
|
||||
if path == "link_count": self.rpc_return(conn, self.get_link_count())
|
||||
if path == "link_distribution": self.rpc_return(conn, self.get_link_distribution())
|
||||
if path == "active_link_count": self.rpc_return(conn, self.get_active_link_count())
|
||||
if path == "packet_rssi": self.rpc_return(conn, self.get_packet_rssi(call["packet_hash"]))
|
||||
if path == "packet_snr": self.rpc_return(conn, self.get_packet_snr(call["packet_hash"]))
|
||||
|
|
@ -1684,6 +1685,7 @@ class Reticulum:
|
|||
pql = RNS.Transport.INBOUND_PR_QUEUE_LENGTH
|
||||
ilql = RNS.Transport.INBOUND_IL_QUEUE_LENGTH
|
||||
tql = dql+aql+pql+ilql
|
||||
ppcs = RNS.Transport.path_pool_counts()
|
||||
|
||||
stats = {}
|
||||
stats["interfaces"] = interfaces
|
||||
|
|
@ -1705,6 +1707,12 @@ class Reticulum:
|
|||
stats["ptxf"] = RNS.Transport.pr_freq_tx
|
||||
stats["rxpps"] = RNS.Transport.rx_pps
|
||||
stats["txpps"] = RNS.Transport.tx_pps
|
||||
stats["rxpkts"] = RNS.Transport.rx_packets
|
||||
stats["txpkts"] = RNS.Transport.tx_packets
|
||||
stats["arxpkts"] = RNS.Transport.announce_rxc
|
||||
stats["atxpkts"] = RNS.Transport.announce_txc
|
||||
stats["prxpkts"] = RNS.Transport.pr_rxc
|
||||
stats["ptxpkts"] = RNS.Transport.pr_txc
|
||||
stats["rxqt"] = qsnapshot[0]
|
||||
stats["rxqd"] = qsnapshot[1][0]
|
||||
stats["rxqa"] = qsnapshot[1][1]
|
||||
|
|
@ -1721,6 +1729,12 @@ class Reticulum:
|
|||
stats["pqpressure"] = qsnapshot[1][2]/pql if qsnapshot[1][2] else 0
|
||||
stats["ilqpressure"] = qsnapshot[1][3]/ilql if qsnapshot[1][3] else 0
|
||||
stats["txq"] = None
|
||||
stats["tlinks"] = RNS.Transport.link_count()
|
||||
stats["alinks"] = RNS.Transport.active_link_count()
|
||||
stats["plinks"] = stats["tlinks"]-stats["alinks"]
|
||||
stats["tpcount"] = ppcs[0]
|
||||
stats["unupcount"] = ppcs[1]
|
||||
stats["actpcount"] = stats["tpcount"] - stats["unupcount"]
|
||||
|
||||
if Reticulum.transport_enabled():
|
||||
stats["transport_id"] = RNS.Transport.identity.hash
|
||||
|
|
@ -1753,6 +1767,7 @@ class Reticulum:
|
|||
entry = { "hash": dst_hash, "timestamp": RNS.Transport.path_table[dst_hash][0],
|
||||
"via": RNS.Transport.path_table[dst_hash][1], "hops": path_hops,
|
||||
"expires": RNS.Transport.path_table[dst_hash][3],
|
||||
"used": RNS.Transport.path_table[dst_hash][7],
|
||||
"interface": str(RNS.Transport.path_table[dst_hash][5]) }
|
||||
path_table.append(entry)
|
||||
|
||||
|
|
@ -1767,15 +1782,10 @@ class Reticulum:
|
|||
|
||||
else:
|
||||
rate_table = []
|
||||
for dst_hash in RNS.Transport.announce_rate_table:
|
||||
entry = {
|
||||
"hash": dst_hash,
|
||||
"last": RNS.Transport.announce_rate_table[dst_hash]["last"],
|
||||
"rate_violations": RNS.Transport.announce_rate_table[dst_hash]["rate_violations"],
|
||||
"blocked_until": RNS.Transport.announce_rate_table[dst_hash]["blocked_until"],
|
||||
"timestamps": RNS.Transport.announce_rate_table[dst_hash]["timestamps"],
|
||||
}
|
||||
rate_table.append(entry)
|
||||
with RNS.Transport.announce_rate_table_lock:
|
||||
for dst_hash, entry in RNS.Transport.announce_rate_table.items():
|
||||
entry["hash"] = dst_hash
|
||||
rate_table.append(entry)
|
||||
|
||||
return rate_table
|
||||
|
||||
|
|
@ -1925,6 +1935,16 @@ class Reticulum:
|
|||
else:
|
||||
return RNS.Transport.active_link_count()
|
||||
|
||||
def get_link_distribution(self):
|
||||
if self.is_connected_to_shared_instance:
|
||||
rpc_connection = self.get_rpc_client()
|
||||
rpc_connection.send_bytes(mp.packb({"get": "link_distribution"}))
|
||||
response = mp.unpackb(rpc_connection.recv_bytes())
|
||||
return response
|
||||
|
||||
else:
|
||||
return RNS.Transport.link_distribution()
|
||||
|
||||
def get_packet_rssi(self, packet_hash):
|
||||
if self.is_connected_to_shared_instance:
|
||||
rpc_connection = self.get_rpc_client()
|
||||
|
|
|
|||
580
RNS/Storage.py
Normal file
580
RNS/Storage.py
Normal file
|
|
@ -0,0 +1,580 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
# in the Software without restriction, including without limitation the rights
|
||||
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
||||
# copies of the Software, and to permit persons to whom the Software is
|
||||
# furnished to do so, subject to the following conditions:
|
||||
#
|
||||
# - The Software shall not be used in any kind of system which includes amongst
|
||||
# its functions the ability to purposefully do harm to human beings.
|
||||
#
|
||||
# - The Software shall not be used, directly or indirectly, in the creation of
|
||||
# an artificial intelligence, machine learning or language model training
|
||||
# dataset, including but not limited to any use that contributes to the
|
||||
# training or development of such a model or algorithm.
|
||||
#
|
||||
# - The above copyright notice and this permission notice shall be included in
|
||||
# all copies or substantial portions of the Software.
|
||||
#
|
||||
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
||||
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
||||
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
||||
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
||||
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
||||
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
||||
# SOFTWARE.
|
||||
|
||||
from array import array
|
||||
import hashlib
|
||||
import threading
|
||||
import os
|
||||
import time
|
||||
|
||||
class AnnounceRateTable:
|
||||
# Columnar store for announce rate tracking. Layout:
|
||||
# _index : dict[destination_hash -> slot]
|
||||
# _keys : list[slot] -> destination_hash (for swap-remove bookkeeping)
|
||||
# _last : array('d') last non-violating announce time
|
||||
# _blocked: array('d') blocked_until
|
||||
# _viol : array('i') rate_violations
|
||||
# _ts : array('d') flat ring of max_timestamps doubles per slot
|
||||
# _ts_len : array('B') valid timestamp count per slot
|
||||
# _ts_head: array('B') ring write cursor per slot
|
||||
|
||||
def __init__(self, max_timestamps=16):
|
||||
assert max_timestamps <= 255, "Timestamp ring size must fit in a single byte"
|
||||
self.max_ts = max_timestamps
|
||||
self._index = {}
|
||||
self._keys = []
|
||||
self._last = array('d')
|
||||
self._blocked = array('d')
|
||||
self._viol = array('i')
|
||||
self._ts = array('d')
|
||||
self._ts_len = array('B')
|
||||
self._ts_head = array('B')
|
||||
self._cursor = 0
|
||||
|
||||
def __len__(self): return len(self._keys)
|
||||
def __contains__(self, destination_hash): return destination_hash in self._index
|
||||
|
||||
# Perform the announce-rate check-and-update for one announce.
|
||||
# Caller must hold the announce_rate_table_lock.
|
||||
def check_and_update(self, destination_hash, now, target, grace, penalty):
|
||||
slot = self._index.get(destination_hash)
|
||||
|
||||
if slot is None:
|
||||
slot = len(self._keys)
|
||||
self._index[destination_hash] = slot
|
||||
self._keys.append(destination_hash)
|
||||
self._last.append(now)
|
||||
self._blocked.append(0.0)
|
||||
self._viol.append(0)
|
||||
self._ts.extend([0.0]*self.max_ts)
|
||||
self._ts[slot*self.max_ts] = now
|
||||
self._ts_len.append(1)
|
||||
self._ts_head.append(1 % self.max_ts)
|
||||
return False
|
||||
|
||||
base = slot*self.max_ts
|
||||
head = self._ts_head[slot]
|
||||
self._ts[base+head] = now
|
||||
self._ts_head[slot] = (head+1) % self.max_ts
|
||||
if self._ts_len[slot] < self.max_ts: self._ts_len[slot] += 1
|
||||
|
||||
current_rate = now - self._last[slot]
|
||||
rate_blocked = False
|
||||
|
||||
if now > self._blocked[slot]:
|
||||
if current_rate < target: self._viol[slot] += 1
|
||||
else:
|
||||
v = self._viol[slot] - 1
|
||||
self._viol[slot] = v if v > 0 else 0
|
||||
|
||||
if self._viol[slot] > grace:
|
||||
self._blocked[slot] = self._last[slot] + target + penalty
|
||||
rate_blocked = True
|
||||
else: self._last[slot] = now
|
||||
|
||||
else: rate_blocked = True
|
||||
|
||||
return rate_blocked
|
||||
|
||||
# Caller must hold the table lock
|
||||
def cull(self, now, ttl, max_scan=1024, max_cull=None):
|
||||
if max_cull is None: max_cull = max_scan
|
||||
horizon = now - ttl
|
||||
culled = 0
|
||||
scanned = 0
|
||||
n = len(self._keys)
|
||||
|
||||
while scanned < max_scan and culled < max_cull and n > 0:
|
||||
if self._cursor >= n: self._cursor = 0
|
||||
slot = self._cursor
|
||||
|
||||
if self._last[slot] < horizon and self._blocked[slot] <= now:
|
||||
last_i = n - 1
|
||||
stale_key = self._keys[slot]
|
||||
|
||||
if slot != last_i:
|
||||
moved_key = self._keys[last_i]
|
||||
self._keys[slot] = moved_key
|
||||
self._index[moved_key] = slot
|
||||
self._last[slot] = self._last[last_i]
|
||||
self._blocked[slot] = self._blocked[last_i]
|
||||
self._viol[slot] = self._viol[last_i]
|
||||
self._ts_len[slot] = self._ts_len[last_i]
|
||||
self._ts_head[slot] = self._ts_head[last_i]
|
||||
|
||||
f = last_i*self.max_ts; t = slot*self.max_ts
|
||||
self._ts[t:t+self.max_ts] = self._ts[f:f+self.max_ts]
|
||||
|
||||
self._keys.pop()
|
||||
self._last.pop(); self._blocked.pop(); self._viol.pop()
|
||||
self._ts_len.pop(); self._ts_head.pop()
|
||||
del self._ts[len(self._ts)-self.max_ts:] # drop tail block of removed slot
|
||||
del self._index[stale_key]
|
||||
|
||||
n -= 1
|
||||
culled += 1
|
||||
# Do not advance cursor, re-examine the slot a moved entry landed in
|
||||
|
||||
else:
|
||||
self._cursor = slot + 1
|
||||
scanned += 1
|
||||
|
||||
if self._cursor >= n: self._cursor = 0
|
||||
|
||||
return culled
|
||||
|
||||
# Hold table lock, or accept a possibly inconsistent snapshot
|
||||
def items(self):
|
||||
for slot, key in enumerate(self._keys):
|
||||
n = self._ts_len[slot]
|
||||
head = self._ts_head[slot]
|
||||
base = slot*self.max_ts
|
||||
if n < self.max_ts: ts = list(self._ts[base:base+n])
|
||||
else: ts = list(self._ts[base+head:base+self.max_ts]) + list(self._ts[base:base+head])
|
||||
yield key, { "last": self._last[slot], "rate_violations": self._viol[slot],
|
||||
"blocked_until": self._blocked[slot], "timestamps": ts }
|
||||
|
||||
class PacketHashlist:
|
||||
# Bucketed byte-blob packet filter hashlist, with the primary
|
||||
# design goal of memory coherence and density. All operations
|
||||
# are safe to call without holding a lock.
|
||||
|
||||
def __init__(self, hash_width=32, bucket_count=16384):
|
||||
assert bucket_count > 0 and bucket_count & (bucket_count-1) == 0, "Bucket count must be a power of two"
|
||||
self.width = hash_width
|
||||
self._nb = bucket_count
|
||||
self._mask = bucket_count - 1
|
||||
self._cur = [bytearray() for _ in range(bucket_count)]
|
||||
self._prv = [bytearray() for _ in range(bucket_count)]
|
||||
self._count = 0
|
||||
|
||||
def __len__(self):
|
||||
return self._count
|
||||
|
||||
def __contains__(self, packet_hash):
|
||||
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
|
||||
return packet_hash in self._cur[bucket] or packet_hash in self._prv[bucket]
|
||||
|
||||
def add(self, packet_hash):
|
||||
assert len(packet_hash) == self.width, "Packet hash width mismatch"
|
||||
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
|
||||
if packet_hash not in self._cur[bucket]:
|
||||
self._cur[bucket].extend(packet_hash)
|
||||
self._count += 1
|
||||
|
||||
def discard(self, packet_hash):
|
||||
w = self.width
|
||||
bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask
|
||||
for generation in (self._cur, self._prv):
|
||||
entries = generation[bucket]
|
||||
pos = entries.find(packet_hash)
|
||||
while pos != -1:
|
||||
if pos % w == 0:
|
||||
del entries[pos:pos+w]
|
||||
if generation is self._cur: self._count -= 1
|
||||
pos = entries.find(packet_hash, pos)
|
||||
else:
|
||||
pos = entries.find(packet_hash, pos+1)
|
||||
|
||||
def swap(self):
|
||||
self._prv = self._cur
|
||||
self._cur = [bytearray() for _ in range(self._nb)]
|
||||
self._count = 0
|
||||
|
||||
# Current-generation buckets for persist operations
|
||||
def buckets(self): return tuple(self._cur)
|
||||
|
||||
class KnownDestinationsStore:
|
||||
# Write-behind SQLite storage backend for known destinations.
|
||||
# The actual in-memory table is used directly for all transport
|
||||
# operations; this serves purely as the store, and no read operations
|
||||
# utilize the database directly.
|
||||
|
||||
def __init__(self, db_path, hash_width=16, read_only=False):
|
||||
import sqlite3
|
||||
self.sqlite3 = sqlite3
|
||||
self.db_path = db_path
|
||||
self.hash_width = hash_width
|
||||
self.read_only = read_only
|
||||
self._dirty_upsert = set()
|
||||
self._dirty_delete = set()
|
||||
self._ratchet_upserts = {}
|
||||
self._ratchet_deletes = set()
|
||||
self._flush_lock = threading.Lock()
|
||||
|
||||
if read_only: self._conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False, isolation_level=None)
|
||||
else:
|
||||
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None)
|
||||
self._conn.execute("PRAGMA journal_mode=WAL")
|
||||
self._conn.execute("PRAGMA synchronous=NORMAL")
|
||||
self._conn.execute("PRAGMA journal_size_limit=8388608")
|
||||
self.__create_schema()
|
||||
|
||||
def close(self):
|
||||
if self._conn != None:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
|
||||
def __create_schema(self):
|
||||
self._conn.execute("""CREATE TABLE IF NOT EXISTS known_destinations (
|
||||
destination_hash BLOB PRIMARY KEY,
|
||||
last_heard REAL NOT NULL,
|
||||
packet_hash BLOB NOT NULL,
|
||||
public_key BLOB NOT NULL,
|
||||
app_data BLOB,
|
||||
used REAL NOT NULL DEFAULT 0,
|
||||
stamp BLOB,
|
||||
stamp_value INTEGER,
|
||||
identity_hash BLOB NOT NULL)""")
|
||||
|
||||
self._conn.execute("""CREATE TABLE IF NOT EXISTS ratchets (
|
||||
destination_hash BLOB PRIMARY KEY,
|
||||
ratchet BLOB NOT NULL,
|
||||
received REAL NOT NULL)""")
|
||||
|
||||
self._conn.execute("PRAGMA user_version = 1")
|
||||
self._conn.commit()
|
||||
|
||||
def __entry_row(self, destination_hash, entry):
|
||||
stamp = entry[5] if len(entry) > 5 else None
|
||||
stamp_value = entry[6] if len(entry) > 6 else 0
|
||||
identity_hash = hashlib.sha256(entry[2]).digest()[:self.hash_width]
|
||||
return (destination_hash, entry[0], entry[1], entry[2], entry[3], entry[4], stamp, stamp_value, identity_hash)
|
||||
|
||||
# Caller must hold Identity.known_destinations_lock
|
||||
def mark_dirty(self, destination_hash):
|
||||
if self.read_only: return
|
||||
self._dirty_delete.discard(destination_hash)
|
||||
self._dirty_upsert.add(destination_hash)
|
||||
|
||||
# Caller must hold Identity.known_destinations_lock
|
||||
def mark_deleted(self, destination_hash):
|
||||
if self.read_only: return
|
||||
self._dirty_upsert.discard(destination_hash)
|
||||
self._dirty_delete.add(destination_hash)
|
||||
|
||||
# Caller must hold Identity.known_destinations_lock
|
||||
def take_pending(self):
|
||||
if self.read_only: return set(), set(), {}, set()
|
||||
upserts = self._dirty_upsert
|
||||
deletes = self._dirty_delete
|
||||
ratchet_upserts = self._ratchet_upserts
|
||||
ratchet_deletes = self._ratchet_deletes
|
||||
self._dirty_upsert = set()
|
||||
self._dirty_delete = set()
|
||||
self._ratchet_upserts = {}
|
||||
self._ratchet_deletes = set()
|
||||
return upserts, deletes, ratchet_upserts, ratchet_deletes
|
||||
|
||||
# May be called without any locks held
|
||||
def apply(self, upserts, deletes, ratchet_upserts, ratchet_deletes, known_destinations):
|
||||
if self.read_only: return 0
|
||||
if len(upserts) == 0 and len(deletes) == 0 and len(ratchet_upserts) == 0 and len(ratchet_deletes) == 0: return 0
|
||||
|
||||
with self._flush_lock:
|
||||
rows = []
|
||||
removables = []
|
||||
ratchet_rows = []
|
||||
ratchet_removables = []
|
||||
|
||||
for destination_hash in upserts:
|
||||
entry = known_destinations.get(destination_hash)
|
||||
if entry != None: rows.append(self.__entry_row(destination_hash, entry))
|
||||
|
||||
for destination_hash in deletes:
|
||||
if destination_hash not in known_destinations: removables.append((destination_hash,))
|
||||
|
||||
for destination_hash, ratchet_entry in ratchet_upserts.items():
|
||||
# Ratchets of purged destinations must be removed as well
|
||||
if destination_hash in known_destinations: ratchet_rows.append((destination_hash, *ratchet_entry))
|
||||
else: ratchet_removables.append((destination_hash,))
|
||||
|
||||
for destination_hash in ratchet_deletes:
|
||||
if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,))
|
||||
|
||||
try:
|
||||
self._conn.execute("BEGIN")
|
||||
for i in range(0, len(rows), 4096): self._conn.executemany("INSERT INTO known_destinations VALUES (?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, identity_hash=excluded.identity_hash", rows[i:i+4096])
|
||||
for i in range(0, len(removables), 4096): self._conn.executemany("DELETE FROM known_destinations WHERE destination_hash=?", removables[i:i+4096])
|
||||
for i in range(0, len(ratchet_rows), 4096): self._conn.executemany("INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received", ratchet_rows[i:i+4096])
|
||||
for i in range(0, len(ratchet_removables), 4096): self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", ratchet_removables[i:i+4096])
|
||||
self._conn.commit()
|
||||
|
||||
except Exception:
|
||||
# Re-mark everything as dirty, so the changes are retried
|
||||
# on the next flush, then propagate.
|
||||
self._dirty_upsert |= {row[0] for row in rows}
|
||||
self._dirty_delete |= {row[0] for row in removables}
|
||||
self._ratchet_upserts |= {row[0]: row[1:] for row in ratchet_rows}
|
||||
self._ratchet_deletes |= {row[0] for row in ratchet_removables}
|
||||
self._conn.rollback()
|
||||
raise
|
||||
|
||||
return len(rows) + len(removables) + len(ratchet_rows) + len(ratchet_removables)
|
||||
|
||||
def load_all(self):
|
||||
cursor = self._conn.execute("SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value FROM known_destinations")
|
||||
while True:
|
||||
rows = cursor.fetchmany(4096)
|
||||
if not rows: break
|
||||
for row in rows: yield row[0], [row[1], row[2], row[3], row[4], row[5], row[6], row[7]]
|
||||
|
||||
# Caller must hold Identity.known_destinations_lock.
|
||||
def remember_ratchet(self, destination_hash, ratchet, now):
|
||||
if self.read_only: return False
|
||||
|
||||
pending = self._ratchet_upserts.get(destination_hash)
|
||||
if pending != None:
|
||||
if pending[0] == ratchet:
|
||||
self._ratchet_deletes.discard(destination_hash)
|
||||
return False
|
||||
|
||||
else:
|
||||
row = self._conn.execute("SELECT ratchet FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
|
||||
if row != None and row[0] == ratchet:
|
||||
# Discard any pending delete mark queued
|
||||
# before this ratchet arrived
|
||||
self._ratchet_deletes.discard(destination_hash)
|
||||
return False
|
||||
|
||||
self._ratchet_upserts[destination_hash] = (ratchet, now)
|
||||
self._ratchet_deletes.discard(destination_hash)
|
||||
return True
|
||||
|
||||
def fetch_ratchet(self, destination_hash):
|
||||
pending = self._ratchet_upserts.get(destination_hash)
|
||||
if pending != None: return pending[0], pending[1]
|
||||
if destination_hash in self._ratchet_deletes: return None
|
||||
|
||||
row = self._conn.execute("SELECT ratchet, received FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone()
|
||||
if row != None: return row[0], row[1]
|
||||
|
||||
return None
|
||||
|
||||
# Caller must hold Identity.known_destinations_lock
|
||||
def mark_ratchet_deleted(self, destination_hash):
|
||||
if self.read_only: return
|
||||
self._ratchet_upserts.pop(destination_hash, None)
|
||||
self._ratchet_deletes.add(destination_hash)
|
||||
|
||||
def expire_ratchets(self, cutoff):
|
||||
if self.read_only: return []
|
||||
|
||||
with self._flush_lock:
|
||||
# Drop expired entries in pending as well
|
||||
expired = set(row[0] for row in self._conn.execute("SELECT destination_hash FROM ratchets WHERE received<?", (cutoff,)))
|
||||
for destination_hash, entry in list(self._ratchet_upserts.items()):
|
||||
if entry[1] < cutoff:
|
||||
expired.add(destination_hash)
|
||||
self._ratchet_upserts.pop(destination_hash, None)
|
||||
|
||||
expired.difference_update(self._ratchet_deletes)
|
||||
if len(expired) > 0:
|
||||
self._conn.execute("BEGIN")
|
||||
for i in range(0, len(expired), 4096):
|
||||
self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", [(h,) for h in list(expired)[i:i+4096]])
|
||||
self._conn.commit()
|
||||
|
||||
return list(expired)
|
||||
|
||||
def load_ratchets(self):
|
||||
cursor = self._conn.execute("SELECT destination_hash, ratchet, received FROM ratchets")
|
||||
while True:
|
||||
rows = cursor.fetchmany(4096)
|
||||
if not rows: break
|
||||
for row in rows: yield row[0], row[1], row[2]
|
||||
|
||||
def import_legacy_ratchets(self, dir_path, expiry_cutoff, known_destinations, ratchet_width=32):
|
||||
import RNS
|
||||
from .vendor import umsgpack
|
||||
imported = 0
|
||||
for filename in os.listdir(dir_path):
|
||||
file_path = os.path.join(dir_path, filename)
|
||||
try:
|
||||
if not os.path.isfile(file_path): continue
|
||||
with open(file_path, "rb") as rf:
|
||||
ratchet_data = umsgpack.unpackb(rf.read())
|
||||
|
||||
destination_hash = bytes.fromhex(filename)
|
||||
ratchet = ratchet_data["ratchet"]; received = ratchet_data["received"]
|
||||
if len(ratchet) != ratchet_width: raise ValueError("Invalid ratchet width")
|
||||
if received < expiry_cutoff: raise ValueError("Ratchet expired")
|
||||
if destination_hash not in known_destinations: raise ValueError("Destination unknown")
|
||||
|
||||
self._ratchet_upserts[destination_hash] = (ratchet, received)
|
||||
imported += 1
|
||||
os.unlink(file_path)
|
||||
|
||||
except Exception as e:
|
||||
RNS.log(f"Could not import ratchet for {filename}: {e}", RNS.LOG_WARNING)
|
||||
try: os.unlink(file_path)
|
||||
except: pass
|
||||
|
||||
pending = self._ratchet_upserts
|
||||
self._ratchet_upserts = {}
|
||||
self.apply(set(), set(), pending, set(), known_destinations)
|
||||
|
||||
return imported
|
||||
|
||||
def import_legacy(self, path):
|
||||
from .vendor import umsgpack
|
||||
with open(path, "rb") as file: legacy = umsgpack.load(file)
|
||||
|
||||
rows = []
|
||||
now = time.time()
|
||||
for destination_hash, entry in legacy.items():
|
||||
if len(entry) < 5: continue
|
||||
entry = list(entry)
|
||||
while len(entry) < 7: entry.append(None if len(entry) == 5 else 0)
|
||||
if entry[0] == None or entry[0] <= 0: entry[0] = now
|
||||
rows.append(self.__entry_row(destination_hash, entry))
|
||||
|
||||
self._conn.execute("BEGIN")
|
||||
for i in range(0, len(rows), 4096): self._conn.executemany("INSERT OR REPLACE INTO known_destinations VALUES (?,?,?,?,?,?,?,?,?)", rows[i:i+4096])
|
||||
self._conn.commit()
|
||||
|
||||
return len(rows)
|
||||
|
||||
class AnnounceCache:
|
||||
|
||||
def __init__(self, db_path):
|
||||
import sqlite3
|
||||
self.db_path = db_path
|
||||
self._pending = {}
|
||||
self._flush_lock = threading.Lock()
|
||||
self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None)
|
||||
|
||||
self._conn.execute("PRAGMA journal_mode=WAL")
|
||||
self._conn.execute("PRAGMA synchronous=NORMAL")
|
||||
self._conn.execute("PRAGMA journal_size_limit=8388608")
|
||||
self._conn.execute("PRAGMA auto_vacuum=INCREMENTAL")
|
||||
self.__create_schema()
|
||||
|
||||
def __create_schema(self):
|
||||
self._conn.execute("""CREATE TABLE IF NOT EXISTS cached_announces (
|
||||
packet_hash BLOB PRIMARY KEY,
|
||||
received REAL NOT NULL,
|
||||
raw BLOB NOT NULL,
|
||||
interface_reference TEXT)""")
|
||||
|
||||
self._conn.execute("PRAGMA user_version = 1")
|
||||
|
||||
def store(self, packet_hash, raw, interface_reference):
|
||||
self._pending[packet_hash] = (time.time(), raw, interface_reference)
|
||||
|
||||
def pending_count(self):
|
||||
return len(self._pending)
|
||||
|
||||
def fetch(self, packet_hash):
|
||||
pending = self._pending.get(packet_hash)
|
||||
if pending != None: return pending[1], pending[2]
|
||||
|
||||
row = self._conn.execute("SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=?", (packet_hash,)).fetchone()
|
||||
if row != None: return row[0], row[1]
|
||||
|
||||
return None
|
||||
|
||||
def flush(self):
|
||||
with self._flush_lock: return self.__apply_pending()
|
||||
|
||||
def __apply_pending(self):
|
||||
pending = self._pending
|
||||
self._pending = {}
|
||||
if len(pending) == 0: return 0
|
||||
|
||||
try:
|
||||
self._conn.execute("BEGIN")
|
||||
items = list(pending.items())
|
||||
for i in range(0, len(items), 4096):
|
||||
self._conn.executemany("INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference",
|
||||
[(packet_hash, *entry) for packet_hash, entry in items[i:i+4096]])
|
||||
self._conn.commit()
|
||||
|
||||
except Exception:
|
||||
# Re-mark everything as pending, so the changes
|
||||
# are retried on the next flush, then propagate.
|
||||
self._pending.update(pending)
|
||||
self._conn.rollback()
|
||||
raise
|
||||
|
||||
return len(pending)
|
||||
|
||||
def clean(self, active_hashes):
|
||||
with self._flush_lock:
|
||||
self.__apply_pending()
|
||||
|
||||
try:
|
||||
self._conn.execute("BEGIN")
|
||||
self._conn.execute("DROP TABLE IF EXISTS temp.active_announces")
|
||||
self._conn.execute("CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY)")
|
||||
active_list = list(active_hashes)
|
||||
for i in range(0, len(active_list), 4096):
|
||||
self._conn.executemany("INSERT OR IGNORE INTO active_announces VALUES (?)", [(h,) for h in active_list[i:i+4096]])
|
||||
|
||||
changes_before = self._conn.total_changes
|
||||
self._conn.execute("DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces)")
|
||||
removed = self._conn.total_changes - changes_before
|
||||
self._conn.commit()
|
||||
|
||||
except Exception:
|
||||
self._conn.rollback()
|
||||
raise
|
||||
|
||||
if removed > 0: self._conn.execute("PRAGMA incremental_vacuum")
|
||||
|
||||
return removed
|
||||
|
||||
def import_legacy(self, dir_path):
|
||||
from .vendor import umsgpack
|
||||
imported = 0
|
||||
for filename in os.listdir(dir_path):
|
||||
file_path = os.path.join(dir_path, filename)
|
||||
try:
|
||||
if not os.path.isfile(file_path): continue
|
||||
with open(file_path, "rb") as file:
|
||||
cached_data = umsgpack.unpackb(file.read())
|
||||
|
||||
packet_hash = bytes.fromhex(filename)
|
||||
interface_reference = cached_data[1] if len(cached_data) > 1 else None
|
||||
self.store(packet_hash, cached_data[0], interface_reference)
|
||||
imported += 1
|
||||
os.unlink(file_path)
|
||||
|
||||
except Exception:
|
||||
try: os.unlink(file_path)
|
||||
except Exception: pass
|
||||
|
||||
self.flush()
|
||||
return imported
|
||||
|
||||
def close(self):
|
||||
if self._conn != None:
|
||||
self._conn.close()
|
||||
self._conn = None
|
||||
360
RNS/Transport.py
360
RNS/Transport.py
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
@ -42,6 +42,7 @@ from queue import Full, Empty
|
|||
from threading import Lock, Condition
|
||||
from .vendor import umsgpack as umsgpack
|
||||
from RNS.Interfaces.BackboneInterface import BackboneInterface
|
||||
from RNS.Storage import AnnounceRateTable, PacketHashlist, AnnounceCache
|
||||
|
||||
class InboundQueues:
|
||||
__slots__ = ("_cond", "_queues", "_sizes", "_dropped", "_hw_mark")
|
||||
|
|
@ -123,7 +124,6 @@ class Transport:
|
|||
PATHFINDER_R = 1 # Retransmit retries
|
||||
PATHFINDER_G = 5 # Retry grace period
|
||||
PATHFINDER_RW = 0.5 # Random window for announce rebroadcast
|
||||
PATHFINDER_E = 60*60*24*7 # Path expiration of one week
|
||||
AP_PATH_TIME = 60*60*24 # Path expiration of one day for Access Point paths
|
||||
ROAMING_PATH_TIME = 60*60*6 # Path expiration of 6 hours for Roaming paths
|
||||
|
||||
|
|
@ -152,10 +152,14 @@ class Transport:
|
|||
LINK_TIMEOUT = RNS.Link.STALE_TIME * 1.25
|
||||
ALLOW_LINK_PATH_REBALANCE = True
|
||||
REVERSE_TIMEOUT = 8*60 # Reverse table entries are removed after 8 minutes
|
||||
DESTINATION_TIMEOUT = 60*60*24*7 # Destination table entries are removed if unused for one week
|
||||
UNUSED_DESTINATION_LINGER = 6*60 # Linger time for pathless and never used destinations
|
||||
PATH_TIMEOUT = 60*60*24*7 # Path table entry retainment after last cryptographically verifiable liveness event
|
||||
UNUSED_PATH_LINGER = 60*60*1.75 # Linger time for never used paths
|
||||
UNUSED_PATH_LINGER_LEAF = 60*5 # Linger time for never used paths on non-transport instances
|
||||
UNUSED_DESTINATION_LINGER = 60*6 # Linger time for pathless and never used destinations
|
||||
PATH_POOL_UNUSED_TARGET = 2048 # Max unused path table pool target, unused destinations are culled to reach it
|
||||
TUNNEL_TIMEOUT = 60*60*8 # Tunnel table entries are removed if unused for eight hours
|
||||
TUNNEL_PATH_TIMEOUT = 60*60*8 # Tunnel path table entries are removed if unused for eight hours
|
||||
RATE_TABLE_RETAIN = 60*60 # Default announce rate table retainment, increased dynamically
|
||||
MAX_RECEIPTS = 1024 # Maximum number of receipts to keep track of
|
||||
MAX_RATE_TIMESTAMPS = 16 # Maximum number of announce timestamps to keep per destination
|
||||
PERSIST_RANDOM_BLOBS = 32 # Maximum number of random blobs per destination to persist to disk
|
||||
|
|
@ -169,9 +173,8 @@ class Transport:
|
|||
active_links = [] # Links that are active
|
||||
pending_links_map = {} # Link ID hash map of pending links
|
||||
active_links_map = {} # Link ID hash map of active links
|
||||
packet_hashlist = set() # A list of packet hashes for duplicate detection
|
||||
packet_hashlist_prev = set()
|
||||
receipts = [] # Receipts of all outgoing packets for proof processing
|
||||
packet_hashlist = PacketHashlist(hash_width=RNS.Identity.HASHLENGTH//8)
|
||||
|
||||
# Notes on memory usage: 1 megabyte of memory can store approximately
|
||||
# 55.100 path table entries or approximately 22.300 link table entries.
|
||||
|
|
@ -183,11 +186,11 @@ class Transport:
|
|||
held_announces = {} # A table containing temporarily held announce-table entries
|
||||
announce_handlers = [] # A table storing externally registered announce handlers
|
||||
tunnels = {} # A table storing tunnels to other transport instances
|
||||
announce_rate_table = {} # A table for keeping track of announce rates
|
||||
path_requests = {} # A table for storing path request timestamps
|
||||
inflight_path_requests = {} # A table for tracking in-flight path request
|
||||
path_states = {} # A table for keeping track of path states
|
||||
blackholed_identities = {} # A table for keeping track of blackholed identities
|
||||
announce_rate_table = AnnounceRateTable(max_timestamps=MAX_RATE_TIMESTAMPS)
|
||||
|
||||
discovery_path_requests = {} # A table for keeping track of path requests on behalf of other nodes
|
||||
discovery_pr_tags = set() # A table for keeping track of tagged path requests
|
||||
|
|
@ -260,20 +263,36 @@ class Transport:
|
|||
known_destinations_interval = 5*60
|
||||
tables_last_culled = 0.0
|
||||
tables_cull_interval = 5.0
|
||||
rate_table_last_culled = 0.0
|
||||
rate_table_cull_interval = 5.0
|
||||
rate_table_cull_chunk = 1024
|
||||
pool_size_last_checked = 0.0
|
||||
pool_size_check_interval = 5*60
|
||||
kd_flush_interval = 10.0
|
||||
announce_cache = None
|
||||
ac_flush_interval = 15.0
|
||||
ac_last_flush = 0.0
|
||||
announce_cache_max_pending = 10000
|
||||
kd_last_flush = 0.0
|
||||
interface_last_jobs = 0.0
|
||||
interface_jobs_interval = 5.0
|
||||
last_mgmt_announce = 0
|
||||
mgmt_announce_interval = 2*60*60
|
||||
blackhole_last_checked = 0
|
||||
blackhole_check_interval = 60
|
||||
unused_path_linger = None
|
||||
interface_announcer = None
|
||||
discovery_handler = None
|
||||
blackhole_updater = None
|
||||
|
||||
traffic_rxb = 0
|
||||
traffic_txb = 0
|
||||
announce_rxc = 0
|
||||
announce_txc = 0
|
||||
announce_rxb = 0
|
||||
announce_txb = 0
|
||||
pr_rxc = 0
|
||||
pr_txc = 0
|
||||
pr_rxb = 0
|
||||
pr_txb = 0
|
||||
speed_rx = 0
|
||||
|
|
@ -299,6 +318,7 @@ class Transport:
|
|||
|
||||
_identity = None
|
||||
_should_run = True
|
||||
_unused_path_count_estimate = 0
|
||||
|
||||
@staticmethod
|
||||
def internal_identity(): return Transport._identity
|
||||
|
|
@ -336,6 +356,12 @@ class Transport:
|
|||
|
||||
if RNS.Reticulum.local_hops_delta(): Transport.local_hops_delta = (ord(os.urandom(1))%6)+2
|
||||
|
||||
if not RNS.Reticulum.transport_enabled() or Transport.owner.is_connected_to_shared_instance:
|
||||
Transport.unused_path_linger = Transport.UNUSED_PATH_LINGER_LEAF
|
||||
else:
|
||||
Transport.unused_path_linger = Transport.UNUSED_PATH_LINGER
|
||||
RNS.log(f"Unused path linger time is {RNS.prettytime(Transport.unused_path_linger)}", RNS.LOG_DEBUG)
|
||||
|
||||
packet_hashlist_path = RNS.Reticulum.storagepath+"/packet_hashlist.raw"
|
||||
if RNS.Reticulum.transport_enabled() and not Transport.owner.is_connected_to_shared_instance:
|
||||
if os.path.isfile(packet_hashlist_path):
|
||||
|
|
@ -393,6 +419,9 @@ class Transport:
|
|||
# Defer cleaning known destinations
|
||||
Transport.destinations_last_cleaned = time.time()
|
||||
|
||||
# Defer path table unused pool cull
|
||||
Transport.pool_size_last_checked = time.time()
|
||||
|
||||
# Defer sending management announces for 15 seconds
|
||||
Transport.last_mgmt_announce = time.time() - Transport.mgmt_announce_interval + 15
|
||||
|
||||
|
|
@ -401,6 +430,8 @@ class Transport:
|
|||
threading.Thread(target=Transport.count_traffic_loop, daemon=True).start()
|
||||
|
||||
# Load transport-related data
|
||||
if not Transport.owner.is_connected_to_shared_instance: Transport.init_announce_cache()
|
||||
|
||||
if RNS.Reticulum.transport_enabled():
|
||||
path_table_path = RNS.Reticulum.storagepath+"/destination_table"
|
||||
tunnel_table_path = RNS.Reticulum.storagepath+"/tunnels"
|
||||
|
|
@ -412,6 +443,7 @@ class Transport:
|
|||
serialised_destinations = umsgpack.unpackb(file.read())
|
||||
file.close()
|
||||
|
||||
now = time.time()
|
||||
for serialised_entry in serialised_destinations:
|
||||
destination_hash = serialised_entry[0]
|
||||
|
||||
|
|
@ -423,6 +455,7 @@ class Transport:
|
|||
random_blobs = serialised_entry[5]
|
||||
receiving_interface = Transport.find_interface_from_hash(serialised_entry[6])
|
||||
announce_packet = Transport.get_cached_packet(serialised_entry[7], packet_type="announce")
|
||||
used = serialised_entry[8] if len(serialised_entry) > 8 else now
|
||||
blackholed = False
|
||||
|
||||
if len(Transport.blackholed_identities) > 0:
|
||||
|
|
@ -438,7 +471,10 @@ class Transport:
|
|||
# increased hop-count.
|
||||
announce_packet.hops += 1
|
||||
with Transport.path_table_lock:
|
||||
Transport.path_table[destination_hash] = [timestamp, received_from, hops, expires, random_blobs, receiving_interface, announce_packet.packet_hash]
|
||||
Transport.path_table[destination_hash] = [timestamp, received_from, hops, expires,
|
||||
random_blobs, receiving_interface,
|
||||
announce_packet.packet_hash, used]
|
||||
|
||||
RNS.log("Loaded path table entry for "+RNS.prettyhexrep(destination_hash)+" from storage", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
else:
|
||||
RNS.log("Could not reconstruct path table entry from storage for "+RNS.prettyhexrep(destination_hash), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
|
|
@ -466,6 +502,7 @@ class Transport:
|
|||
serialised_tunnels = umsgpack.unpackb(file.read())
|
||||
file.close()
|
||||
|
||||
now = time.time()
|
||||
for serialised_tunnel in serialised_tunnels:
|
||||
tunnel_id = serialised_tunnel[IDX_TT_TUNNEL_ID]
|
||||
interface_hash = serialised_tunnel[IDX_TT_IF]
|
||||
|
|
@ -482,6 +519,7 @@ class Transport:
|
|||
random_blobs = list(set(serialised_entry[5]))
|
||||
receiving_interface = Transport.find_interface_from_hash(serialised_entry[6])
|
||||
announce_packet = Transport.get_cached_packet(serialised_entry[7], packet_type="announce")
|
||||
used = serialised_entry[8] if len(serialised_entry) > 8 else now
|
||||
|
||||
if announce_packet != None:
|
||||
if not announce_packet.unpack(): continue
|
||||
|
|
@ -491,7 +529,7 @@ class Transport:
|
|||
# increased hop-count.
|
||||
announce_packet.hops += 1
|
||||
|
||||
tunnel_path = [timestamp, received_from, hops, path_expires, random_blobs, receiving_interface, announce_packet.packet_hash]
|
||||
tunnel_path = [timestamp, received_from, hops, path_expires, random_blobs, receiving_interface, announce_packet.packet_hash, used]
|
||||
tunnel_paths[destination_hash] = tunnel_path
|
||||
|
||||
if len(tunnel_paths) > 0:
|
||||
|
|
@ -543,6 +581,9 @@ class Transport:
|
|||
with Transport.interfaces_lock:
|
||||
if not interface in Transport.interfaces:
|
||||
Transport.interfaces.append(interface)
|
||||
if interface.announce_rate_target and interface.announce_rate_target > Transport.RATE_TABLE_RETAIN:
|
||||
Transport.RATE_TABLE_RETAIN = interface.announce_rate_target
|
||||
RNS.log(f"Increased announce rate table retainment time to {RNS.prettytime(Transport.RATE_TABLE_RETAIN)}", RNS.LOG_DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def remove_interface(interface):
|
||||
|
|
@ -828,10 +869,9 @@ class Transport:
|
|||
Transport.announces_last_checked = time.time()
|
||||
|
||||
|
||||
# Cull the packet hashlist if it has reached its max size
|
||||
# Swap the packet hashlist generations if it has reached its max size
|
||||
if len(Transport.packet_hashlist) > Transport.hashlist_maxsize//2:
|
||||
Transport.packet_hashlist_prev = Transport.packet_hashlist
|
||||
Transport.packet_hashlist = set()
|
||||
Transport.packet_hashlist.swap()
|
||||
|
||||
# Cull invalidated path requests
|
||||
if time.time() > Transport.pending_prs_last_checked+Transport.pending_prs_check_interval:
|
||||
|
|
@ -852,6 +892,24 @@ class Transport:
|
|||
Transport.discovery_pr_tags_prev = Transport.discovery_pr_tags
|
||||
Transport.discovery_pr_tags = set()
|
||||
|
||||
# Incrementally cull stale announce rate table entries
|
||||
if time.time() > Transport.rate_table_last_culled+Transport.rate_table_cull_interval:
|
||||
with Transport.announce_rate_table_lock:
|
||||
Transport.announce_rate_table.cull(time.time(), Transport.RATE_TABLE_RETAIN,
|
||||
Transport.rate_table_cull_chunk, Transport.rate_table_cull_chunk)
|
||||
|
||||
Transport.rate_table_last_culled = time.time()
|
||||
|
||||
# Flush pending known destinations to storage
|
||||
if RNS.Identity.known_destinations_store != None and time.time() > Transport.kd_last_flush+Transport.kd_flush_interval:
|
||||
Transport.kd_last_flush = time.time()
|
||||
RNS.Identity.flush_known_destinations()
|
||||
|
||||
# Flush pending cached announces to storage
|
||||
if Transport.announce_cache != None and (time.time() > Transport.ac_last_flush+Transport.ac_flush_interval or Transport.announce_cache.pending_count() > Transport.announce_cache_max_pending):
|
||||
Transport.ac_last_flush = time.time()
|
||||
Transport.announce_cache.flush()
|
||||
|
||||
if time.time() > Transport.tables_last_culled + Transport.tables_cull_interval:
|
||||
# Remove unneeded path state entries
|
||||
stale_path_states = []
|
||||
|
|
@ -957,25 +1015,42 @@ class Transport:
|
|||
# Cull the path table
|
||||
stale_paths = []
|
||||
with Transport.path_table_lock:
|
||||
now = time.time()
|
||||
check_unused = now > Transport.pool_size_last_checked+Transport.pool_size_check_interval
|
||||
unused_paths = []
|
||||
|
||||
for destination_hash in Transport.path_table:
|
||||
destination_entry = Transport.path_table[destination_hash]
|
||||
attached_interface = destination_entry[IDX_PT_RVCD_IF]
|
||||
path_entry = Transport.path_table[destination_hash]
|
||||
attached_interface = path_entry[IDX_PT_RVCD_IF]
|
||||
path_timestamp = path_entry[IDX_PT_TIMESTAMP]
|
||||
path_expiry = path_entry[IDX_PT_EXPIRES]
|
||||
path_used = path_entry[IDX_PT_USED]
|
||||
|
||||
if attached_interface != None and hasattr(attached_interface, "mode") and attached_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT:
|
||||
destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.AP_PATH_TIME
|
||||
elif attached_interface != None and hasattr(attached_interface, "mode") and attached_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING:
|
||||
destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.ROAMING_PATH_TIME
|
||||
else:
|
||||
destination_expiry = destination_entry[IDX_PT_TIMESTAMP] + Transport.DESTINATION_TIMEOUT
|
||||
if check_unused and not path_used: unused_paths.append((path_expiry, destination_hash))
|
||||
|
||||
if time.time() > destination_expiry:
|
||||
if now > path_expiry:
|
||||
stale_paths.append(destination_hash)
|
||||
should_collect = True
|
||||
RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} timed out and was removed", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
elif not attached_interface in Transport.interfaces:
|
||||
stale_paths.append(destination_hash)
|
||||
should_collect = True
|
||||
RNS.log("Path to "+RNS.prettyhexrep(destination_hash)+" was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
RNS.log(f"Path to {RNS.prettyhexrep(destination_hash)} was removed since the attached interface no longer exists", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
elif path_used == 0 and now > path_timestamp + Transport.unused_path_linger:
|
||||
stale_paths.append(destination_hash)
|
||||
should_collect = True
|
||||
RNS.log(f"Removing never used path to {RNS.prettyhexrep(destination_hash)}", RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
|
||||
if len(unused_paths) > Transport.PATH_POOL_UNUSED_TARGET:
|
||||
cull_count = len(unused_paths) - Transport.PATH_POOL_UNUSED_TARGET
|
||||
RNS.log(f"Culling {cull_count} never used path table entries to reach pool size target {Transport.PATH_POOL_UNUSED_TARGET}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
unused_paths.sort()
|
||||
for cull_entry in unused_paths[-cull_count:]:
|
||||
stale_paths.append(cull_entry[1])
|
||||
|
||||
if check_unused:
|
||||
Transport._unused_path_count_estimate = len(unused_paths)
|
||||
Transport.pool_size_last_checked = time.time()
|
||||
|
||||
# Cull the pending path requests table
|
||||
stale_path_requests = []
|
||||
|
|
@ -1387,6 +1462,7 @@ class Transport:
|
|||
return False
|
||||
|
||||
outbound_interface = path_entry[IDX_PT_RVCD_IF]
|
||||
path_entry[IDX_PT_USED] = outbound_time
|
||||
|
||||
# If there's more than one hop to the destination, and we know
|
||||
# a path, we insert the packet into transport by adding the next
|
||||
|
|
@ -1403,7 +1479,6 @@ class Transport:
|
|||
new_raw += packet.raw[2:]
|
||||
packet_sent(packet)
|
||||
Transport.transmit(outbound_interface, new_raw)
|
||||
path_entry[IDX_PT_TIMESTAMP] = time.time()
|
||||
sent = True
|
||||
|
||||
# In the special case where we are connected to a local shared
|
||||
|
|
@ -1423,7 +1498,6 @@ class Transport:
|
|||
new_raw += packet.raw[2:]
|
||||
packet_sent(packet)
|
||||
Transport.transmit(outbound_interface, new_raw)
|
||||
path_entry[IDX_PT_TIMESTAMP] = time.time()
|
||||
sent = True
|
||||
|
||||
# If none of the above applies, we know the destination is
|
||||
|
|
@ -1597,8 +1671,8 @@ class Transport:
|
|||
else: raw = Transport.mangle_hops(packet.raw, Transport.local_hops_delta, transport_insert=packet.header_type==RNS.Packet.HEADER_1)
|
||||
|
||||
Transport.transmit(interface, raw)
|
||||
if packet.packet_type == RNS.Packet.ANNOUNCE: interface.sent_announce(size=len(raw))
|
||||
if packet.destination.type == RNS.Destination.PLAIN and packet.is_outbound_pr: interface.sent_path_request(size=len(raw))
|
||||
if packet.packet_type == RNS.Packet.ANNOUNCE: interface.sent_announce(size=len(raw)); Transport.announce_txc += 1
|
||||
if packet.destination.type == RNS.Destination.PLAIN and packet.is_outbound_pr: interface.sent_path_request(size=len(raw)); Transport.pr_txc += 1
|
||||
packet_sent(packet)
|
||||
sent = True
|
||||
|
||||
|
|
@ -1665,7 +1739,7 @@ class Transport:
|
|||
if packet.receiving_interface: packet.receiving_interface.protocol_violation("Announce packet with GROUP type")
|
||||
return False
|
||||
|
||||
if not packet.packet_hash in Transport.packet_hashlist and not packet.packet_hash in Transport.packet_hashlist_prev: return True
|
||||
if not packet.packet_hash in Transport.packet_hashlist: return True
|
||||
else:
|
||||
if packet.packet_type == RNS.Packet.ANNOUNCE:
|
||||
if packet.destination_type == RNS.Destination.SINGLE:
|
||||
|
|
@ -1809,8 +1883,9 @@ class Transport:
|
|||
return packet.receiving_interface.protocol_violation(f"Invalid announce signature for {RNS.prettyhexrep(packet.destination_hash)}") if packet.receiving_interface else None
|
||||
|
||||
elif packet.receiving_interface != None: packet.receiving_interface.received_announce(size=len(packet.raw))
|
||||
announced_destination_known = packet.destination_hash in Transport.path_table
|
||||
Transport.announce_rxc += 1
|
||||
|
||||
announced_destination_known = packet.destination_hash in Transport.path_table
|
||||
if not announced_destination_known:
|
||||
# This is an unknown destination, and we'll apply
|
||||
# potential ingress limiting. Already known
|
||||
|
|
@ -1856,6 +1931,7 @@ class Transport:
|
|||
return
|
||||
|
||||
if packet.receiving_interface: packet.receiving_interface.received_path_request(size=len(raw))
|
||||
Transport.pr_rxc += 1
|
||||
if interface.should_ingress_limit_pr():
|
||||
traffic_class = Transport.TC_INGRESS_LIMITED
|
||||
|
||||
|
|
@ -2110,7 +2186,7 @@ class Transport:
|
|||
|
||||
if Transport.local_hops_delta != 0 and from_local_client and not to_local_client: new_raw = Transport.mangle_hops(new_raw, Transport.local_hops_delta)
|
||||
Transport.transmit(outbound_interface, new_raw)
|
||||
path_entry[IDX_PT_TIMESTAMP] = time.time()
|
||||
path_entry[IDX_PT_USED] = time.time()
|
||||
|
||||
else:
|
||||
# TODO: There should probably be some kind of REJECT
|
||||
|
|
@ -2159,7 +2235,11 @@ class Transport:
|
|||
new_raw += struct.pack("!B", packet.hops if not from_local_client or instance_local_link or Transport.local_hops_delta == 0 else Transport.local_hops_delta)
|
||||
new_raw += packet.raw[2:]
|
||||
Transport.transmit(outbound_interface, new_raw)
|
||||
Transport.link_table[packet.destination_hash][IDX_LT_TIMESTAMP] = time.time()
|
||||
|
||||
active_time = time.time()
|
||||
Transport.link_table[packet.destination_hash][IDX_LT_TIMESTAMP] = active_time
|
||||
link_path_entry = Transport.path_table.get(Transport.link_table[packet.destination_hash][IDX_LT_DSTHASH])
|
||||
if link_path_entry: link_path_entry[IDX_PT_USED] = active_time
|
||||
|
||||
else: RNS.log(f"No-outbound return on link packet from {packet.receiving_interface}", RNS.LOG_EXTREME) if RNS.sl(RNS.LOG_EXTREME) else None
|
||||
|
||||
|
|
@ -2213,27 +2293,30 @@ class Transport:
|
|||
if local_and_hops_condition:
|
||||
announce_emitted = Transport.announce_emitted(packet)
|
||||
|
||||
path_used = 0
|
||||
random_blobs = []
|
||||
random_blob = packet.data[RNS.Identity.KEYSIZE//8+RNS.Identity.NAME_HASH_LENGTH//8:RNS.Identity.KEYSIZE//8+RNS.Identity.NAME_HASH_LENGTH//8+10]
|
||||
if not random_blob: return packet.receiving_interface.protocol_violation("No random blob in data") if packet.receiving_interface else None
|
||||
|
||||
with Transport.inbound_announce_lock:
|
||||
announced_destination_known = packet.destination_hash in Transport.path_table
|
||||
existing_path_entry = Transport.path_table.get(packet.destination_hash)
|
||||
announced_destination_known = existing_path_entry != None
|
||||
if not announced_destination_known:
|
||||
# If this destination is unknown in our table
|
||||
# we should add it
|
||||
should_add = True
|
||||
|
||||
else:
|
||||
random_blobs = Transport.path_table[packet.destination_hash][IDX_PT_RANDBLOBS]
|
||||
current_gravity = Transport.path_table[packet.destination_hash][IDX_PT_RVCD_IF].gravity
|
||||
random_blobs = existing_path_entry[IDX_PT_RANDBLOBS]
|
||||
current_gravity = existing_path_entry[IDX_PT_RVCD_IF].gravity
|
||||
path_used = existing_path_entry[IDX_PT_USED]
|
||||
announce_gravity = packet.receiving_interface.gravity if packet.receiving_interface != None else None
|
||||
|
||||
# If we already have a path to the announced destination,
|
||||
# but a more recently emitted announce arrives with a hop
|
||||
# count equal to or less than the existing path, we will
|
||||
# update our tables.
|
||||
if packet.hops <= Transport.path_table[packet.destination_hash][IDX_PT_HOPS]:
|
||||
if packet.hops <= existing_path_entry[IDX_PT_HOPS]:
|
||||
path_timebase = Transport.timebase_from_random_blobs(random_blobs)
|
||||
if not random_blob in random_blobs and announce_emitted > path_timebase:
|
||||
Transport.mark_path_unknown_state(packet.destination_hash)
|
||||
|
|
@ -2255,7 +2338,7 @@ class Transport:
|
|||
# ignore it, unless the path is expired, or
|
||||
# the emission timestamp is more recent.
|
||||
now = time.time()
|
||||
path_expires = Transport.path_table[packet.destination_hash][IDX_PT_EXPIRES]
|
||||
path_expires = existing_path_entry[IDX_PT_EXPIRES]
|
||||
|
||||
path_announce_emitted = 0
|
||||
for path_random_blob in random_blobs:
|
||||
|
|
@ -2302,33 +2385,10 @@ class Transport:
|
|||
rate_blocked = False
|
||||
if packet.context != RNS.Packet.PATH_RESPONSE and packet.receiving_interface.announce_rate_target != None:
|
||||
with Transport.announce_rate_table_lock:
|
||||
if not packet.destination_hash in Transport.announce_rate_table:
|
||||
rate_entry = { "last": now, "rate_violations": 0, "blocked_until": 0, "timestamps": [now]}
|
||||
Transport.announce_rate_table[packet.destination_hash] = rate_entry
|
||||
|
||||
else:
|
||||
rate_entry = Transport.announce_rate_table[packet.destination_hash]
|
||||
rate_entry["timestamps"].append(now)
|
||||
|
||||
while len(rate_entry["timestamps"]) > Transport.MAX_RATE_TIMESTAMPS:
|
||||
rate_entry["timestamps"].pop(0)
|
||||
|
||||
current_rate = now - rate_entry["last"]
|
||||
|
||||
if now > rate_entry["blocked_until"]:
|
||||
if current_rate < packet.receiving_interface.announce_rate_target: rate_entry["rate_violations"] += 1
|
||||
else: rate_entry["rate_violations"] = max(0, rate_entry["rate_violations"]-1)
|
||||
|
||||
if rate_entry["rate_violations"] > packet.receiving_interface.announce_rate_grace:
|
||||
rate_target = packet.receiving_interface.announce_rate_target
|
||||
rate_penalty = packet.receiving_interface.announce_rate_penalty
|
||||
rate_entry["blocked_until"] = rate_entry["last"] + rate_target + rate_penalty
|
||||
rate_blocked = True
|
||||
else:
|
||||
rate_entry["last"] = now
|
||||
|
||||
else: rate_blocked = True
|
||||
|
||||
rate_blocked = Transport.announce_rate_table.check_and_update(packet.destination_hash, now,
|
||||
packet.receiving_interface.announce_rate_target,
|
||||
packet.receiving_interface.announce_rate_grace,
|
||||
packet.receiving_interface.announce_rate_penalty)
|
||||
retries = 0
|
||||
announce_hops = packet.hops
|
||||
local_rebroadcasts = 0
|
||||
|
|
@ -2336,13 +2396,7 @@ class Transport:
|
|||
attached_interface = None
|
||||
|
||||
retransmit_timeout = now + (RNS.rand() * Transport.PATHFINDER_RW)
|
||||
|
||||
if hasattr(packet.receiving_interface, "mode") and packet.receiving_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT:
|
||||
expires = now + Transport.AP_PATH_TIME
|
||||
elif hasattr(packet.receiving_interface, "mode") and packet.receiving_interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING:
|
||||
expires = now + Transport.ROAMING_PATH_TIME
|
||||
else:
|
||||
expires = now + Transport.PATHFINDER_E
|
||||
expires = now + Transport.path_timeout_from_interface(packet.receiving_interface)
|
||||
|
||||
if not random_blob in random_blobs:
|
||||
random_blobs.append(random_blob)
|
||||
|
|
@ -2455,8 +2509,21 @@ class Transport:
|
|||
new_announce.send()
|
||||
|
||||
if not Transport.owner.is_connected_to_shared_instance: Transport.cache(packet, force_cache=True, packet_type="announce")
|
||||
path_table_entry = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet.packet_hash]
|
||||
with Transport.path_table_lock: Transport.path_table[packet.destination_hash] = path_table_entry
|
||||
if rate_blocked:
|
||||
# Rate-limited announces do not update liveness
|
||||
# metrics, only the essential routing data.
|
||||
with Transport.path_table_lock:
|
||||
rate_limited_path_entry = Transport.path_table.get(packet.destination_hash)
|
||||
if rate_limited_path_entry != None:
|
||||
rate_limited_path_entry[IDX_PT_NEXT_HOP] = received_from
|
||||
rate_limited_path_entry[IDX_PT_HOPS] = announce_hops
|
||||
rate_limited_path_entry[IDX_PT_RANDBLOBS] = random_blobs
|
||||
rate_limited_path_entry[IDX_PT_RVCD_IF] = packet.receiving_interface
|
||||
rate_limited_path_entry[IDX_PT_PACKET] = packet.packet_hash
|
||||
else:
|
||||
path_table_entry = [now, received_from, announce_hops, expires, random_blobs, packet.receiving_interface, packet.packet_hash, path_used]
|
||||
with Transport.path_table_lock: Transport.path_table[packet.destination_hash] = path_table_entry
|
||||
|
||||
Transport.mark_path_unknown_state(packet.destination_hash)
|
||||
RNS.log("Destination "+RNS.prettyhexrep(packet.destination_hash)+" is now "+str(announce_hops)+" hops away via "+RNS.prettyhexrep(received_from)+" on "+str(packet.receiving_interface), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
if packet.destination_hash in Transport.path_requests:
|
||||
|
|
@ -2464,13 +2531,13 @@ class Transport:
|
|||
|
||||
# If the receiving interface is a tunnel, we add the
|
||||
# announce to the tunnels table
|
||||
if hasattr(packet.receiving_interface, "tunnel_id") and packet.receiving_interface.tunnel_id != None:
|
||||
if not rate_blocked and hasattr(packet.receiving_interface, "tunnel_id") and packet.receiving_interface.tunnel_id != None:
|
||||
with Transport.tunnels_lock:
|
||||
if not packet.receiving_interface.tunnel_id in Transport.tunnels: RNS.log(f"Tunnel ID for {packet.receiving_interface} was not found in tunnel table", RNS.LOG_WARNING)
|
||||
else:
|
||||
tunnel_entry = Transport.tunnels[packet.receiving_interface.tunnel_id]
|
||||
paths = tunnel_entry[IDX_TT_PATHS]
|
||||
paths[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, None, packet.packet_hash]
|
||||
paths[packet.destination_hash] = [now, received_from, announce_hops, expires, random_blobs, None, packet.packet_hash, path_used]
|
||||
expires = time.time() + Transport.TUNNEL_TIMEOUT
|
||||
tunnel_entry[IDX_TT_EXPIRES] = expires
|
||||
RNS.log("Path to "+RNS.prettyhexrep(packet.destination_hash)+" associated with tunnel "+RNS.prettyhexrep(packet.receiving_interface.tunnel_id), RNS.LOG_PATHING) if RNS.sl(RNS.LOG_PATHING) else None
|
||||
|
|
@ -2590,8 +2657,7 @@ class Transport:
|
|||
# to another path, we remove this packet hash from
|
||||
# the filter hashlist so the link can receive the
|
||||
# packet when it finally arrives over another path.
|
||||
while packet.packet_hash in Transport.packet_hashlist: Transport.packet_hashlist.remove(packet.packet_hash)
|
||||
while packet.packet_hash in Transport.packet_hashlist_prev: Transport.packet_hashlist_prev.remove(packet.packet_hash)
|
||||
Transport.packet_hashlist.discard(packet.packet_hash)
|
||||
else:
|
||||
destination = Transport.destinations_map.get(packet.destination_hash)
|
||||
if destination and destination.type == packet.destination_type:
|
||||
|
|
@ -2632,6 +2698,7 @@ class Transport:
|
|||
link_entry[IDX_LT_REM_HOPS] = packet.hops
|
||||
path_entry = Transport.path_table.get(link_destination)
|
||||
if path_entry: path_entry[IDX_PT_HOPS] = packet.hops
|
||||
Transport.mark_path_responsive(link_destination)
|
||||
|
||||
elif not link_entry[IDX_LT_VALIDATED]: RNS.log(f"Aborting link request proof path re-balancing for {RNS.prettyhexrep(link_destination)} on link {RNS.prettyhexrep(packet.destination_hash)} due to invalid signature", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
|
||||
else: pass
|
||||
|
|
@ -2662,6 +2729,7 @@ class Transport:
|
|||
Transport.transmit(link_entry[IDX_LT_RCVD_IF], new_raw)
|
||||
if not Transport.owner.is_connected_to_shared_instance:
|
||||
RNS.Identity._used_destination_data(link_entry[IDX_LT_DSTHASH])
|
||||
Transport.mark_path_responsive(link_entry[IDX_LT_DSTHASH])
|
||||
|
||||
else:
|
||||
RNS.log("Invalid link request proof in transport for link "+RNS.prettyhexrep(packet.destination_hash)+", dropping proof.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
|
|
@ -2706,6 +2774,7 @@ class Transport:
|
|||
if path_entry:
|
||||
path_entry[IDX_PT_HOPS] = packet.hops
|
||||
RNS.log(f"Path table re-balanced for {RNS.prettyhexrep(link.destination.hash)}", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
|
||||
Transport.mark_path_responsive(link.destination.hash)
|
||||
|
||||
else: RNS.log(f"Aborting path re-balancing at link terminus for {RNS.prettyhexrep(link.destination.hash)} on link {link} due to invalid signature", REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
|
||||
except Exception as e: RNS.log("Error while validating link request proof for path re-balancing at link terminus. The contained exception was: "+str(e), REBALANCE_LOGLEVEL) if RNS.sl(REBALANCE_LOGLEVEL) else None
|
||||
|
|
@ -2837,13 +2906,15 @@ class Transport:
|
|||
deprecated_paths = []
|
||||
with Transport.tunnels_lock:
|
||||
for destination_hash, path_entry in paths.items():
|
||||
path_timestamp = path_entry[0]
|
||||
received_from = path_entry[1]
|
||||
announce_hops = path_entry[2]
|
||||
expires = path_entry[3]
|
||||
random_blobs = list(set(path_entry[4]))
|
||||
receiving_interface = interface
|
||||
packet_hash = path_entry[6]
|
||||
new_entry = [time.time(), received_from, announce_hops, expires, random_blobs, receiving_interface, packet_hash]
|
||||
used = path_entry[7]
|
||||
new_entry = [path_timestamp, received_from, announce_hops, expires, random_blobs, receiving_interface, packet_hash, used]
|
||||
|
||||
should_add = False
|
||||
with Transport.path_table_lock:
|
||||
|
|
@ -2991,6 +3062,21 @@ class Transport:
|
|||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def init_announce_cache():
|
||||
try:
|
||||
Transport.announce_cache = AnnounceCache(os.path.join(RNS.Reticulum.cachepath, "announces.db"))
|
||||
|
||||
legacy_dir = os.path.join(RNS.Reticulum.cachepath, "announces")
|
||||
if os.path.isdir(legacy_dir) and len(os.listdir(legacy_dir)) > 0:
|
||||
st = time.time()
|
||||
imported = Transport.announce_cache.import_legacy(legacy_dir)
|
||||
RNS.log(f"Imported {imported} cached announces from legacy file storage in {RNS.prettyshorttime(time.time()-st)}", RNS.LOG_VERBOSE)
|
||||
|
||||
except Exception as e:
|
||||
RNS.log("Could not initialise announce cache database, falling back to file-based caching. The contained exception was: "+str(e), RNS.LOG_ERROR)
|
||||
Transport.announce_cache = None
|
||||
|
||||
@staticmethod
|
||||
def should_cache(packet):
|
||||
# TODO: Rework the caching system. It's currently
|
||||
|
|
@ -3024,10 +3110,17 @@ class Transport:
|
|||
@staticmethod
|
||||
def clean_announce_cache():
|
||||
st = time.time()
|
||||
with Transport.path_table_lock: active_path_hashes = set([Transport.path_table[dst_hash][IDX_PT_PACKET] for dst_hash in Transport.path_table])
|
||||
with Transport.tunnels_lock: tunnel_path_hashes = set([path_dict[dst_hash][IDX_PT_PACKET] for path_dict in [Transport.tunnels[tunnel_id][IDX_TT_PATHS] for tunnel_id in Transport.tunnels] for dst_hash in path_dict])
|
||||
active_path_hashes |= tunnel_path_hashes
|
||||
|
||||
if Transport.announce_cache != None:
|
||||
removed = Transport.announce_cache.clean(active_path_hashes)
|
||||
if removed > 0: RNS.log(f"Removed {removed} cached announces in {RNS.prettytime(time.time()-st)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
return
|
||||
|
||||
target_path = os.path.join(RNS.Reticulum.cachepath, "announces")
|
||||
cached_announce_hashes = os.listdir(target_path)
|
||||
with Transport.path_table_lock: active_path_hashes = list(set([Transport.path_table[dst_hash][IDX_PT_PACKET] for dst_hash in Transport.path_table]))
|
||||
with Transport.tunnels_lock: tunnel_path_hashes = list(set([path_dict[dst_hash][IDX_PT_PACKET] for path_dict in [Transport.tunnels[tunnel_id][IDX_TT_PATHS] for tunnel_id in Transport.tunnels] for dst_hash in path_dict]))
|
||||
removed = 0; total = 0
|
||||
for packet_hash in cached_announce_hashes:
|
||||
remove = False
|
||||
|
|
@ -3053,22 +3146,40 @@ class Transport:
|
|||
def cache(packet, force_cache=False, packet_type=None):
|
||||
if force_cache or RNS.Transport.should_cache(packet):
|
||||
try:
|
||||
packet_hash = RNS.hexrep(packet.get_hash(), delimit=False)
|
||||
interface_reference = None
|
||||
if packet.receiving_interface != None: interface_reference = str(packet.receiving_interface)
|
||||
if packet_type == "announce": cachepath = os.path.join(RNS.Reticulum.cachepath, "announces", packet_hash)
|
||||
else: cachepath = os.path.join(RNS.Reticulum.cachepath, packet_hash)
|
||||
|
||||
if packet_type == "announce":
|
||||
if Transport.announce_cache != None:
|
||||
Transport.announce_cache.store(packet.get_hash(), packet.raw, interface_reference)
|
||||
return
|
||||
|
||||
cachepath = os.path.join(RNS.Reticulum.cachepath, "announces", RNS.hexrep(packet.get_hash(), delimit=False))
|
||||
else:
|
||||
cachepath = os.path.join(RNS.Reticulum.cachepath, RNS.hexrep(packet.get_hash(), delimit=False))
|
||||
|
||||
file = open(cachepath, "wb")
|
||||
file.write(umsgpack.packb([packet.raw, interface_reference]))
|
||||
file.close()
|
||||
|
||||
except Exception as e:
|
||||
RNS.log("Error writing packet to cache. The contained exception was: "+str(e), RNS.LOG_ERROR)
|
||||
except Exception as e: RNS.log("Error writing packet to cache. The contained exception was: "+str(e), RNS.LOG_ERROR)
|
||||
|
||||
@staticmethod
|
||||
def get_cached_packet(packet_hash, packet_type=None):
|
||||
try:
|
||||
if packet_type == "announce" and Transport.announce_cache != None:
|
||||
cached = Transport.announce_cache.fetch(packet_hash)
|
||||
if cached == None: return None
|
||||
|
||||
packet = RNS.Packet(None, cached[0])
|
||||
interface_reference = cached[1]
|
||||
|
||||
for interface in Transport.interfaces:
|
||||
if str(interface) == interface_reference:
|
||||
packet.receiving_interface = interface
|
||||
|
||||
return packet
|
||||
|
||||
packet_hash = RNS.hexrep(packet_hash, delimit=False)
|
||||
if packet_type == "announce": path = os.path.join(RNS.Reticulum.cachepath, "announces", packet_hash)
|
||||
else: path = os.path.join(RNS.Reticulum.cachepath, packet_hash)
|
||||
|
|
@ -3207,6 +3318,22 @@ class Transport:
|
|||
if not Transport.lowest_interface_bitrate: return 0
|
||||
return 2*(RNS.Reticulum.MTU*8/max(Transport.lowest_interface_bitrate, RNS.Reticulum.MINIMUM_BITRATE)) + RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT
|
||||
|
||||
@staticmethod
|
||||
def path_count():
|
||||
return len(Transport.path_table)
|
||||
|
||||
@staticmethod
|
||||
def unused_path_count():
|
||||
with Transport.path_table_lock:
|
||||
return sum(1 for e in Transport.path_table.values() if not e[IDX_PT_USED])
|
||||
|
||||
@staticmethod
|
||||
def path_pool_counts():
|
||||
with Transport.path_table_lock:
|
||||
total = len(Transport.path_table)
|
||||
unused = sum(1 for e in Transport.path_table.values() if not e[IDX_PT_USED])
|
||||
return total, unused
|
||||
|
||||
@staticmethod
|
||||
def link_count():
|
||||
return len(Transport.link_table)
|
||||
|
|
@ -3215,16 +3342,46 @@ class Transport:
|
|||
def active_link_count():
|
||||
return sum(1 for e in (True for entry in Transport.link_table if entry[IDX_LT_VALIDATED]))
|
||||
|
||||
@staticmethod
|
||||
def link_distribution():
|
||||
link_destinations = {}
|
||||
|
||||
# TODO: Finish implementation
|
||||
return link_destinations
|
||||
|
||||
# try:
|
||||
# for link_id in Transport.link_table:
|
||||
# link_entry = Transport.link_table.get(link_id)
|
||||
# if link_entry:
|
||||
# link_destination = link_entry[IDX_LT_DSTHASH]
|
||||
# active = link_entry[IDX_LT_VALIDATED]
|
||||
|
||||
# if not link_destination in link_destinations:
|
||||
# link_destinations[link_destination] = {}
|
||||
|
||||
# else:
|
||||
# pass
|
||||
|
||||
# except Exception as e: RNS.log(f"Error while getting link distribution stats: {e}", RNS.LOG_DEBUG)
|
||||
|
||||
@staticmethod
|
||||
def expire_path(destination_hash):
|
||||
with Transport.path_table_lock:
|
||||
if destination_hash in Transport.path_table:
|
||||
Transport.path_table[destination_hash][IDX_PT_TIMESTAMP] = 0
|
||||
Transport.path_table[destination_hash][IDX_PT_EXPIRES] = 0
|
||||
Transport.tables_last_culled = 0
|
||||
return True
|
||||
|
||||
else: return False
|
||||
|
||||
@staticmethod
|
||||
def path_timeout_from_interface(interface):
|
||||
if interface != None and hasattr(interface, "mode"):
|
||||
if interface.mode == RNS.Interfaces.Interface.Interface.MODE_ACCESS_POINT: return Transport.AP_PATH_TIME
|
||||
elif interface.mode == RNS.Interfaces.Interface.Interface.MODE_ROAMING: return Transport.ROAMING_PATH_TIME
|
||||
return Transport.PATH_TIMEOUT
|
||||
|
||||
@staticmethod
|
||||
def mark_path_unresponsive(destination_hash):
|
||||
if destination_hash in Transport.path_table:
|
||||
|
|
@ -3235,11 +3392,22 @@ class Transport:
|
|||
|
||||
@staticmethod
|
||||
def mark_path_responsive(destination_hash):
|
||||
if destination_hash in Transport.path_table:
|
||||
with Transport.path_states_lock: Transport.path_states[destination_hash] = Transport.STATE_RESPONSIVE
|
||||
return True
|
||||
|
||||
else: return False
|
||||
# Marking a path live must only occur on events that
|
||||
# cryptographically prove recent destination liveness.
|
||||
now = time.time()
|
||||
with Transport.path_table_lock:
|
||||
path_entry = Transport.path_table.get(destination_hash)
|
||||
if path_entry != None:
|
||||
path_entry[IDX_PT_TIMESTAMP] = now
|
||||
path_entry[IDX_PT_EXPIRES] = now + Transport.path_timeout_from_interface(path_entry[IDX_PT_RVCD_IF])
|
||||
path_entry[IDX_PT_USED] = now
|
||||
|
||||
with Transport.path_states_lock:
|
||||
Transport.path_states[destination_hash] = Transport.STATE_RESPONSIVE
|
||||
|
||||
return True
|
||||
|
||||
else: return False
|
||||
|
||||
@staticmethod
|
||||
def mark_path_unknown_state(destination_hash):
|
||||
|
|
@ -3767,14 +3935,14 @@ class Transport:
|
|||
|
||||
packet_hashlist_path = RNS.Reticulum.storagepath+"/packet_hashlist.raw"
|
||||
with open(packet_hashlist_path, "wb") as file:
|
||||
for packet_hash in Transport.packet_hashlist.copy():
|
||||
for bucket in Transport.packet_hashlist.buckets():
|
||||
if background:
|
||||
if time.time() - round_started_at > yield_threshold:
|
||||
# Low priority, yield thread
|
||||
round_started_at = time.time()
|
||||
time.sleep(0.001)
|
||||
|
||||
file.write(packet_hash)
|
||||
file.write(bucket)
|
||||
|
||||
RNS.log(f"Saved packet hashlist in {RNS.prettyshorttime(time.time()-save_start)}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
|
||||
|
||||
|
|
@ -3839,6 +4007,7 @@ class Transport:
|
|||
expires = de[IDX_PT_EXPIRES]
|
||||
random_blobs = de[IDX_PT_RANDBLOBS]
|
||||
packet_hash = de[IDX_PT_PACKET]
|
||||
used = de[IDX_PT_USED]
|
||||
interface_hash = interface.get_hash()
|
||||
|
||||
serialised_entry = [ destination_hash,
|
||||
|
|
@ -3848,7 +4017,8 @@ class Transport:
|
|||
expires,
|
||||
random_blobs,
|
||||
interface_hash,
|
||||
packet_hash ]
|
||||
packet_hash,
|
||||
used ]
|
||||
|
||||
serialised_destinations.append(serialised_entry)
|
||||
|
||||
|
|
@ -3922,6 +4092,7 @@ class Transport:
|
|||
path_expires = de[3]
|
||||
random_blobs = de[4][-Transport.PERSIST_RANDOM_BLOBS:]
|
||||
packet_hash = de[6]
|
||||
used = de[7]
|
||||
|
||||
serialised_entry = [
|
||||
destination_hash,
|
||||
|
|
@ -3931,7 +4102,8 @@ class Transport:
|
|||
path_expires,
|
||||
random_blobs,
|
||||
interface_hash,
|
||||
packet_hash
|
||||
packet_hash,
|
||||
used,
|
||||
]
|
||||
|
||||
serialised_paths.append(serialised_entry)
|
||||
|
|
@ -3979,6 +4151,7 @@ class Transport:
|
|||
Transport.void_queues()
|
||||
if not Transport.owner.is_connected_to_shared_instance:
|
||||
Transport.persist_data()
|
||||
if Transport.announce_cache != None: Transport.announce_cache.flush()
|
||||
|
||||
@staticmethod
|
||||
def blackhole_identity(identity_hash, until=None, reason=None):
|
||||
|
|
@ -4126,6 +4299,7 @@ IDX_PT_EXPIRES = 3
|
|||
IDX_PT_RANDBLOBS = 4
|
||||
IDX_PT_RVCD_IF = 5
|
||||
IDX_PT_PACKET = 6
|
||||
IDX_PT_USED = 7
|
||||
|
||||
# Transport.reverse_table entry indices
|
||||
IDX_RT_RCVD_IF = 0
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
@ -102,7 +102,7 @@ def parse_hash(input_str):
|
|||
def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity, timeout, drop_queues,
|
||||
drop_via, max_hops, remote=None, management_identity=None, remote_timeout=RNS.Transport.PATH_REQUEST_TIMEOUT,
|
||||
blackholed=False, blackhole=False, unblackhole=False, blackhole_duration=None, blackhole_reason=None,
|
||||
remote_blackhole_list=False, remote_blackhole_list_filter=None, no_output=False, json=False):
|
||||
remote_blackhole_list=False, remote_blackhole_list_filter=None, usage=False, no_output=False, json=False):
|
||||
|
||||
global remote_link, reticulum
|
||||
reticulum = RNS.Reticulum(configdir = configdir, loglevel = 3+verbosity)
|
||||
|
|
@ -281,13 +281,22 @@ def program_setup(configdir, table, rates, drop, destination_hexhash, verbosity,
|
|||
exit()
|
||||
|
||||
else:
|
||||
now = time.time()
|
||||
if usage:
|
||||
try: table = sorted(table, key=lambda e: e["used"], reverse=True)
|
||||
except Exception as e: pass
|
||||
|
||||
for path in table:
|
||||
if destination_hash == None or destination_hash == path["hash"]:
|
||||
displayed += 1
|
||||
exp_str = RNS.timestamp_str(path["expires"])
|
||||
if path["hops"] == 1: m_str = " "
|
||||
else: m_str = "s"
|
||||
print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"]))
|
||||
if not "used" in path: u_str = ", no usage info"
|
||||
else:
|
||||
if usage: u_str = f", last used {RNS.prettytime(now-path['used'])} ago" if path["used"] else ", never used"
|
||||
else: u_str = ""
|
||||
print(RNS.prettyhexrep(path["hash"])+" is "+str(path["hops"])+" hop"+m_str+" away via "+RNS.prettyhexrep(path["via"])+" on "+path["interface"]+" expires "+RNS.timestamp_str(path["expires"])+u_str)
|
||||
|
||||
if destination_hash != None and displayed == 0:
|
||||
print("No path known")
|
||||
|
|
@ -484,6 +493,7 @@ def main():
|
|||
parser.add_argument("--config", action="store", default=None, help="path to alternative Reticulum config directory", type=str)
|
||||
parser.add_argument("--version", action="version", version="rnpath {version}".format(version=__version__))
|
||||
parser.add_argument("-t", "--table", action="store_true", help="show all known paths", default=False)
|
||||
parser.add_argument("-u", "--usage", action="store_true", help="show path usage time", default=False)
|
||||
parser.add_argument("-m", "--max", action="store", metavar="hops", type=int, help="maximum hops to filter path table by", default=None)
|
||||
parser.add_argument("-r", "--rates", action="store_true", help="show announce rate info", default=False)
|
||||
parser.add_argument("-d", "--drop", action="store_true", help="remove the path to a destination", default=False)
|
||||
|
|
@ -518,7 +528,7 @@ def main():
|
|||
verbosity = args.verbose, timeout = args.w, drop_queues = args.drop_announces, drop_via = args.drop_via, max_hops = args.max,
|
||||
remote=args.R, management_identity=args.i, remote_timeout=args.W, blackholed=args.blackholed, blackhole=args.blackhole,
|
||||
unblackhole=args.unblackhole, blackhole_duration=args.duration, blackhole_reason=args.reason, remote_blackhole_list=args.blackholed_list,
|
||||
remote_blackhole_list_filter=args.list_filter, json=args.json)
|
||||
remote_blackhole_list_filter=args.list_filter, usage=args.usage, json=args.json)
|
||||
|
||||
sys.exit(0)
|
||||
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -2,7 +2,7 @@
|
|||
|
||||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
# Reticulum License
|
||||
#
|
||||
# Copyright (c) 2016-2025 Mark Qvist
|
||||
# Copyright (c) 2016-2026 Mark Qvist
|
||||
#
|
||||
# Permission is hereby granted, free of charge, to any person obtaining a copy
|
||||
# of this software and associated documentation files (the "Software"), to deal
|
||||
|
|
|
|||
|
|
@ -304,9 +304,7 @@ def build_scenarios(instance, interface, interface_b, requested=None):
|
|||
salt=b"bench", context=None)
|
||||
now = time.time()
|
||||
with RNS.Transport.path_table_lock:
|
||||
RNS.Transport.path_table[dst_h] = [
|
||||
now, next_hop, 3, now + 3600, [], interface, bytes(32)
|
||||
]
|
||||
RNS.Transport.path_table[dst_h] = [now, next_hop, 3, now + 3600, [], interface, bytes(32), now]
|
||||
frames = [_transit_single_frame(interface, dst_h,
|
||||
_payload(i, size, 0x57))
|
||||
for i in range(total)]
|
||||
|
|
@ -327,9 +325,7 @@ def build_scenarios(instance, interface, interface_b, requested=None):
|
|||
salt=b"bench", context=None)
|
||||
now = time.time()
|
||||
with RNS.Transport.path_table_lock:
|
||||
RNS.Transport.path_table[dst_h] = [
|
||||
now, next_hop, 1, now + 3600, [], interface, bytes(32)
|
||||
]
|
||||
RNS.Transport.path_table[dst_h] = [now, next_hop, 1, now + 3600, [], interface, bytes(32), now]
|
||||
frames = [_transit_single_frame(interface, dst_h,
|
||||
_payload(i, 135, 0x46))
|
||||
for i in range(total)]
|
||||
|
|
@ -450,9 +446,7 @@ def build_scenarios(instance, interface, interface_b, requested=None):
|
|||
salt=b"bench", context=None)
|
||||
now = time.time()
|
||||
with RNS.Transport.path_table_lock:
|
||||
RNS.Transport.path_table[dst_h] = [
|
||||
now, next_hop, 3, now + 3600, [], interface, bytes(32)
|
||||
]
|
||||
RNS.Transport.path_table[dst_h] = [now, next_hop, 3, now + 3600, [], interface, bytes(32), now]
|
||||
remote_id = RNS.Identity(create_keys=False)
|
||||
remote_id.load_public_key(os.urandom(RNS.Identity.KEYSIZE//8))
|
||||
outbound_destination = RNS.Destination(remote_id, RNS.Destination.OUT,
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue