# Reticulum License # # Copyright (c) 2016-2026 Mark Qvist # # Permission is hereby granted, free of charge, to any person obtaining a copy # of this software and associated documentation files (the "Software"), to deal # in the Software without restriction, including without limitation the rights # to use, copy, modify, merge, publish, distribute, sublicense, and/or sell # copies of the Software, and to permit persons to whom the Software is # furnished to do so, subject to the following conditions: # # - The Software shall not be used in any kind of system which includes amongst # its functions the ability to purposefully do harm to human beings. # # - The Software shall not be used, directly or indirectly, in the creation of # an artificial intelligence, machine learning or language model training # dataset, including but not limited to any use that contributes to the # training or development of such a model or algorithm. # # - The above copyright notice and this permission notice shall be included in # all copies or substantial portions of the Software. # # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR # IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, # FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE # AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER # LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, # OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. from array import array import hashlib import threading import os import time class AnnounceRateTable: # Columnar store for announce rate tracking. Layout: # _index : dict[destination_hash -> slot] # _keys : list[slot] -> destination_hash (for swap-remove bookkeeping) # _last : array('d') last non-violating announce time # _blocked: array('d') blocked_until # _viol : array('i') rate_violations # _ts : array('d') flat ring of max_timestamps doubles per slot # _ts_len : array('B') valid timestamp count per slot # _ts_head: array('B') ring write cursor per slot def __init__(self, max_timestamps=16): assert max_timestamps <= 255, "Timestamp ring size must fit in a single byte" self.max_ts = max_timestamps self._index = {} self._keys = [] self._last = array('d') self._blocked = array('d') self._viol = array('i') self._ts = array('d') self._ts_len = array('B') self._ts_head = array('B') self._cursor = 0 def __len__(self): return len(self._keys) def __contains__(self, destination_hash): return destination_hash in self._index # Perform the announce-rate check-and-update for one announce. # Caller must hold the announce_rate_table_lock. def check_and_update(self, destination_hash, now, target, grace, penalty): slot = self._index.get(destination_hash) if slot is None: slot = len(self._keys) self._index[destination_hash] = slot self._keys.append(destination_hash) self._last.append(now) self._blocked.append(0.0) self._viol.append(0) self._ts.extend([0.0]*self.max_ts) self._ts[slot*self.max_ts] = now self._ts_len.append(1) self._ts_head.append(1 % self.max_ts) return False base = slot*self.max_ts head = self._ts_head[slot] self._ts[base+head] = now self._ts_head[slot] = (head+1) % self.max_ts if self._ts_len[slot] < self.max_ts: self._ts_len[slot] += 1 current_rate = now - self._last[slot] rate_blocked = False if now > self._blocked[slot]: if current_rate < target: self._viol[slot] += 1 else: v = self._viol[slot] - 1 self._viol[slot] = v if v > 0 else 0 if self._viol[slot] > grace: self._blocked[slot] = self._last[slot] + target + penalty rate_blocked = True else: self._last[slot] = now else: rate_blocked = True return rate_blocked # Caller must hold the table lock def cull(self, now, ttl, max_scan=1024, max_cull=None): if max_cull is None: max_cull = max_scan horizon = now - ttl culled = 0 scanned = 0 n = len(self._keys) while scanned < max_scan and culled < max_cull and n > 0: if self._cursor >= n: self._cursor = 0 slot = self._cursor if self._last[slot] < horizon and self._blocked[slot] <= now: last_i = n - 1 stale_key = self._keys[slot] if slot != last_i: moved_key = self._keys[last_i] self._keys[slot] = moved_key self._index[moved_key] = slot self._last[slot] = self._last[last_i] self._blocked[slot] = self._blocked[last_i] self._viol[slot] = self._viol[last_i] self._ts_len[slot] = self._ts_len[last_i] self._ts_head[slot] = self._ts_head[last_i] f = last_i*self.max_ts; t = slot*self.max_ts self._ts[t:t+self.max_ts] = self._ts[f:f+self.max_ts] self._keys.pop() self._last.pop(); self._blocked.pop(); self._viol.pop() self._ts_len.pop(); self._ts_head.pop() del self._ts[len(self._ts)-self.max_ts:] # drop tail block of removed slot del self._index[stale_key] n -= 1 culled += 1 # Do not advance cursor, re-examine the slot a moved entry landed in else: self._cursor = slot + 1 scanned += 1 if self._cursor >= n: self._cursor = 0 return culled # Hold table lock, or accept a possibly inconsistent snapshot def items(self): for slot, key in enumerate(self._keys): n = self._ts_len[slot] head = self._ts_head[slot] base = slot*self.max_ts if n < self.max_ts: ts = list(self._ts[base:base+n]) else: ts = list(self._ts[base+head:base+self.max_ts]) + list(self._ts[base:base+head]) yield key, { "last": self._last[slot], "rate_violations": self._viol[slot], "blocked_until": self._blocked[slot], "timestamps": ts } class PacketHashlist: # Bucketed byte-blob packet filter hashlist, with the primary # design goal of memory coherence and density. All operations # are safe to call without holding a lock. def __init__(self, hash_width=32, bucket_count=16384): assert bucket_count > 0 and bucket_count & (bucket_count-1) == 0, "Bucket count must be a power of two" self.width = hash_width self._nb = bucket_count self._mask = bucket_count - 1 self._cur = [bytearray() for _ in range(bucket_count)] self._prv = [bytearray() for _ in range(bucket_count)] self._count = 0 def __len__(self): return self._count def __contains__(self, packet_hash): bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask return packet_hash in self._cur[bucket] or packet_hash in self._prv[bucket] def add(self, packet_hash): assert len(packet_hash) == self.width, "Packet hash width mismatch" bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask if packet_hash not in self._cur[bucket]: self._cur[bucket].extend(packet_hash) self._count += 1 def discard(self, packet_hash): w = self.width bucket = (packet_hash[0] | (packet_hash[1] << 8)) & self._mask for generation in (self._cur, self._prv): entries = generation[bucket] pos = entries.find(packet_hash) while pos != -1: if pos % w == 0: del entries[pos:pos+w] if generation is self._cur: self._count -= 1 pos = entries.find(packet_hash, pos) else: pos = entries.find(packet_hash, pos+1) def swap(self): self._prv = self._cur self._cur = [bytearray() for _ in range(self._nb)] self._count = 0 # Current-generation buckets for persist operations def buckets(self): return tuple(self._cur) class KnownDestinationsStore: # Write-behind SQLite storage backend for known destinations. # The actual in-memory table is used directly for all transport # operations; this serves purely as the store, and no read operations # utilize the database directly. def __init__(self, db_path, hash_width=16, read_only=False): import sqlite3 self.sqlite3 = sqlite3 self.db_path = db_path self.hash_width = hash_width self.read_only = read_only self._dirty_upsert = set() self._dirty_delete = set() self._ratchet_upserts = {} self._ratchet_deletes = set() self._flush_lock = threading.Lock() if read_only: self._conn = sqlite3.connect(f"file:{db_path}?mode=ro", uri=True, check_same_thread=False, isolation_level=None) else: self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None) self._conn.execute("PRAGMA journal_mode=WAL") self._conn.execute("PRAGMA synchronous=NORMAL") self._conn.execute("PRAGMA journal_size_limit=8388608") self.__create_schema() def close(self): if self._conn != None: self._conn.close() self._conn = None def __create_schema(self): db_version = self._conn.execute("PRAGMA user_version").fetchone()[0] self._conn.execute("""CREATE TABLE IF NOT EXISTS known_destinations ( destination_hash BLOB PRIMARY KEY, last_heard REAL NOT NULL, packet_hash BLOB NOT NULL, public_key BLOB NOT NULL, app_data BLOB, used REAL NOT NULL DEFAULT 0, stamp BLOB, stamp_value REAL, lr_cost REAL, identity_hash BLOB NOT NULL)""") self._conn.execute("""CREATE TABLE IF NOT EXISTS ratchets ( destination_hash BLOB PRIMARY KEY, ratchet BLOB NOT NULL, received REAL NOT NULL)""") if 0 < db_version < 2: self._conn.execute("ALTER TABLE known_destinations ADD COLUMN lr_cost REAL") self._conn.execute("PRAGMA user_version = 2") self._conn.commit() def __entry_row(self, destination_hash, entry): stamp = entry[5] if len(entry) > 5 else None stamp_value = entry[6] if len(entry) > 6 else 0 lr_cost = entry[7] if len(entry) > 7 else None identity_hash = hashlib.sha256(entry[2]).digest()[:self.hash_width] return (destination_hash, entry[0], entry[1], entry[2], entry[3], entry[4], stamp, stamp_value, lr_cost, identity_hash) # Caller must hold Identity.known_destinations_lock def mark_dirty(self, destination_hash): if self.read_only: return self._dirty_delete.discard(destination_hash) self._dirty_upsert.add(destination_hash) # Caller must hold Identity.known_destinations_lock def mark_deleted(self, destination_hash): if self.read_only: return self._dirty_upsert.discard(destination_hash) self._dirty_delete.add(destination_hash) # Caller must hold Identity.known_destinations_lock def take_pending(self): if self.read_only: return set(), set(), {}, set() upserts = self._dirty_upsert deletes = self._dirty_delete ratchet_upserts = self._ratchet_upserts ratchet_deletes = self._ratchet_deletes self._dirty_upsert = set() self._dirty_delete = set() self._ratchet_upserts = {} self._ratchet_deletes = set() return upserts, deletes, ratchet_upserts, ratchet_deletes # May be called without any locks held def apply(self, upserts, deletes, ratchet_upserts, ratchet_deletes, known_destinations): if self.read_only: return 0 if len(upserts) == 0 and len(deletes) == 0 and len(ratchet_upserts) == 0 and len(ratchet_deletes) == 0: return 0 with self._flush_lock: rows = [] removables = [] ratchet_rows = [] ratchet_removables = [] for destination_hash in upserts: entry = known_destinations.get(destination_hash) if entry != None: rows.append(self.__entry_row(destination_hash, entry)) for destination_hash in deletes: if destination_hash not in known_destinations: removables.append((destination_hash,)) for destination_hash, ratchet_entry in ratchet_upserts.items(): # Ratchets of purged destinations must be removed as well if destination_hash in known_destinations: ratchet_rows.append((destination_hash, *ratchet_entry)) else: ratchet_removables.append((destination_hash,)) for destination_hash in ratchet_deletes: if destination_hash not in known_destinations: ratchet_removables.append((destination_hash,)) try: self._conn.execute("BEGIN") for i in range(0, len(rows), 4096): self._conn.executemany("INSERT INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, lr_cost=excluded.lr_cost, identity_hash=excluded.identity_hash", rows[i:i+4096]) for i in range(0, len(removables), 4096): self._conn.executemany("DELETE FROM known_destinations WHERE destination_hash=?", removables[i:i+4096]) for i in range(0, len(ratchet_rows), 4096): self._conn.executemany("INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received", ratchet_rows[i:i+4096]) for i in range(0, len(ratchet_removables), 4096): self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", ratchet_removables[i:i+4096]) self._conn.commit() except Exception: # Re-mark everything as dirty, so the changes are retried # on the next flush, then propagate. self._dirty_upsert |= {row[0] for row in rows} self._dirty_delete |= {row[0] for row in removables} self._ratchet_upserts |= {row[0]: row[1:] for row in ratchet_rows} self._ratchet_deletes |= {row[0] for row in ratchet_removables} self._conn.rollback() raise return len(rows) + len(removables) + len(ratchet_rows) + len(ratchet_removables) def load_all(self): cursor = self._conn.execute("SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost FROM known_destinations") while True: rows = cursor.fetchmany(4096) if not rows: break for row in rows: yield row[0], [row[1], row[2], row[3], row[4], row[5], row[6], row[7], row[8]] # Caller must hold Identity.known_destinations_lock. def remember_ratchet(self, destination_hash, ratchet, now): if self.read_only: return False pending = self._ratchet_upserts.get(destination_hash) if pending != None: if pending[0] == ratchet: self._ratchet_deletes.discard(destination_hash) return False else: row = self._conn.execute("SELECT ratchet FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone() if row != None and row[0] == ratchet: # Discard any pending delete mark queued # before this ratchet arrived self._ratchet_deletes.discard(destination_hash) return False self._ratchet_upserts[destination_hash] = (ratchet, now) self._ratchet_deletes.discard(destination_hash) return True def fetch_ratchet(self, destination_hash): pending = self._ratchet_upserts.get(destination_hash) if pending != None: return pending[0], pending[1] if destination_hash in self._ratchet_deletes: return None row = self._conn.execute("SELECT ratchet, received FROM ratchets WHERE destination_hash=?", (destination_hash,)).fetchone() if row != None: return row[0], row[1] return None # Caller must hold Identity.known_destinations_lock def mark_ratchet_deleted(self, destination_hash): if self.read_only: return self._ratchet_upserts.pop(destination_hash, None) self._ratchet_deletes.add(destination_hash) def expire_ratchets(self, cutoff): if self.read_only: return [] with self._flush_lock: # Drop expired entries in pending as well expired = set(row[0] for row in self._conn.execute("SELECT destination_hash FROM ratchets WHERE received 0: self._conn.execute("BEGIN") for i in range(0, len(expired), 4096): self._conn.executemany("DELETE FROM ratchets WHERE destination_hash=?", [(h,) for h in list(expired)[i:i+4096]]) self._conn.commit() return list(expired) def load_ratchets(self): cursor = self._conn.execute("SELECT destination_hash, ratchet, received FROM ratchets") while True: rows = cursor.fetchmany(4096) if not rows: break for row in rows: yield row[0], row[1], row[2] def import_legacy_ratchets(self, dir_path, expiry_cutoff, known_destinations, ratchet_width=32): import RNS from .vendor import umsgpack imported = 0 for filename in os.listdir(dir_path): file_path = os.path.join(dir_path, filename) try: if not os.path.isfile(file_path): continue with open(file_path, "rb") as rf: ratchet_data = umsgpack.unpackb(rf.read()) destination_hash = bytes.fromhex(filename) ratchet = ratchet_data["ratchet"]; received = ratchet_data["received"] if len(ratchet) != ratchet_width: raise ValueError("Invalid ratchet width") if received < expiry_cutoff: raise ValueError("Ratchet expired") if destination_hash not in known_destinations: raise ValueError("Destination unknown") self._ratchet_upserts[destination_hash] = (ratchet, received) imported += 1 os.unlink(file_path) except Exception as e: RNS.log(f"Could not import ratchet for {filename}: {e}", RNS.LOG_WARNING) try: os.unlink(file_path) except: pass pending = self._ratchet_upserts self._ratchet_upserts = {} self.apply(set(), set(), pending, set(), known_destinations) return imported def import_legacy(self, path): from .vendor import umsgpack with open(path, "rb") as file: legacy = umsgpack.load(file) rows = [] now = time.time() for destination_hash, entry in legacy.items(): if len(entry) < 5: continue entry = list(entry) while len(entry) < 8: entry.append(None if len(entry) in (5, 7) else 0) if entry[0] == None or entry[0] <= 0: entry[0] = now rows.append(self.__entry_row(destination_hash, entry)) self._conn.execute("BEGIN") for i in range(0, len(rows), 4096): self._conn.executemany("INSERT OR REPLACE INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?)", rows[i:i+4096]) self._conn.commit() return len(rows) class AnnounceCache: def __init__(self, db_path): import sqlite3 self.db_path = db_path self._pending = {} self._flush_lock = threading.Lock() self._conn = sqlite3.connect(db_path, check_same_thread=False, isolation_level=None) self._conn.execute("PRAGMA journal_mode=WAL") self._conn.execute("PRAGMA synchronous=NORMAL") self._conn.execute("PRAGMA journal_size_limit=8388608") self._conn.execute("PRAGMA auto_vacuum=INCREMENTAL") self.__create_schema() def __create_schema(self): self._conn.execute("""CREATE TABLE IF NOT EXISTS cached_announces ( packet_hash BLOB PRIMARY KEY, received REAL NOT NULL, raw BLOB NOT NULL, interface_reference TEXT)""") self._conn.execute("PRAGMA user_version = 1") def store(self, packet_hash, raw, interface_reference): self._pending[packet_hash] = (time.time(), raw, interface_reference) def pending_count(self): return len(self._pending) def fetch(self, packet_hash): pending = self._pending.get(packet_hash) if pending != None: return pending[1], pending[2] row = self._conn.execute("SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=?", (packet_hash,)).fetchone() if row != None: return row[0], row[1] return None def flush(self): with self._flush_lock: return self.__apply_pending() def __apply_pending(self): pending = self._pending self._pending = {} if len(pending) == 0: return 0 try: self._conn.execute("BEGIN") items = list(pending.items()) for i in range(0, len(items), 4096): self._conn.executemany("INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference", [(packet_hash, *entry) for packet_hash, entry in items[i:i+4096]]) self._conn.commit() except Exception: # Re-mark everything as pending, so the changes # are retried on the next flush, then propagate. self._pending.update(pending) self._conn.rollback() raise return len(pending) def clean(self, active_hashes): with self._flush_lock: self.__apply_pending() try: self._conn.execute("BEGIN") self._conn.execute("DROP TABLE IF EXISTS temp.active_announces") self._conn.execute("CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY)") active_list = list(active_hashes) for i in range(0, len(active_list), 4096): self._conn.executemany("INSERT OR IGNORE INTO active_announces VALUES (?)", [(h,) for h in active_list[i:i+4096]]) changes_before = self._conn.total_changes self._conn.execute("DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces)") removed = self._conn.total_changes - changes_before self._conn.commit() except Exception: self._conn.rollback() raise if removed > 0: self._conn.execute("PRAGMA incremental_vacuum") return removed def import_legacy(self, dir_path): from .vendor import umsgpack imported = 0 for filename in os.listdir(dir_path): file_path = os.path.join(dir_path, filename) try: if not os.path.isfile(file_path): continue with open(file_path, "rb") as file: cached_data = umsgpack.unpackb(file.read()) packet_hash = bytes.fromhex(filename) interface_reference = cached_data[1] if len(cached_data) > 1 else None self.store(packet_hash, cached_data[0], interface_reference) imported += 1 os.unlink(file_path) except Exception: try: os.unlink(file_path) except Exception: pass self.flush() return imported def close(self): if self._conn != None: self._conn.close() self._conn = None