2026-10-24 00:20:27 +02:00
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
2026-10-31 16:27:00 +01:00
import RNS
2026-10-24 00:20:27 +02:00
import hashlib
2026-11-13 16:50:24 +01:00
import struct
import sys
2026-10-24 00:20:27 +02:00
import threading
import os
import time
2026-10-31 16:27:00 +01:00
from array import array
2026-10-24 00:20:27 +02:00
class AnnounceRateTable :
# Columnar store for announce rate tracking. Layout:
# _index : dict[destination_hash -> slot]
# _keys : list[slot] -> destination_hash (for swap-remove bookkeeping)
# _last : array('d') last non-violating announce time
# _blocked: array('d') blocked_until
# _viol : array('i') rate_violations
# _ts : array('d') flat ring of max_timestamps doubles per slot
# _ts_len : array('B') valid timestamp count per slot
# _ts_head: array('B') ring write cursor per slot
def __init__ ( self , max_timestamps = 16 ) :
assert max_timestamps < = 255 , " Timestamp ring size must fit in a single byte "
self . max_ts = max_timestamps
self . _index = { }
self . _keys = [ ]
self . _last = array ( ' d ' )
self . _blocked = array ( ' d ' )
self . _viol = array ( ' i ' )
self . _ts = array ( ' d ' )
self . _ts_len = array ( ' B ' )
self . _ts_head = array ( ' B ' )
self . _cursor = 0
def __len__ ( self ) : return len ( self . _keys )
def __contains__ ( self , destination_hash ) : return destination_hash in self . _index
# Perform the announce-rate check-and-update for one announce.
# Caller must hold the announce_rate_table_lock.
def check_and_update ( self , destination_hash , now , target , grace , penalty ) :
slot = self . _index . get ( destination_hash )
if slot is None :
slot = len ( self . _keys )
self . _index [ destination_hash ] = slot
self . _keys . append ( destination_hash )
self . _last . append ( now )
self . _blocked . append ( 0.0 )
self . _viol . append ( 0 )
self . _ts . extend ( [ 0.0 ] * self . max_ts )
self . _ts [ slot * self . max_ts ] = now
self . _ts_len . append ( 1 )
self . _ts_head . append ( 1 % self . max_ts )
return False
base = slot * self . max_ts
head = self . _ts_head [ slot ]
self . _ts [ base + head ] = now
self . _ts_head [ slot ] = ( head + 1 ) % self . max_ts
if self . _ts_len [ slot ] < self . max_ts : self . _ts_len [ slot ] + = 1
current_rate = now - self . _last [ slot ]
rate_blocked = False
if now > self . _blocked [ slot ] :
if current_rate < target : self . _viol [ slot ] + = 1
else :
v = self . _viol [ slot ] - 1
self . _viol [ slot ] = v if v > 0 else 0
if self . _viol [ slot ] > grace :
self . _blocked [ slot ] = self . _last [ slot ] + target + penalty
rate_blocked = True
else : self . _last [ slot ] = now
else : rate_blocked = True
return rate_blocked
# Caller must hold the table lock
def cull ( self , now , ttl , max_scan = 1024 , max_cull = None ) :
if max_cull is None : max_cull = max_scan
horizon = now - ttl
culled = 0
scanned = 0
n = len ( self . _keys )
while scanned < max_scan and culled < max_cull and n > 0 :
if self . _cursor > = n : self . _cursor = 0
slot = self . _cursor
if self . _last [ slot ] < horizon and self . _blocked [ slot ] < = now :
last_i = n - 1
stale_key = self . _keys [ slot ]
if slot != last_i :
moved_key = self . _keys [ last_i ]
self . _keys [ slot ] = moved_key
self . _index [ moved_key ] = slot
self . _last [ slot ] = self . _last [ last_i ]
self . _blocked [ slot ] = self . _blocked [ last_i ]
self . _viol [ slot ] = self . _viol [ last_i ]
self . _ts_len [ slot ] = self . _ts_len [ last_i ]
self . _ts_head [ slot ] = self . _ts_head [ last_i ]
f = last_i * self . max_ts ; t = slot * self . max_ts
self . _ts [ t : t + self . max_ts ] = self . _ts [ f : f + self . max_ts ]
self . _keys . pop ( )
self . _last . pop ( ) ; self . _blocked . pop ( ) ; self . _viol . pop ( )
self . _ts_len . pop ( ) ; self . _ts_head . pop ( )
del self . _ts [ len ( self . _ts ) - self . max_ts : ] # drop tail block of removed slot
del self . _index [ stale_key ]
n - = 1
culled + = 1
# Do not advance cursor, re-examine the slot a moved entry landed in
else :
self . _cursor = slot + 1
scanned + = 1
if self . _cursor > = n : self . _cursor = 0
return culled
# Hold table lock, or accept a possibly inconsistent snapshot
def items ( self ) :
for slot , key in enumerate ( self . _keys ) :
n = self . _ts_len [ slot ]
head = self . _ts_head [ slot ]
base = slot * self . max_ts
if n < self . max_ts : ts = list ( self . _ts [ base : base + n ] )
else : ts = list ( self . _ts [ base + head : base + self . max_ts ] ) + list ( self . _ts [ base : base + head ] )
yield key , { " last " : self . _last [ slot ] , " rate_violations " : self . _viol [ slot ] ,
" blocked_until " : self . _blocked [ slot ] , " timestamps " : ts }
2026-11-13 16:50:24 +01:00
# Persists the announce rate table to a file, in
# compacted, little-endian binary format:
#
# Header (16 bytes)
# 0x00 6 Magic bytes
# 0x06 1 Format version
# 0x07 1 Flags, reserved, must be zero
# 0x08 1 Timestamp ring size (max_ts)
# 0x09 1 Destination hash length in bytes (keylen)
# 0x0A 2 Reserved, zero
# 0x0C 4 Entry count, uint32, little-endian
#
# Entries (columnar, in native slot order)
# Destination hash keys, followed by the _last, _blocked,
# _viol, _ts_len, _ts_head and _ts columns, all encoded as
# little-endian fixed-width values.
#
# Caller must hold the announce rate table lock
PERSIST_MAGIC = b " RNSART "
PERSIST_VERSION = 0x01
def persist ( self , path , retain = None ) :
try :
count = len ( self . _keys )
if count > 0 :
keylen = len ( self . _keys [ 0 ] )
for key in self . _keys :
if len ( key ) != keylen : raise ValueError ( " Inconsistent destination hash lengths in table " )
else :
keylen = RNS . Identity . TRUNCATED_HASHLENGTH / / 8
if retain != None and count > 0 :
now = time . time ( )
horizon = now - retain
keep = [ slot for slot in range ( count ) if not ( self . _last [ slot ] < horizon and self . _blocked [ slot ] < = now ) ]
else :
keep = None
if keep != None and len ( keep ) != count :
count = len ( keep )
keys = [ self . _keys [ slot ] for slot in keep ]
last = array ( " d " , [ self . _last [ slot ] for slot in keep ] )
blocked = array ( " d " , [ self . _blocked [ slot ] for slot in keep ] )
viol = array ( " i " , [ self . _viol [ slot ] for slot in keep ] )
ts_len = array ( " B " , [ self . _ts_len [ slot ] for slot in keep ] )
ts_head = array ( " B " , [ self . _ts_head [ slot ] for slot in keep ] )
ts = array ( " d " )
for slot in keep : ts . extend ( self . _ts [ slot * self . max_ts : ( slot + 1 ) * self . max_ts ] )
else :
keys = self . _keys
last = self . _last
blocked = self . _blocked
viol = self . _viol
ts_len = self . _ts_len
ts_head = self . _ts_head
ts = self . _ts
if last . itemsize != 8 or viol . itemsize != 4 :
raise ValueError ( f " Unsupported array itemsize at persist: d= { last . itemsize } , i= { viol . itemsize } " )
header = self . PERSIST_MAGIC
header + = bytes ( [ self . PERSIST_VERSION , 0x00 , self . max_ts & 0xFF , keylen & 0xFF ] )
header + = struct . pack ( " <H " , 0 )
header + = struct . pack ( " <I " , count )
blob = header
blob + = b " " . join ( keys )
blob + = self . _column_le ( last )
blob + = self . _column_le ( blocked )
blob + = self . _column_le ( viol )
blob + = ts_len . tobytes ( )
blob + = ts_head . tobytes ( )
blob + = self . _column_le ( ts )
tmp_path = path + " .tmp "
with open ( tmp_path , " wb " ) as file : file . write ( blob )
os . replace ( tmp_path , path )
return True
except Exception as e :
RNS . log ( f " Could not persist announce rate table to { path } : { e } " , RNS . LOG_ERROR )
return False
# Caller must hold the announce rate table lock
def load ( self , path ) :
try :
with open ( path , " rb " ) as file : blob = file . read ( )
except Exception : return False
try :
if len ( blob ) < 16 : return False
if blob [ 0 : 6 ] != self . PERSIST_MAGIC : return False
if blob [ 6 ] != self . PERSIST_VERSION : return False
if blob [ 7 ] != 0x00 : return False
file_max_ts = blob [ 8 ]
keylen = blob [ 9 ]
count = struct . unpack_from ( " <I " , blob , 12 ) [ 0 ]
if file_max_ts < 1 or file_max_ts > 255 : return False
if keylen < 1 : return False
entry_size = keylen + 22 + file_max_ts * 8
if len ( blob ) != 16 + count * entry_size : return False
if file_max_ts == self . max_ts :
# Ring size matches, load directly
keys = [ blob [ 16 + slot * keylen : 16 + ( slot + 1 ) * keylen ] for slot in range ( count ) ]
index = { }
for slot in range ( count ) :
if keys [ slot ] in index : return False
index [ keys [ slot ] ] = slot
offset = 16 + count * keylen
last = self . _column_from_le ( " d " , blob [ offset : offset + count * 8 ] ) ; offset + = count * 8
blocked = self . _column_from_le ( " d " , blob [ offset : offset + count * 8 ] ) ; offset + = count * 8
viol = self . _column_from_le ( " i " , blob [ offset : offset + count * 4 ] ) ; offset + = count * 4
ts_len = array ( " B " , blob [ offset : offset + count ] ) ; offset + = count
ts_head = array ( " B " , blob [ offset : offset + count ] ) ; offset + = count
ts = self . _column_from_le ( " d " , blob [ offset : offset + count * self . max_ts * 8 ] ) ; offset + = count * self . max_ts * 8
for slot in range ( count ) :
if ts_len [ slot ] > self . max_ts or ts_head [ slot ] > = self . max_ts : return False
else :
# Different ring sizes from current running config,
# rebuild entries best effort.
keys = [ blob [ 16 + slot * keylen : 16 + ( slot + 1 ) * keylen ] for slot in range ( count ) ]
index = { }
for slot in range ( count ) :
if keys [ slot ] in index : return False
index [ keys [ slot ] ] = slot
offset = 16 + count * keylen
last = self . _column_from_le ( " d " , blob [ offset : offset + count * 8 ] ) ; offset + = count * 8
blocked = self . _column_from_le ( " d " , blob [ offset : offset + count * 8 ] ) ; offset + = count * 8
viol = self . _column_from_le ( " i " , blob [ offset : offset + count * 4 ] ) ; offset + = count * 4
f_ts_len = array ( " B " , blob [ offset : offset + count ] ) ; offset + = count
f_ts_head = array ( " B " , blob [ offset : offset + count ] ) ; offset + = count
f_ts = self . _column_from_le ( " d " , blob [ offset : offset + count * file_max_ts * 8 ] ) ; offset + = count * file_max_ts * 8
for slot in range ( count ) :
if f_ts_len [ slot ] > file_max_ts or f_ts_head [ slot ] > = file_max_ts : return False
ts_len = array ( " B " ) ; ts_head = array ( " B " ) ; ts = array ( " d " )
for slot in range ( count ) :
base = slot * file_max_ts
if f_ts_len [ slot ] < file_max_ts : chronological = list ( f_ts [ base : base + f_ts_len [ slot ] ] )
else : chronological = list ( f_ts [ base + f_ts_head [ slot ] : base + file_max_ts ] ) + list ( f_ts [ base : base + f_ts_head [ slot ] ] )
chronological = chronological [ - self . max_ts : ]
ts_count = len ( chronological )
ts_len . append ( ts_count )
ts . extend ( chronological )
if ts_count < self . max_ts : ts . extend ( [ 0.0 ] * ( self . max_ts - ts_count ) )
ts_head . append ( ts_count % self . max_ts )
self . _index = index
self . _keys = keys
self . _last = last
self . _blocked = blocked
self . _viol = viol
self . _ts = ts
self . _ts_len = ts_len
self . _ts_head = ts_head
self . _cursor = 0
return True
except Exception as e :
RNS . log ( f " Could not load announce rate table from { path } : { e } " , RNS . LOG_ERROR )
return False
def _column_le ( self , column ) :
if sys . byteorder == " big " : swapped = array ( column . typecode , column ) ; swapped . byteswap ( ) ; return swapped . tobytes ( )
else : return column . tobytes ( )
def _column_from_le ( self , typecode , data ) :
column = array ( typecode ) ; column . frombytes ( data )
if sys . byteorder == " big " : column . byteswap ( )
return column
2026-10-24 00:20:27 +02:00
class PacketHashlist :
# Bucketed byte-blob packet filter hashlist, with the primary
# design goal of memory coherence and density. All operations
# are safe to call without holding a lock.
def __init__ ( self , hash_width = 32 , bucket_count = 16384 ) :
assert bucket_count > 0 and bucket_count & ( bucket_count - 1 ) == 0 , " Bucket count must be a power of two "
self . width = hash_width
self . _nb = bucket_count
self . _mask = bucket_count - 1
self . _cur = [ bytearray ( ) for _ in range ( bucket_count ) ]
self . _prv = [ bytearray ( ) for _ in range ( bucket_count ) ]
self . _count = 0
def __len__ ( self ) :
return self . _count
def __contains__ ( self , packet_hash ) :
bucket = ( packet_hash [ 0 ] | ( packet_hash [ 1 ] << 8 ) ) & self . _mask
return packet_hash in self . _cur [ bucket ] or packet_hash in self . _prv [ bucket ]
def add ( self , packet_hash ) :
assert len ( packet_hash ) == self . width , " Packet hash width mismatch "
bucket = ( packet_hash [ 0 ] | ( packet_hash [ 1 ] << 8 ) ) & self . _mask
if packet_hash not in self . _cur [ bucket ] :
self . _cur [ bucket ] . extend ( packet_hash )
self . _count + = 1
def discard ( self , packet_hash ) :
w = self . width
bucket = ( packet_hash [ 0 ] | ( packet_hash [ 1 ] << 8 ) ) & self . _mask
for generation in ( self . _cur , self . _prv ) :
entries = generation [ bucket ]
pos = entries . find ( packet_hash )
while pos != - 1 :
if pos % w == 0 :
del entries [ pos : pos + w ]
if generation is self . _cur : self . _count - = 1
pos = entries . find ( packet_hash , pos )
else :
pos = entries . find ( packet_hash , pos + 1 )
def swap ( self ) :
self . _prv = self . _cur
self . _cur = [ bytearray ( ) for _ in range ( self . _nb ) ]
self . _count = 0
# Current-generation buckets for persist operations
def buckets ( self ) : return tuple ( self . _cur )
class KnownDestinationsStore :
# Write-behind SQLite storage backend for known destinations.
# The actual in-memory table is used directly for all transport
# operations; this serves purely as the store, and no read operations
# utilize the database directly.
def __init__ ( self , db_path , hash_width = 16 , read_only = False ) :
import sqlite3
2026-10-31 16:27:00 +01:00
self . sqlite3 = sqlite3
self . db_path = db_path
self . hash_width = hash_width
self . read_only = read_only
self . _dirty_upsert = set ( )
self . _dirty_delete = set ( )
self . _ratchet_upserts = { }
self . _ratchet_deletes = set ( )
self . _kd_quarantine = { }
self . _ratchet_quarantine = { }
self . _flush_lock = threading . Lock ( )
2026-10-24 00:20:27 +02:00
2026-11-03 21:50:20 +01:00
if read_only : self . _conn = sqlite3 . connect ( f " file: { db_path } ?mode=ro " , uri = True , check_same_thread = False , isolation_level = None , cached_statements = 0 )
2026-10-24 00:20:27 +02:00
else :
2026-11-03 21:50:20 +01:00
self . _conn = sqlite3 . connect ( db_path , check_same_thread = False , isolation_level = None , cached_statements = 0 )
2026-10-24 00:20:27 +02:00
self . _conn . execute ( " PRAGMA journal_mode=WAL " )
self . _conn . execute ( " PRAGMA synchronous=NORMAL " )
self . _conn . execute ( " PRAGMA journal_size_limit=8388608 " )
self . __create_schema ( )
def close ( self ) :
if self . _conn != None :
self . _conn . close ( )
self . _conn = None
def __create_schema ( self ) :
2026-10-27 14:17:25 +01:00
db_version = self . _conn . execute ( " PRAGMA user_version " ) . fetchone ( ) [ 0 ]
2026-10-31 17:00:26 +01:00
# Pre-release database cleanup
# TODO: Remove this in 1.6.1
if db_version > 0 :
RNS . log ( f " Pre-release database detected, rebuilding known destinations DB to release format " , RNS . LOG_WARNING )
self . _conn . execute ( " DROP TABLE IF EXISTS known_destinations " )
2026-10-24 00:20:27 +02:00
self . _conn . execute ( """ CREATE TABLE IF NOT EXISTS known_destinations (
destination_hash BLOB PRIMARY KEY ,
last_heard REAL NOT NULL ,
2026-10-31 12:12:58 +01:00
packet_hash BLOB ,
2026-10-24 00:20:27 +02:00
public_key BLOB NOT NULL ,
app_data BLOB ,
used REAL NOT NULL DEFAULT 0 ,
stamp BLOB ,
2026-10-30 00:52:50 +01:00
stamp_value REAL ,
lr_cost REAL ,
2026-10-24 00:20:27 +02:00
identity_hash BLOB NOT NULL ) """ )
self . _conn . execute ( """ CREATE TABLE IF NOT EXISTS ratchets (
destination_hash BLOB PRIMARY KEY ,
ratchet BLOB NOT NULL ,
received REAL NOT NULL ) """ )
2026-10-31 17:00:26 +01:00
# Schema migrations
2026-10-31 13:46:30 +01:00
try :
2026-10-31 13:52:27 +01:00
self . _conn . execute ( " BEGIN " )
2026-10-31 17:00:26 +01:00
# No migrations yet
self . _conn . execute ( " PRAGMA user_version = 0 " )
2026-10-31 13:52:27 +01:00
self . _conn . execute ( " COMMIT " )
2026-10-31 13:46:30 +01:00
self . _conn . commit ( )
2026-10-31 16:27:00 +01:00
2026-10-31 13:46:30 +01:00
except Exception as e :
2026-10-31 17:00:26 +01:00
RNS . log ( f " Known destinations database version migration failed: { e } " , RNS . LOG_CRITICAL )
RNS . log ( f " You will need to remove the known_destinations.db* files from your storage directory so it can be rebuilt " , RNS . LOG_CRITICAL )
2026-10-24 00:20:27 +02:00
def __entry_row ( self , destination_hash , entry ) :
stamp = entry [ 5 ] if len ( entry ) > 5 else None
stamp_value = entry [ 6 ] if len ( entry ) > 6 else 0
2026-10-27 14:17:25 +01:00
lr_cost = entry [ 7 ] if len ( entry ) > 7 else None
2026-10-24 00:20:27 +02:00
identity_hash = hashlib . sha256 ( entry [ 2 ] ) . digest ( ) [ : self . hash_width ]
2026-10-27 14:17:25 +01:00
return ( destination_hash , entry [ 0 ] , entry [ 1 ] , entry [ 2 ] , entry [ 3 ] , entry [ 4 ] , stamp , stamp_value , lr_cost , identity_hash )
2026-10-24 00:20:27 +02:00
# Caller must hold Identity.known_destinations_lock
def mark_dirty ( self , destination_hash ) :
if self . read_only : return
self . _dirty_delete . discard ( destination_hash )
2026-10-31 16:27:00 +01:00
self . _kd_quarantine . pop ( destination_hash , None )
2026-10-24 00:20:27 +02:00
self . _dirty_upsert . add ( destination_hash )
# Caller must hold Identity.known_destinations_lock
def mark_deleted ( self , destination_hash ) :
if self . read_only : return
self . _dirty_upsert . discard ( destination_hash )
self . _dirty_delete . add ( destination_hash )
2026-10-31 16:27:00 +01:00
self . _kd_quarantine . pop ( destination_hash , None )
self . _ratchet_quarantine . pop ( destination_hash , None )
2026-10-24 00:20:27 +02:00
# Caller must hold Identity.known_destinations_lock
def take_pending ( self ) :
if self . read_only : return set ( ) , set ( ) , { } , set ( )
upserts = self . _dirty_upsert
deletes = self . _dirty_delete
ratchet_upserts = self . _ratchet_upserts
ratchet_deletes = self . _ratchet_deletes
self . _dirty_upsert = set ( )
self . _dirty_delete = set ( )
self . _ratchet_upserts = { }
self . _ratchet_deletes = set ( )
return upserts , deletes , ratchet_upserts , ratchet_deletes
2026-10-31 16:27:00 +01:00
# Caller must hold the flush lock and have an already open transaction
def __wrap_exec ( self , sql , rows ) :
rejects = [ ]
# By default, run batched queries in chunks.
for i in range ( 0 , len ( rows ) , 4096 ) :
chunk = rows [ i : i + 4096 ]
try :
self . _conn . execute ( " SAVEPOINT chunk " )
self . _conn . executemany ( sql , chunk )
self . _conn . execute ( " RELEASE chunk " )
# But on integrity and data errors, roll back the chunk
# and iterate per-row, so we can catch any violating
# entries and quarantine them. A later, sane update
# will then unquarantine and heal upon arrival.
#
# Other exceptions are intentionally not caught here,
# so they can propagate directly.
except ( self . sqlite3 . IntegrityError , self . sqlite3 . DataError ) :
self . _conn . execute ( " ROLLBACK TO chunk " )
for row in chunk :
try :
self . _conn . execute ( " SAVEPOINT row " )
self . _conn . execute ( sql , row )
self . _conn . execute ( " RELEASE row " )
except ( self . sqlite3 . IntegrityError , self . sqlite3 . DataError ) as e :
self . _conn . execute ( " ROLLBACK TO row " )
rejects . append ( ( row [ 0 ] , str ( e ) ) )
self . _conn . execute ( " RELEASE chunk " )
return rejects
2026-10-24 00:20:27 +02:00
# May be called without any locks held
def apply ( self , upserts , deletes , ratchet_upserts , ratchet_deletes , known_destinations ) :
if self . read_only : return 0
if len ( upserts ) == 0 and len ( deletes ) == 0 and len ( ratchet_upserts ) == 0 and len ( ratchet_deletes ) == 0 : return 0
with self . _flush_lock :
rows = [ ]
removables = [ ]
ratchet_rows = [ ]
ratchet_removables = [ ]
for destination_hash in upserts :
2026-10-31 16:27:00 +01:00
if destination_hash in self . _kd_quarantine : continue
2026-10-24 00:20:27 +02:00
entry = known_destinations . get ( destination_hash )
if entry != None : rows . append ( self . __entry_row ( destination_hash , entry ) )
for destination_hash in deletes :
if destination_hash not in known_destinations : removables . append ( ( destination_hash , ) )
for destination_hash , ratchet_entry in ratchet_upserts . items ( ) :
# Ratchets of purged destinations must be removed as well
2026-10-31 16:27:00 +01:00
if destination_hash not in known_destinations : ratchet_removables . append ( ( destination_hash , ) )
elif destination_hash not in self . _ratchet_quarantine : ratchet_rows . append ( ( destination_hash , * ratchet_entry ) )
2026-10-24 00:20:27 +02:00
for destination_hash in ratchet_deletes :
if destination_hash not in known_destinations : ratchet_removables . append ( ( destination_hash , ) )
2026-10-31 16:27:00 +01:00
kd_rejects = [ ]
ratchet_rejects = [ ]
2026-10-24 00:20:27 +02:00
try :
self . _conn . execute ( " BEGIN " )
2026-10-31 16:27:00 +01:00
kd_rejects + = self . __wrap_exec ( " INSERT INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, lr_cost=excluded.lr_cost, identity_hash=excluded.identity_hash " , rows )
kd_rejects + = self . __wrap_exec ( " DELETE FROM known_destinations WHERE destination_hash=? " , removables )
ratchet_rejects + = self . __wrap_exec ( " INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received " , ratchet_rows )
ratchet_rejects + = self . __wrap_exec ( " DELETE FROM ratchets WHERE destination_hash=? " , ratchet_removables )
2026-10-24 00:20:27 +02:00
self . _conn . commit ( )
except Exception :
2026-10-31 16:27:00 +01:00
# If an exception is not handled in the batch query wrapper,
# we will have to roll back the transaction, and mark all
# attempted entries as dirty. A failure state like such
# should not happen, and this is a last resort; at this
# point, writes may stall until an updated entry for
# the violating row(s) arrive and heal the invalid
# dirty entries, so this is raised for logging.
2026-10-24 00:20:27 +02:00
self . _dirty_upsert | = { row [ 0 ] for row in rows }
self . _dirty_delete | = { row [ 0 ] for row in removables }
self . _ratchet_upserts | = { row [ 0 ] : row [ 1 : ] for row in ratchet_rows }
self . _ratchet_deletes | = { row [ 0 ] for row in ratchet_removables }
self . _conn . rollback ( )
raise
2026-10-31 16:27:00 +01:00
# Track failed persists and one-time log per failure. A later
# update of the destination clears the quarantine state and
# allows persist to retry with the (hopefully) valid data.
if len ( kd_rejects ) > 0 or len ( ratchet_rejects ) > 0 :
for destination_hash , error in kd_rejects :
self . _kd_quarantine [ destination_hash ] = ( error , time . time ( ) )
RNS . log ( f " Invalid known destination data for { RNS . prettyhexrep ( destination_hash ) } at persist. The contained exception was: { error } " , RNS . LOG_ERROR )
for destination_hash , error in ratchet_rejects :
self . _ratchet_quarantine [ destination_hash ] = ( error , time . time ( ) )
RNS . log ( f " Invalid ratchet data for { RNS . prettyhexrep ( destination_hash ) } at persist. The contained exception was: { error } " , RNS . LOG_ERROR )
return len ( rows ) + len ( removables ) + len ( ratchet_rows ) + len ( ratchet_removables ) - len ( kd_rejects ) - len ( ratchet_rejects )
2026-10-24 00:20:27 +02:00
def load_all ( self ) :
2026-10-27 14:17:25 +01:00
cursor = self . _conn . execute ( " SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost FROM known_destinations " )
2026-10-24 00:20:27 +02:00
while True :
rows = cursor . fetchmany ( 4096 )
if not rows : break
2026-10-27 14:17:25 +01:00
for row in rows : yield row [ 0 ] , [ row [ 1 ] , row [ 2 ] , row [ 3 ] , row [ 4 ] , row [ 5 ] , row [ 6 ] , row [ 7 ] , row [ 8 ] ]
2026-10-24 00:20:27 +02:00
def remember_ratchet ( self , destination_hash , ratchet , now ) :
if self . read_only : return False
2026-10-31 16:27:00 +01:00
self . _ratchet_quarantine . pop ( destination_hash , None )
2026-10-24 00:20:27 +02:00
pending = self . _ratchet_upserts . get ( destination_hash )
if pending != None :
if pending [ 0 ] == ratchet :
self . _ratchet_deletes . discard ( destination_hash )
return False
else :
2026-11-03 21:50:20 +01:00
with self . _flush_lock :
row = self . _conn . execute ( " SELECT ratchet FROM ratchets WHERE destination_hash=? " , ( destination_hash , ) ) . fetchone ( )
2026-10-24 00:20:27 +02:00
if row != None and row [ 0 ] == ratchet :
# Discard any pending delete mark queued
# before this ratchet arrived
self . _ratchet_deletes . discard ( destination_hash )
return False
self . _ratchet_upserts [ destination_hash ] = ( ratchet , now )
self . _ratchet_deletes . discard ( destination_hash )
return True
def fetch_ratchet ( self , destination_hash ) :
pending = self . _ratchet_upserts . get ( destination_hash )
if pending != None : return pending [ 0 ] , pending [ 1 ]
if destination_hash in self . _ratchet_deletes : return None
2026-11-03 21:50:20 +01:00
with self . _flush_lock :
row = self . _conn . execute ( " SELECT ratchet, received FROM ratchets WHERE destination_hash=? " , ( destination_hash , ) ) . fetchone ( )
2026-10-24 00:20:27 +02:00
if row != None : return row [ 0 ] , row [ 1 ]
return None
# Caller must hold Identity.known_destinations_lock
def mark_ratchet_deleted ( self , destination_hash ) :
if self . read_only : return
self . _ratchet_upserts . pop ( destination_hash , None )
self . _ratchet_deletes . add ( destination_hash )
2026-10-31 16:27:00 +01:00
self . _ratchet_quarantine . pop ( destination_hash , None )
2026-10-24 00:20:27 +02:00
def expire_ratchets ( self , cutoff ) :
if self . read_only : return [ ]
with self . _flush_lock :
# Drop expired entries in pending as well
expired = set ( row [ 0 ] for row in self . _conn . execute ( " SELECT destination_hash FROM ratchets WHERE received<? " , ( cutoff , ) ) )
for destination_hash , entry in list ( self . _ratchet_upserts . items ( ) ) :
if entry [ 1 ] < cutoff :
expired . add ( destination_hash )
self . _ratchet_upserts . pop ( destination_hash , None )
expired . difference_update ( self . _ratchet_deletes )
if len ( expired ) > 0 :
self . _conn . execute ( " BEGIN " )
for i in range ( 0 , len ( expired ) , 4096 ) :
self . _conn . executemany ( " DELETE FROM ratchets WHERE destination_hash=? " , [ ( h , ) for h in list ( expired ) [ i : i + 4096 ] ] )
self . _conn . commit ( )
return list ( expired )
def load_ratchets ( self ) :
cursor = self . _conn . execute ( " SELECT destination_hash, ratchet, received FROM ratchets " )
while True :
rows = cursor . fetchmany ( 4096 )
if not rows : break
for row in rows : yield row [ 0 ] , row [ 1 ] , row [ 2 ]
def import_legacy_ratchets ( self , dir_path , expiry_cutoff , known_destinations , ratchet_width = 32 ) :
from . vendor import umsgpack
imported = 0
for filename in os . listdir ( dir_path ) :
file_path = os . path . join ( dir_path , filename )
try :
if not os . path . isfile ( file_path ) : continue
with open ( file_path , " rb " ) as rf :
ratchet_data = umsgpack . unpackb ( rf . read ( ) )
destination_hash = bytes . fromhex ( filename )
ratchet = ratchet_data [ " ratchet " ] ; received = ratchet_data [ " received " ]
if len ( ratchet ) != ratchet_width : raise ValueError ( " Invalid ratchet width " )
if received < expiry_cutoff : raise ValueError ( " Ratchet expired " )
if destination_hash not in known_destinations : raise ValueError ( " Destination unknown " )
self . _ratchet_upserts [ destination_hash ] = ( ratchet , received )
imported + = 1
os . unlink ( file_path )
except Exception as e :
RNS . log ( f " Could not import ratchet for { filename } : { e } " , RNS . LOG_WARNING )
try : os . unlink ( file_path )
except : pass
pending = self . _ratchet_upserts
self . _ratchet_upserts = { }
self . apply ( set ( ) , set ( ) , pending , set ( ) , known_destinations )
return imported
def import_legacy ( self , path ) :
from . vendor import umsgpack
with open ( path , " rb " ) as file : legacy = umsgpack . load ( file )
rows = [ ]
now = time . time ( )
for destination_hash , entry in legacy . items ( ) :
if len ( entry ) < 5 : continue
entry = list ( entry )
2026-10-27 14:17:25 +01:00
while len ( entry ) < 8 : entry . append ( None if len ( entry ) in ( 5 , 7 ) else 0 )
2026-10-24 00:20:27 +02:00
if entry [ 0 ] == None or entry [ 0 ] < = 0 : entry [ 0 ] = now
rows . append ( self . __entry_row ( destination_hash , entry ) )
self . _conn . execute ( " BEGIN " )
2026-10-27 14:17:25 +01:00
for i in range ( 0 , len ( rows ) , 4096 ) : self . _conn . executemany ( " INSERT OR REPLACE INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) " , rows [ i : i + 4096 ] )
2026-10-24 00:20:27 +02:00
self . _conn . commit ( )
return len ( rows )
class AnnounceCache :
def __init__ ( self , db_path ) :
import sqlite3
self . db_path = db_path
self . _pending = { }
self . _flush_lock = threading . Lock ( )
2026-11-03 21:50:20 +01:00
self . _conn = sqlite3 . connect ( db_path , check_same_thread = False , isolation_level = None , cached_statements = 0 )
2026-10-24 00:20:27 +02:00
2026-11-03 21:29:29 +01:00
self . _conn . execute ( " PRAGMA auto_vacuum=INCREMENTAL " )
2026-10-24 00:20:27 +02:00
self . _conn . execute ( " PRAGMA journal_mode=WAL " )
self . _conn . execute ( " PRAGMA synchronous=NORMAL " )
self . _conn . execute ( " PRAGMA journal_size_limit=8388608 " )
self . __create_schema ( )
def __create_schema ( self ) :
self . _conn . execute ( """ CREATE TABLE IF NOT EXISTS cached_announces (
packet_hash BLOB PRIMARY KEY ,
received REAL NOT NULL ,
raw BLOB NOT NULL ,
interface_reference TEXT ) """ )
self . _conn . execute ( " PRAGMA user_version = 1 " )
def store ( self , packet_hash , raw , interface_reference ) :
self . _pending [ packet_hash ] = ( time . time ( ) , raw , interface_reference )
def pending_count ( self ) :
return len ( self . _pending )
def fetch ( self , packet_hash ) :
pending = self . _pending . get ( packet_hash )
if pending != None : return pending [ 1 ] , pending [ 2 ]
2026-11-03 21:50:20 +01:00
with self . _flush_lock :
row = self . _conn . execute ( " SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=? " , ( packet_hash , ) ) . fetchone ( )
2026-10-24 00:20:27 +02:00
if row != None : return row [ 0 ] , row [ 1 ]
return None
def flush ( self ) :
with self . _flush_lock : return self . __apply_pending ( )
def __apply_pending ( self ) :
pending = self . _pending
self . _pending = { }
if len ( pending ) == 0 : return 0
try :
self . _conn . execute ( " BEGIN " )
items = list ( pending . items ( ) )
for i in range ( 0 , len ( items ) , 4096 ) :
self . _conn . executemany ( " INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference " ,
[ ( packet_hash , * entry ) for packet_hash , entry in items [ i : i + 4096 ] ] )
self . _conn . commit ( )
except Exception :
# Re-mark everything as pending, so the changes
# are retried on the next flush, then propagate.
self . _pending . update ( pending )
self . _conn . rollback ( )
raise
return len ( pending )
def clean ( self , active_hashes ) :
with self . _flush_lock :
self . __apply_pending ( )
try :
self . _conn . execute ( " BEGIN " )
self . _conn . execute ( " DROP TABLE IF EXISTS temp.active_announces " )
self . _conn . execute ( " CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY) " )
active_list = list ( active_hashes )
for i in range ( 0 , len ( active_list ) , 4096 ) :
self . _conn . executemany ( " INSERT OR IGNORE INTO active_announces VALUES (?) " , [ ( h , ) for h in active_list [ i : i + 4096 ] ] )
changes_before = self . _conn . total_changes
self . _conn . execute ( " DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces) " )
removed = self . _conn . total_changes - changes_before
self . _conn . commit ( )
except Exception :
self . _conn . rollback ( )
raise
if removed > 0 : self . _conn . execute ( " PRAGMA incremental_vacuum " )
return removed
def import_legacy ( self , dir_path ) :
from . vendor import umsgpack
imported = 0
for filename in os . listdir ( dir_path ) :
file_path = os . path . join ( dir_path , filename )
try :
if not os . path . isfile ( file_path ) : continue
with open ( file_path , " rb " ) as file :
cached_data = umsgpack . unpackb ( file . read ( ) )
packet_hash = bytes . fromhex ( filename )
interface_reference = cached_data [ 1 ] if len ( cached_data ) > 1 else None
self . store ( packet_hash , cached_data [ 0 ] , interface_reference )
imported + = 1
os . unlink ( file_path )
except Exception :
try : os . unlink ( file_path )
except Exception : pass
self . flush ( )
return imported
def close ( self ) :
if self . _conn != None :
self . _conn . close ( )
self . _conn = None