2026-10-24 00:20:27 +02:00
# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
from array import array
import hashlib
import threading
import os
import time
class AnnounceRateTable :
# Columnar store for announce rate tracking. Layout:
# _index : dict[destination_hash -> slot]
# _keys : list[slot] -> destination_hash (for swap-remove bookkeeping)
# _last : array('d') last non-violating announce time
# _blocked: array('d') blocked_until
# _viol : array('i') rate_violations
# _ts : array('d') flat ring of max_timestamps doubles per slot
# _ts_len : array('B') valid timestamp count per slot
# _ts_head: array('B') ring write cursor per slot
def __init__ ( self , max_timestamps = 16 ) :
assert max_timestamps < = 255 , " Timestamp ring size must fit in a single byte "
self . max_ts = max_timestamps
self . _index = { }
self . _keys = [ ]
self . _last = array ( ' d ' )
self . _blocked = array ( ' d ' )
self . _viol = array ( ' i ' )
self . _ts = array ( ' d ' )
self . _ts_len = array ( ' B ' )
self . _ts_head = array ( ' B ' )
self . _cursor = 0
def __len__ ( self ) : return len ( self . _keys )
def __contains__ ( self , destination_hash ) : return destination_hash in self . _index
# Perform the announce-rate check-and-update for one announce.
# Caller must hold the announce_rate_table_lock.
def check_and_update ( self , destination_hash , now , target , grace , penalty ) :
slot = self . _index . get ( destination_hash )
if slot is None :
slot = len ( self . _keys )
self . _index [ destination_hash ] = slot
self . _keys . append ( destination_hash )
self . _last . append ( now )
self . _blocked . append ( 0.0 )
self . _viol . append ( 0 )
self . _ts . extend ( [ 0.0 ] * self . max_ts )
self . _ts [ slot * self . max_ts ] = now
self . _ts_len . append ( 1 )
self . _ts_head . append ( 1 % self . max_ts )
return False
base = slot * self . max_ts
head = self . _ts_head [ slot ]
self . _ts [ base + head ] = now
self . _ts_head [ slot ] = ( head + 1 ) % self . max_ts
if self . _ts_len [ slot ] < self . max_ts : self . _ts_len [ slot ] + = 1
current_rate = now - self . _last [ slot ]
rate_blocked = False
if now > self . _blocked [ slot ] :
if current_rate < target : self . _viol [ slot ] + = 1
else :
v = self . _viol [ slot ] - 1
self . _viol [ slot ] = v if v > 0 else 0
if self . _viol [ slot ] > grace :
self . _blocked [ slot ] = self . _last [ slot ] + target + penalty
rate_blocked = True
else : self . _last [ slot ] = now
else : rate_blocked = True
return rate_blocked
# Caller must hold the table lock
def cull ( self , now , ttl , max_scan = 1024 , max_cull = None ) :
if max_cull is None : max_cull = max_scan
horizon = now - ttl
culled = 0
scanned = 0
n = len ( self . _keys )
while scanned < max_scan and culled < max_cull and n > 0 :
if self . _cursor > = n : self . _cursor = 0
slot = self . _cursor
if self . _last [ slot ] < horizon and self . _blocked [ slot ] < = now :
last_i = n - 1
stale_key = self . _keys [ slot ]
if slot != last_i :
moved_key = self . _keys [ last_i ]
self . _keys [ slot ] = moved_key
self . _index [ moved_key ] = slot
self . _last [ slot ] = self . _last [ last_i ]
self . _blocked [ slot ] = self . _blocked [ last_i ]
self . _viol [ slot ] = self . _viol [ last_i ]
self . _ts_len [ slot ] = self . _ts_len [ last_i ]
self . _ts_head [ slot ] = self . _ts_head [ last_i ]
f = last_i * self . max_ts ; t = slot * self . max_ts
self . _ts [ t : t + self . max_ts ] = self . _ts [ f : f + self . max_ts ]
self . _keys . pop ( )
self . _last . pop ( ) ; self . _blocked . pop ( ) ; self . _viol . pop ( )
self . _ts_len . pop ( ) ; self . _ts_head . pop ( )
del self . _ts [ len ( self . _ts ) - self . max_ts : ] # drop tail block of removed slot
del self . _index [ stale_key ]
n - = 1
culled + = 1
# Do not advance cursor, re-examine the slot a moved entry landed in
else :
self . _cursor = slot + 1
scanned + = 1
if self . _cursor > = n : self . _cursor = 0
return culled
# Hold table lock, or accept a possibly inconsistent snapshot
def items ( self ) :
for slot , key in enumerate ( self . _keys ) :
n = self . _ts_len [ slot ]
head = self . _ts_head [ slot ]
base = slot * self . max_ts
if n < self . max_ts : ts = list ( self . _ts [ base : base + n ] )
else : ts = list ( self . _ts [ base + head : base + self . max_ts ] ) + list ( self . _ts [ base : base + head ] )
yield key , { " last " : self . _last [ slot ] , " rate_violations " : self . _viol [ slot ] ,
" blocked_until " : self . _blocked [ slot ] , " timestamps " : ts }
class PacketHashlist :
# Bucketed byte-blob packet filter hashlist, with the primary
# design goal of memory coherence and density. All operations
# are safe to call without holding a lock.
def __init__ ( self , hash_width = 32 , bucket_count = 16384 ) :
assert bucket_count > 0 and bucket_count & ( bucket_count - 1 ) == 0 , " Bucket count must be a power of two "
self . width = hash_width
self . _nb = bucket_count
self . _mask = bucket_count - 1
self . _cur = [ bytearray ( ) for _ in range ( bucket_count ) ]
self . _prv = [ bytearray ( ) for _ in range ( bucket_count ) ]
self . _count = 0
def __len__ ( self ) :
return self . _count
def __contains__ ( self , packet_hash ) :
bucket = ( packet_hash [ 0 ] | ( packet_hash [ 1 ] << 8 ) ) & self . _mask
return packet_hash in self . _cur [ bucket ] or packet_hash in self . _prv [ bucket ]
def add ( self , packet_hash ) :
assert len ( packet_hash ) == self . width , " Packet hash width mismatch "
bucket = ( packet_hash [ 0 ] | ( packet_hash [ 1 ] << 8 ) ) & self . _mask
if packet_hash not in self . _cur [ bucket ] :
self . _cur [ bucket ] . extend ( packet_hash )
self . _count + = 1
def discard ( self , packet_hash ) :
w = self . width
bucket = ( packet_hash [ 0 ] | ( packet_hash [ 1 ] << 8 ) ) & self . _mask
for generation in ( self . _cur , self . _prv ) :
entries = generation [ bucket ]
pos = entries . find ( packet_hash )
while pos != - 1 :
if pos % w == 0 :
del entries [ pos : pos + w ]
if generation is self . _cur : self . _count - = 1
pos = entries . find ( packet_hash , pos )
else :
pos = entries . find ( packet_hash , pos + 1 )
def swap ( self ) :
self . _prv = self . _cur
self . _cur = [ bytearray ( ) for _ in range ( self . _nb ) ]
self . _count = 0
# Current-generation buckets for persist operations
def buckets ( self ) : return tuple ( self . _cur )
class KnownDestinationsStore :
# Write-behind SQLite storage backend for known destinations.
# The actual in-memory table is used directly for all transport
# operations; this serves purely as the store, and no read operations
# utilize the database directly.
def __init__ ( self , db_path , hash_width = 16 , read_only = False ) :
import sqlite3
self . sqlite3 = sqlite3
self . db_path = db_path
self . hash_width = hash_width
self . read_only = read_only
self . _dirty_upsert = set ( )
self . _dirty_delete = set ( )
self . _ratchet_upserts = { }
self . _ratchet_deletes = set ( )
self . _flush_lock = threading . Lock ( )
if read_only : self . _conn = sqlite3 . connect ( f " file: { db_path } ?mode=ro " , uri = True , check_same_thread = False , isolation_level = None )
else :
self . _conn = sqlite3 . connect ( db_path , check_same_thread = False , isolation_level = None )
self . _conn . execute ( " PRAGMA journal_mode=WAL " )
self . _conn . execute ( " PRAGMA synchronous=NORMAL " )
self . _conn . execute ( " PRAGMA journal_size_limit=8388608 " )
self . __create_schema ( )
def close ( self ) :
if self . _conn != None :
self . _conn . close ( )
self . _conn = None
def __create_schema ( self ) :
2026-10-27 14:17:25 +01:00
db_version = self . _conn . execute ( " PRAGMA user_version " ) . fetchone ( ) [ 0 ]
2026-10-24 00:20:27 +02:00
self . _conn . execute ( """ CREATE TABLE IF NOT EXISTS known_destinations (
destination_hash BLOB PRIMARY KEY ,
last_heard REAL NOT NULL ,
packet_hash BLOB NOT NULL ,
public_key BLOB NOT NULL ,
app_data BLOB ,
used REAL NOT NULL DEFAULT 0 ,
stamp BLOB ,
2026-10-30 00:52:50 +01:00
stamp_value REAL ,
lr_cost REAL ,
2026-10-24 00:20:27 +02:00
identity_hash BLOB NOT NULL ) """ )
self . _conn . execute ( """ CREATE TABLE IF NOT EXISTS ratchets (
destination_hash BLOB PRIMARY KEY ,
ratchet BLOB NOT NULL ,
received REAL NOT NULL ) """ )
2026-10-30 00:52:50 +01:00
if 0 < db_version < 2 : self . _conn . execute ( " ALTER TABLE known_destinations ADD COLUMN lr_cost REAL " )
2026-10-27 14:17:25 +01:00
self . _conn . execute ( " PRAGMA user_version = 2 " )
2026-10-24 00:20:27 +02:00
self . _conn . commit ( )
def __entry_row ( self , destination_hash , entry ) :
stamp = entry [ 5 ] if len ( entry ) > 5 else None
stamp_value = entry [ 6 ] if len ( entry ) > 6 else 0
2026-10-27 14:17:25 +01:00
lr_cost = entry [ 7 ] if len ( entry ) > 7 else None
2026-10-24 00:20:27 +02:00
identity_hash = hashlib . sha256 ( entry [ 2 ] ) . digest ( ) [ : self . hash_width ]
2026-10-27 14:17:25 +01:00
return ( destination_hash , entry [ 0 ] , entry [ 1 ] , entry [ 2 ] , entry [ 3 ] , entry [ 4 ] , stamp , stamp_value , lr_cost , identity_hash )
2026-10-24 00:20:27 +02:00
# Caller must hold Identity.known_destinations_lock
def mark_dirty ( self , destination_hash ) :
if self . read_only : return
self . _dirty_delete . discard ( destination_hash )
self . _dirty_upsert . add ( destination_hash )
# Caller must hold Identity.known_destinations_lock
def mark_deleted ( self , destination_hash ) :
if self . read_only : return
self . _dirty_upsert . discard ( destination_hash )
self . _dirty_delete . add ( destination_hash )
# Caller must hold Identity.known_destinations_lock
def take_pending ( self ) :
if self . read_only : return set ( ) , set ( ) , { } , set ( )
upserts = self . _dirty_upsert
deletes = self . _dirty_delete
ratchet_upserts = self . _ratchet_upserts
ratchet_deletes = self . _ratchet_deletes
self . _dirty_upsert = set ( )
self . _dirty_delete = set ( )
self . _ratchet_upserts = { }
self . _ratchet_deletes = set ( )
return upserts , deletes , ratchet_upserts , ratchet_deletes
# May be called without any locks held
def apply ( self , upserts , deletes , ratchet_upserts , ratchet_deletes , known_destinations ) :
if self . read_only : return 0
if len ( upserts ) == 0 and len ( deletes ) == 0 and len ( ratchet_upserts ) == 0 and len ( ratchet_deletes ) == 0 : return 0
with self . _flush_lock :
rows = [ ]
removables = [ ]
ratchet_rows = [ ]
ratchet_removables = [ ]
for destination_hash in upserts :
entry = known_destinations . get ( destination_hash )
if entry != None : rows . append ( self . __entry_row ( destination_hash , entry ) )
for destination_hash in deletes :
if destination_hash not in known_destinations : removables . append ( ( destination_hash , ) )
for destination_hash , ratchet_entry in ratchet_upserts . items ( ) :
# Ratchets of purged destinations must be removed as well
if destination_hash in known_destinations : ratchet_rows . append ( ( destination_hash , * ratchet_entry ) )
else : ratchet_removables . append ( ( destination_hash , ) )
for destination_hash in ratchet_deletes :
if destination_hash not in known_destinations : ratchet_removables . append ( ( destination_hash , ) )
try :
self . _conn . execute ( " BEGIN " )
2026-10-27 14:17:25 +01:00
for i in range ( 0 , len ( rows ) , 4096 ) : self . _conn . executemany ( " INSERT INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET last_heard=excluded.last_heard, packet_hash=excluded.packet_hash, public_key=excluded.public_key, app_data=excluded.app_data, used=excluded.used, stamp=excluded.stamp, stamp_value=excluded.stamp_value, lr_cost=excluded.lr_cost, identity_hash=excluded.identity_hash " , rows [ i : i + 4096 ] )
2026-10-24 00:20:27 +02:00
for i in range ( 0 , len ( removables ) , 4096 ) : self . _conn . executemany ( " DELETE FROM known_destinations WHERE destination_hash=? " , removables [ i : i + 4096 ] )
for i in range ( 0 , len ( ratchet_rows ) , 4096 ) : self . _conn . executemany ( " INSERT INTO ratchets VALUES (?,?,?) ON CONFLICT(destination_hash) DO UPDATE SET ratchet=excluded.ratchet, received=excluded.received " , ratchet_rows [ i : i + 4096 ] )
for i in range ( 0 , len ( ratchet_removables ) , 4096 ) : self . _conn . executemany ( " DELETE FROM ratchets WHERE destination_hash=? " , ratchet_removables [ i : i + 4096 ] )
self . _conn . commit ( )
except Exception :
# Re-mark everything as dirty, so the changes are retried
# on the next flush, then propagate.
self . _dirty_upsert | = { row [ 0 ] for row in rows }
self . _dirty_delete | = { row [ 0 ] for row in removables }
self . _ratchet_upserts | = { row [ 0 ] : row [ 1 : ] for row in ratchet_rows }
self . _ratchet_deletes | = { row [ 0 ] for row in ratchet_removables }
self . _conn . rollback ( )
raise
return len ( rows ) + len ( removables ) + len ( ratchet_rows ) + len ( ratchet_removables )
def load_all ( self ) :
2026-10-27 14:17:25 +01:00
cursor = self . _conn . execute ( " SELECT destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost FROM known_destinations " )
2026-10-24 00:20:27 +02:00
while True :
rows = cursor . fetchmany ( 4096 )
if not rows : break
2026-10-27 14:17:25 +01:00
for row in rows : yield row [ 0 ] , [ row [ 1 ] , row [ 2 ] , row [ 3 ] , row [ 4 ] , row [ 5 ] , row [ 6 ] , row [ 7 ] , row [ 8 ] ]
2026-10-24 00:20:27 +02:00
# Caller must hold Identity.known_destinations_lock.
def remember_ratchet ( self , destination_hash , ratchet , now ) :
if self . read_only : return False
pending = self . _ratchet_upserts . get ( destination_hash )
if pending != None :
if pending [ 0 ] == ratchet :
self . _ratchet_deletes . discard ( destination_hash )
return False
else :
row = self . _conn . execute ( " SELECT ratchet FROM ratchets WHERE destination_hash=? " , ( destination_hash , ) ) . fetchone ( )
if row != None and row [ 0 ] == ratchet :
# Discard any pending delete mark queued
# before this ratchet arrived
self . _ratchet_deletes . discard ( destination_hash )
return False
self . _ratchet_upserts [ destination_hash ] = ( ratchet , now )
self . _ratchet_deletes . discard ( destination_hash )
return True
def fetch_ratchet ( self , destination_hash ) :
pending = self . _ratchet_upserts . get ( destination_hash )
if pending != None : return pending [ 0 ] , pending [ 1 ]
if destination_hash in self . _ratchet_deletes : return None
row = self . _conn . execute ( " SELECT ratchet, received FROM ratchets WHERE destination_hash=? " , ( destination_hash , ) ) . fetchone ( )
if row != None : return row [ 0 ] , row [ 1 ]
return None
# Caller must hold Identity.known_destinations_lock
def mark_ratchet_deleted ( self , destination_hash ) :
if self . read_only : return
self . _ratchet_upserts . pop ( destination_hash , None )
self . _ratchet_deletes . add ( destination_hash )
def expire_ratchets ( self , cutoff ) :
if self . read_only : return [ ]
with self . _flush_lock :
# Drop expired entries in pending as well
expired = set ( row [ 0 ] for row in self . _conn . execute ( " SELECT destination_hash FROM ratchets WHERE received<? " , ( cutoff , ) ) )
for destination_hash , entry in list ( self . _ratchet_upserts . items ( ) ) :
if entry [ 1 ] < cutoff :
expired . add ( destination_hash )
self . _ratchet_upserts . pop ( destination_hash , None )
expired . difference_update ( self . _ratchet_deletes )
if len ( expired ) > 0 :
self . _conn . execute ( " BEGIN " )
for i in range ( 0 , len ( expired ) , 4096 ) :
self . _conn . executemany ( " DELETE FROM ratchets WHERE destination_hash=? " , [ ( h , ) for h in list ( expired ) [ i : i + 4096 ] ] )
self . _conn . commit ( )
return list ( expired )
def load_ratchets ( self ) :
cursor = self . _conn . execute ( " SELECT destination_hash, ratchet, received FROM ratchets " )
while True :
rows = cursor . fetchmany ( 4096 )
if not rows : break
for row in rows : yield row [ 0 ] , row [ 1 ] , row [ 2 ]
def import_legacy_ratchets ( self , dir_path , expiry_cutoff , known_destinations , ratchet_width = 32 ) :
import RNS
from . vendor import umsgpack
imported = 0
for filename in os . listdir ( dir_path ) :
file_path = os . path . join ( dir_path , filename )
try :
if not os . path . isfile ( file_path ) : continue
with open ( file_path , " rb " ) as rf :
ratchet_data = umsgpack . unpackb ( rf . read ( ) )
destination_hash = bytes . fromhex ( filename )
ratchet = ratchet_data [ " ratchet " ] ; received = ratchet_data [ " received " ]
if len ( ratchet ) != ratchet_width : raise ValueError ( " Invalid ratchet width " )
if received < expiry_cutoff : raise ValueError ( " Ratchet expired " )
if destination_hash not in known_destinations : raise ValueError ( " Destination unknown " )
self . _ratchet_upserts [ destination_hash ] = ( ratchet , received )
imported + = 1
os . unlink ( file_path )
except Exception as e :
RNS . log ( f " Could not import ratchet for { filename } : { e } " , RNS . LOG_WARNING )
try : os . unlink ( file_path )
except : pass
pending = self . _ratchet_upserts
self . _ratchet_upserts = { }
self . apply ( set ( ) , set ( ) , pending , set ( ) , known_destinations )
return imported
def import_legacy ( self , path ) :
from . vendor import umsgpack
with open ( path , " rb " ) as file : legacy = umsgpack . load ( file )
rows = [ ]
now = time . time ( )
for destination_hash , entry in legacy . items ( ) :
if len ( entry ) < 5 : continue
entry = list ( entry )
2026-10-27 14:17:25 +01:00
while len ( entry ) < 8 : entry . append ( None if len ( entry ) in ( 5 , 7 ) else 0 )
2026-10-24 00:20:27 +02:00
if entry [ 0 ] == None or entry [ 0 ] < = 0 : entry [ 0 ] = now
rows . append ( self . __entry_row ( destination_hash , entry ) )
self . _conn . execute ( " BEGIN " )
2026-10-27 14:17:25 +01:00
for i in range ( 0 , len ( rows ) , 4096 ) : self . _conn . executemany ( " INSERT OR REPLACE INTO known_destinations (destination_hash, last_heard, packet_hash, public_key, app_data, used, stamp, stamp_value, lr_cost, identity_hash) VALUES (?,?,?,?,?,?,?,?,?,?) " , rows [ i : i + 4096 ] )
2026-10-24 00:20:27 +02:00
self . _conn . commit ( )
return len ( rows )
class AnnounceCache :
def __init__ ( self , db_path ) :
import sqlite3
self . db_path = db_path
self . _pending = { }
self . _flush_lock = threading . Lock ( )
self . _conn = sqlite3 . connect ( db_path , check_same_thread = False , isolation_level = None )
self . _conn . execute ( " PRAGMA journal_mode=WAL " )
self . _conn . execute ( " PRAGMA synchronous=NORMAL " )
self . _conn . execute ( " PRAGMA journal_size_limit=8388608 " )
self . _conn . execute ( " PRAGMA auto_vacuum=INCREMENTAL " )
self . __create_schema ( )
def __create_schema ( self ) :
self . _conn . execute ( """ CREATE TABLE IF NOT EXISTS cached_announces (
packet_hash BLOB PRIMARY KEY ,
received REAL NOT NULL ,
raw BLOB NOT NULL ,
interface_reference TEXT ) """ )
self . _conn . execute ( " PRAGMA user_version = 1 " )
def store ( self , packet_hash , raw , interface_reference ) :
self . _pending [ packet_hash ] = ( time . time ( ) , raw , interface_reference )
def pending_count ( self ) :
return len ( self . _pending )
def fetch ( self , packet_hash ) :
pending = self . _pending . get ( packet_hash )
if pending != None : return pending [ 1 ] , pending [ 2 ]
row = self . _conn . execute ( " SELECT raw, interface_reference FROM cached_announces WHERE packet_hash=? " , ( packet_hash , ) ) . fetchone ( )
if row != None : return row [ 0 ] , row [ 1 ]
return None
def flush ( self ) :
with self . _flush_lock : return self . __apply_pending ( )
def __apply_pending ( self ) :
pending = self . _pending
self . _pending = { }
if len ( pending ) == 0 : return 0
try :
self . _conn . execute ( " BEGIN " )
items = list ( pending . items ( ) )
for i in range ( 0 , len ( items ) , 4096 ) :
self . _conn . executemany ( " INSERT INTO cached_announces VALUES (?,?,?,?) ON CONFLICT(packet_hash) DO UPDATE SET received=excluded.received, raw=excluded.raw, interface_reference=excluded.interface_reference " ,
[ ( packet_hash , * entry ) for packet_hash , entry in items [ i : i + 4096 ] ] )
self . _conn . commit ( )
except Exception :
# Re-mark everything as pending, so the changes
# are retried on the next flush, then propagate.
self . _pending . update ( pending )
self . _conn . rollback ( )
raise
return len ( pending )
def clean ( self , active_hashes ) :
with self . _flush_lock :
self . __apply_pending ( )
try :
self . _conn . execute ( " BEGIN " )
self . _conn . execute ( " DROP TABLE IF EXISTS temp.active_announces " )
self . _conn . execute ( " CREATE TEMP TABLE active_announces (packet_hash BLOB PRIMARY KEY) " )
active_list = list ( active_hashes )
for i in range ( 0 , len ( active_list ) , 4096 ) :
self . _conn . executemany ( " INSERT OR IGNORE INTO active_announces VALUES (?) " , [ ( h , ) for h in active_list [ i : i + 4096 ] ] )
changes_before = self . _conn . total_changes
self . _conn . execute ( " DELETE FROM cached_announces WHERE packet_hash NOT IN (SELECT packet_hash FROM active_announces) " )
removed = self . _conn . total_changes - changes_before
self . _conn . commit ( )
except Exception :
self . _conn . rollback ( )
raise
if removed > 0 : self . _conn . execute ( " PRAGMA incremental_vacuum " )
return removed
def import_legacy ( self , dir_path ) :
from . vendor import umsgpack
imported = 0
for filename in os . listdir ( dir_path ) :
file_path = os . path . join ( dir_path , filename )
try :
if not os . path . isfile ( file_path ) : continue
with open ( file_path , " rb " ) as file :
cached_data = umsgpack . unpackb ( file . read ( ) )
packet_hash = bytes . fromhex ( filename )
interface_reference = cached_data [ 1 ] if len ( cached_data ) > 1 else None
self . store ( packet_hash , cached_data [ 0 ] , interface_reference )
imported + = 1
os . unlink ( file_path )
except Exception :
try : os . unlink ( file_path )
except Exception : pass
self . flush ( )
return imported
def close ( self ) :
if self . _conn != None :
self . _conn . close ( )
self . _conn = None