From 044d4c9adc4781165eea1fa223ece5ef278df787 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Sat, 24 Oct 2026 00:58:06 +0200 Subject: [PATCH] Added internal stamps module --- RNS/Discovery.py | 19 +- RNS/Stamps.py | 444 +++++++++++++++++++++++++++++++++++++ docs/source/interfaces.rst | 7 - 3 files changed, 447 insertions(+), 23 deletions(-) create mode 100644 RNS/Stamps.py diff --git a/RNS/Discovery.py b/RNS/Discovery.py index bd584b6c..6551f21c 100644 --- a/RNS/Discovery.py +++ b/RNS/Discovery.py @@ -45,6 +45,7 @@ from collections import deque # version tag in announced discovery information. IMPLEMENTATION_NAME = "RNS" from RNS._version import __version__ as RNS_VERSION +import RNS.Stamps NAME = 0xFF TRANSPORT_ID = 0xFE @@ -78,17 +79,10 @@ class InterfaceAnnouncer(): "RNodeInterface", "WeaveInterface", "I2PInterface", "KISSInterface"] def __init__(self, owner): - import importlib.util - if importlib.util.find_spec('LXMF') != None: from LXMF import LXStamper - else: - RNS.log("Using on-network interface discovery requires the LXMF module to be installed.", RNS.LOG_CRITICAL) - RNS.log("You can install it with the command: pip install lxmf", RNS.LOG_CRITICAL) - RNS.panic() - self.owner = owner self.should_run = False self.job_interval = self.JOB_INTERVAL - self.stamper = LXStamper + self.stamper = RNS.Stamps self.stamp_cache = {} if self.owner.has_network_identity(): identity = self.owner.network_identity @@ -262,17 +256,10 @@ class InterfaceAnnounceHandler: FLAG_ENCRYPTED = 0b00000010 def __init__(self, required_value=InterfaceAnnouncer.DEFAULT_STAMP_VALUE, callback=None): - import importlib.util - if importlib.util.find_spec('LXMF') != None: from LXMF import LXStamper - else: - RNS.log("Using on-network interface discovery requires the LXMF module to be installed.", RNS.LOG_CRITICAL) - RNS.log("You can install it with the command: pip install lxmf", RNS.LOG_CRITICAL) - RNS.panic() - self.aspect_filter = APP_NAME+".discovery.interface" self.required_value = required_value self.callback = callback - self.stamper = LXStamper + self.stamper = RNS.Stamps self.valid_cache = {} self.valid_cache_max = 2048 self.invalid_cache = deque(maxlen=2048) diff --git a/RNS/Stamps.py b/RNS/Stamps.py new file mode 100644 index 00000000..5832aced --- /dev/null +++ b/RNS/Stamps.py @@ -0,0 +1,444 @@ +# Reticulum License +# +# Copyright (c) 2016-2026 Mark Qvist +# +# Permission is hereby granted, free of charge, to any person obtaining a copy +# of this software and associated documentation files (the "Software"), to deal +# in the Software without restriction, including without limitation the rights +# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +# copies of the Software, and to permit persons to whom the Software is +# furnished to do so, subject to the following conditions: +# +# - The Software shall not be used in any kind of system which includes amongst +# its functions the ability to purposefully do harm to human beings. +# +# - The Software shall not be used, directly or indirectly, in the creation of +# an artificial intelligence, machine learning or language model training +# dataset, including but not limited to any use that contributes to the +# training or development of such a model or algorithm. +# +# - The above copyright notice and this permission notice shall be included in +# all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +import RNS +import RNS.vendor.umsgpack as msgpack + +import os +import sys +import time +import math +import itertools +import contextlib +import multiprocessing + +WORKBLOCK_EXPAND_ROUNDS = 3000 +STAMP_SIZE = RNS.Identity.HASHLENGTH//8 +PN_VALIDATION_POOL_MIN_SIZE = 256 +USE_WORKER_MANAGER = False +POOLING_THRESHOLD_COST = 15 +STOP_CHECK_INTERVAL = 64 + +active_jobs = {} + +if sys.version_info[0] >= 3 and sys.version_info[1] >= 14: + USE_WORKER_MANAGER = True + +@contextlib.contextmanager +def worker_context(ctx, stamp_cost, workblock, message_id): + stop_event = ctx.Event() + result_queue = ctx.Queue(1) + rounds_queue = ctx.Queue() + + active_jobs[message_id] = [stop_event, result_queue] + job_procs = [] + + try: yield stop_event, result_queue, rounds_queue, job_procs + finally: + if message_id in active_jobs: del active_jobs[message_id] + stop_event.set() + + for p in job_procs: + if p.is_alive(): p.terminate() + + for p in job_procs: p.join(timeout=0.5) + + result_queue.close() + result_queue.join_thread() + rounds_queue.close() + rounds_queue.join_thread() + stop_event.clear() + +def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=False): + if no_expand or not expand_rounds: return material + wb_st = time.time() + workblock = b"" + for n in range(expand_rounds): + workblock += RNS.Cryptography.hkdf(length=256, derive_from=material, + salt=RNS.Identity.full_hash(material+msgpack.packb(n)), + context=None) + wb_time = time.time() - wb_st + + return workblock + +def stamp_value(workblock, stamp): + value = 0 + bits = 256 + material = RNS.Identity.full_hash(workblock+stamp) + i = int.from_bytes(material, byteorder="big") + while ((i & (1 << (bits - 1))) == 0): + i = (i << 1) + value += 1 + + return value + +def stamp_valid(stamp, target_cost, workblock): + target = 0b1 << 256-target_cost + result = RNS.Identity.full_hash(workblock+stamp) + if int.from_bytes(result, byteorder="big") > target: return False + else: return True + +def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=False): + RNS.log(f"Generating stamp with cost {stamp_cost} for {RNS.prettyhexrep(message_id)}...", RNS.LOG_DEBUG) + workblock = stamp_workblock(message_id, expand_rounds=expand_rounds, no_expand=no_expand) + + start_time = time.time() + stamp = None + rounds = 0 + value = 0 + + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): + RNS.log(f"Running stamp generation on {platform}, work limited to single CPU core. This will be slower than ideal.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id) + elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id) + else: + if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id) if stamp_cost >= POOLING_THRESHOLD_COST else job_simple(stamp_cost, workblock, message_id) + else: stamp, rounds = job_linux(stamp_cost, workblock, message_id) if stamp_cost >= POOLING_THRESHOLD_COST else job_simple(stamp_cost, workblock, message_id) + + duration = max(time.time()-start_time, 1E-6) + speed = rounds/duration + if stamp != None: value = stamp_value(workblock, stamp) + + RNS.log(f"Stamp with value {value} generated in {RNS.prettytime(duration)}, {rounds} rounds, {RNS.prettysize(speed, suffix='rps')}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + + return stamp, value + +def cancel_work(message_id): + if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): + try: + if message_id in active_jobs: + active_jobs[message_id] = True + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + elif RNS.vendor.platformutils.is_android(): + try: + if message_id in active_jobs: + active_jobs[message_id] = True + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + else: + try: + if message_id in active_jobs: + stop_event = active_jobs[message_id][0] + result_queue = active_jobs[message_id][1] + stop_event.set() + result_queue.put(None) + active_jobs.pop(message_id) + + except Exception as e: + RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + +def job_simple(stamp_cost, workblock, message_id): + # A simple, single-process stamp generator. + # should work on any platform, and is used + # as a fall-back, in case of limited multi- + # processing and/or acceleration support. + + rounds = 0 + pstamp = os.urandom(256//8) + st = time.time() + + active_jobs[message_id] = False + + target = 0b1<<256-stamp_cost + c = 0 + + while True: + c += 1 + if c >= STOP_CHECK_INTERVAL: + c = 0 + if active_jobs[message_id]: break + + if int.from_bytes(RNS.Identity.full_hash(workblock+pstamp), byteorder="big") <= target: break + pstamp = os.urandom(256//8); rounds += 1 + + if rounds % 250000 == 0: + speed = rounds / max(time.time()-st, 1E-6) + RNS.log(f"Stamp generation running. {rounds} rounds completed so far, {RNS.prettysize(speed, suffix='')} rounds per second", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + + cancelled = (active_jobs[message_id] == True) + active_jobs.pop(message_id) + return (None if cancelled else pstamp), rounds + +def job_linux_managed(stamp_cost, workblock, message_id): + ctx = multiprocessing.get_context("fork") + cores = multiprocessing.cpu_count() + jobs = cores if cores <= 12 else int(cores/2) + + allow_kill = True + stamp = None + total_rounds = 0 + + with worker_context(ctx, stamp_cost, workblock, message_id) as (stop_event, result_queue, rounds_queue, job_procs): + RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) + + def job(stop_event, sc, wb, pn): + rounds = 0; pstamp = os.urandom(256//8); + target = 0b1<<256 - sc; c = 0 + + while True: + c += 1 + if c >= STOP_CHECK_INTERVAL: + c = 0 + if stop_event.is_set(): break + + m = wb+pstamp + if int.from_bytes(RNS.Identity.full_hash(m), byteorder="big") <= target: break + pstamp = os.urandom(256//8); rounds += 1 + + if not stop_event.is_set(): + stop_event.set() + try: result_queue.put_nowait(pstamp) + except: pass + + try: rounds_queue.put_nowait(rounds) + except: pass + + for jpn in range(jobs): + p = ctx.Process(target=job, args=(stop_event, stamp_cost, workblock, jpn), daemon=True) + job_procs.append(p) + p.start() + + try: stamp = result_queue.get(timeout=None) + except Exception as e: + RNS.log(f"Failed to get result from workers: {e}", RNS.LOG_ERROR) + stamp = None + + # Collect any potential spurious + # results from worker queue. + try: + while True: result_queue.get_nowait() + except: pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.005) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR) + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + else: + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + # RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove + + return stamp, total_rounds + +def job_linux(stamp_cost, workblock, message_id): + allow_kill = True + stamp = None + total_rounds = 0 + cores = multiprocessing.cpu_count() + jobs = cores if cores <= 12 else int(cores/2) + stop_event = multiprocessing.Event() + result_queue = multiprocessing.Queue(1) + rounds_queue = multiprocessing.Queue() + + def job(stop_event, pn, sc, wb): + rounds = 0; pstamp = os.urandom(256//8); + target = 0b1<<256 - sc; c = 0 + + while True: + c += 1 + if c >= STOP_CHECK_INTERVAL: + c = 0 + if stop_event.is_set(): break + + m = wb+pstamp + if int.from_bytes(RNS.Identity.full_hash(m), byteorder="big") <= target: break + pstamp = os.urandom(256//8); rounds += 1 + + if not stop_event.is_set(): + stop_event.set() + result_queue.put(pstamp) + + rounds_queue.put(rounds) + + job_procs = [] + RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) + for jpn in range(jobs): + process = multiprocessing.get_context("fork").Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": stamp_cost, "wb": workblock}, daemon=True) + job_procs.append(process) + process.start() + + active_jobs[message_id] = [stop_event, result_queue] + + try: + stamp = result_queue.get() + + # Collect any potential spurious + # results from worker queue. + try: + while True: result_queue.get_nowait() + except: pass + + for j in range(jobs): + nrounds = 0 + try: + nrounds = rounds_queue.get(timeout=2) + except Exception as e: + RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR) + total_rounds += nrounds + + all_exited = False + exit_timeout = time.time() + 5 + while time.time() < exit_timeout: + if not any(p.is_alive() for p in job_procs): + all_exited = True + break + time.sleep(0.005) + + if not all_exited: + RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR) + if allow_kill: + for j in range(jobs): + process = job_procs[j] + process.kill() + else: + return None + + else: + for j in range(jobs): + process = job_procs[j] + process.join() + + except Exception as e: RNS.log(f"Error while getting stamp generation result: {e}", RNS.LOG_ERROR) + finally: active_jobs.pop(message_id, None) + + return stamp, total_rounds + +def job_android(stamp_cost, workblock, message_id): + # Semaphore support is flaky to non-existent on + # Android, so we need to manually dispatch and + # manage workloads here, while periodically + # checking in on the progress. + + stamp = None + start_time = time.time() + total_rounds = 0 + rounds_per_worker = 1000 + + use_nacl = False + try: + import nacl.encoding + import nacl.hash + use_nacl = True + except: pass + + if use_nacl: + def full_hash(m): return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder) + else: + def full_hash(m): return RNS.Identity.full_hash(m) + + wm = multiprocessing.Manager() + jobs = multiprocessing.cpu_count() + + def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None): + try: + rounds = 0 + found_stamp = None + target = 0b1<<256-sc + fh = full_hash + + while True: + pstamp = os.urandom(256//8) + rounds += 1 + if int.from_bytes(fh(wb+pstamp), byteorder="big") <= target: + found_stamp = pstamp; break + if rounds >= jr: break + + results_dict[procnum] = [found_stamp, rounds] + + except Exception as e: + RNS.log(f"Stamp generation worker error: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + active_jobs[message_id] = False; + + RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + + results_dict = wm.dict() + while stamp == None and active_jobs[message_id] == False: + job_procs = [] + try: + for pnum in range(jobs): + pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":stamp_cost, "jr":rounds_per_worker} + process = multiprocessing.Process(target=job, kwargs=pargs) + job_procs.append(process) + process.start() + + for process in job_procs: + process.join() + + for j in results_dict: + r = results_dict[j] + total_rounds += r[1] + if r[0] != None: + stamp = r[0] + + if stamp == None: + elapsed = max(time.time()-start_time, 1E-6) + speed = total_rounds/elapsed + RNS.log(f"Stamp generation running. {total_rounds} rounds completed so far, {RNS.prettysize(speed, suffix='rps')}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + + except Exception as e: + RNS.log(f"Stamp generation job error: {e}", RNS.LOG_ERROR) + RNS.trace_exception(e) + + active_jobs.pop(message_id) + + return stamp, total_rounds diff --git a/docs/source/interfaces.rst b/docs/source/interfaces.rst index 9a9374a8..403ad1d6 100644 --- a/docs/source/interfaces.rst +++ b/docs/source/interfaces.rst @@ -965,13 +965,6 @@ Reticulum includes a powerful system for publishing your local interfaces to the When an interface is made **discoverable**, your Reticulum instance will periodically broadcast an announce packet containing the connection details and parameters required for other peers to establish a connection. These announces are propagated over the network using the standard Reticulum announce mechanism using the ``rnstransport.discovery.interface`` destination type. -.. note:: - To use the interface discovery functionality, the ``LXMF`` module must be installed in your Python environment. You can install it using pip: - - .. code:: sh - - pip install lxmf - Enabling Discovery ------------------