Reticulum/RNS/Stamps.py
2026-10-31 01:58:33 +01:00

915 lines
No EOL
39 KiB
Python

# Reticulum License
#
# Copyright (c) 2016-2026 Mark Qvist
#
# Permission is hereby granted, free of charge, to any person obtaining a copy
# of this software and associated documentation files (the "Software"), to deal
# in the Software without restriction, including without limitation the rights
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
# copies of the Software, and to permit persons to whom the Software is
# furnished to do so, subject to the following conditions:
#
# - The Software shall not be used in any kind of system which includes amongst
# its functions the ability to purposefully do harm to human beings.
#
# - The Software shall not be used, directly or indirectly, in the creation of
# an artificial intelligence, machine learning or language model training
# dataset, including but not limited to any use that contributes to the
# training or development of such a model or algorithm.
#
# - The above copyright notice and this permission notice shall be included in
# all copies or substantial portions of the Software.
#
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
# SOFTWARE.
import RNS
import RNS.vendor.umsgpack as msgpack
from RNS.Cryptography import sha256
import os
import sys
import time
import math
import mmap
import itertools
import contextlib
import multiprocessing
from bisect import bisect_right
WORKBLOCK_EXPAND_ROUNDS = 3000
STAMP_SIZE = 32
PN_VALIDATION_POOL_MIN_SIZE = 256
USE_WORKER_MANAGER = False
POOLING_THRESHOLD_COST = 15
KEYSTAMP_POOLING_THRESHOLD = 12
STOP_CHECK_INTERVAL = 64
MODE_STAMP = 0x00
MODE_KEYSTAMP = 0x01
active_jobs = {}
if sys.version_info[0] >= 3 and sys.version_info[1] >= 14:
USE_WORKER_MANAGER = True
@contextlib.contextmanager
def worker_context(ctx, stamp_cost, workblock, message_id):
stop_event = ctx.Event()
result_queue = ctx.Queue(1)
rounds_queue = ctx.Queue()
active_jobs[message_id] = [stop_event, result_queue]
job_procs = []
try: yield stop_event, result_queue, rounds_queue, job_procs
finally:
if message_id in active_jobs: del active_jobs[message_id]
stop_event.set()
for p in job_procs:
if p.is_alive(): p.terminate()
for p in job_procs: p.join(timeout=0.5)
result_queue.close()
result_queue.join_thread()
rounds_queue.close()
rounds_queue.join_thread()
stop_event.clear()
def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=False):
if no_expand or not expand_rounds: return material
wb_st = time.time()
workblock = b""
for n in range(expand_rounds):
workblock += RNS.Cryptography.hkdf(length=256, derive_from=material,
salt=RNS.Identity.full_hash(material+msgpack.packb(n)),
context=None)
wb_time = time.time() - wb_st
return workblock
def stamp_value_legacy(workblock, stamp):
if not workblock: workblock = b""
stamp_product = RNS.Identity.full_hash(workblock+stamp)
return 256 - int.from_bytes(stamp_product, byteorder="big").bit_length()
def stamp_value(workblock, stamp):
# This function is very little code, and a whole lot of
# comments. The actual method employed here is relatively
# simple, but without proper explanation, it is also very
# opaque, and could induce a lot of head-scratching.
if not workblock: workblock = b""
stamp_product = RNS.Identity.full_hash(workblock+stamp) # Bytes representation
hn = int.from_bytes(stamp_product, byteorder="big") # Numerical representation
if hn == 0: return 256 # Yeah, but *if* it happens, you know...
nb = MANTISSA_BITS+1 # Normalized mantissa bit count
bl = hn.bit_length() # Bit length of the stamp product
lz = 256 - bl # Leading zeros of the stamp product
##############################################################
# #
# Simplified mantissa extraction example with #
# smaller numbers; 32-bit stamp product and 8 #
# mantissa bits. In the actual version, it is #
# 256-bit stamp products and 33-bit mantissa. #
# #
# mb = 8 nb = 9 (8+1) #
# bl = 27 lz = 5 (32-27) #
# #
# shift = bl - nb = 18 #
# #
# hn = 0b00000111101101101110111010110111 #
# >>18 0b00000000000000000000000111101101|101110111010110111 #
# | \_____ gone #
# The leading one | #
# 8 significand mantissa bits #
# #
# mn = 0b111101101 (normalized with the leading 1) #
# #
##############################################################
# Normalize mantissa to the scale of TAB: Total of 34
# bits, 33 mantissa bits. The leading one required to
# match TAB entries is guaranteed by the construction:
# Since we're shifting by the *difference* between the
# normalized mantissa bit length and the stamp product
# bit length, we always end up with a leading one, and
# then the required number of *actual* mantissa bits.
# See the example above for a simplified illustration.
if bl > nb: mn = (hn >> (bl - nb))
else: mn = (hn << (nb - bl)) # Again, better safe than sorry. You may notice
# here that in this case, we will have a more
# coarse mantissa part, which is inherent to
# the methodology, but in practice the best thing
# we can do. Additionally, this can only happen
# at absolutely extreme values, far above the
# stamp cap anyway. It is only here to handle the
# absurd case where a stamp product randomly lands
# on an astronomically improbable leading zero count.
# We can now construct the fixed-point value representation:
#
# The leading zero count is equivalent to the legacy value, an
# integer representation. Thus, (lz+1)<<COST_FP_BITS gives us
# the fixed-point representation of a value one greater than
# what we currently have. Since the mantissa is normalized, the
# real value always lies somewhere in between, at the ceiling
# minus however far the mantissa has climbed into this range
#
# Right-bisecting TAB with the normalized mantissa returns the
# number of grid points at or below it, and since the table
# values *are* the normalized mantissas of the grid points in
# our 256-step quantization, this count is the climb, quantized
# in 1/256th steps. The *count* always overshoots the *actual*
# climb (one full step on an exact match), so subtracting it
# from the ceiling *floors* the value onto the grid, so we
# never over-value the stamp.
#
# Note that for a mantissa *past* (or equal to) the last table
# entry, there is no special case handling required, since the
# bisect_right count returned in that case is 256 (the largest
# it can ever return). Thus, the arithmetic itself here folds
# correctly: The entire "added one" (lz+1) is consumed, and can
# never over-consume. Remember: In the fixed-point representation
# a value of 1.0 *is* 256 (and, as a side note, this is one of
# the reasons that len(TAB)==COST_FP is asserted as invariant).
value_fp = ((lz + 1) << COST_FP_BITS) - bisect_right(TAB, mn)
# Then, convert to decimal representation and return it
return cost_from_fp(value_fp)
def stamp_valid(stamp, target_cost, workblock):
if not workblock: workblock = b""
target = target_from_fp(cost_to_fp(target_cost))
result = RNS.Identity.full_hash(workblock+stamp)
if int.from_bytes(result, byteorder="big") > target: return False
else: return True
# Synthetic stop check function used where
# worker mechanism provides no support for
# semaphore-based cancellation.
def _never(): return False
# The hash-mode loop primitive, shared
# between platform variants.
def _loop_hash(workblock, target, size, stop_check, progress=None, max_rounds=0, fh=None):
if fh == None: fh = RNS.Identity.full_hash
urandom = os.urandom
rounds = 0
found = False
pstamp = urandom(size)
c = 0
while True:
c += 1
if c >= STOP_CHECK_INTERVAL:
c = 0
if stop_check(): break
if int.from_bytes(fh(workblock+pstamp), byteorder="big") <= target:
found = True; break
pstamp = urandom(size); rounds += 1
if progress != None and rounds % 250000 == 0: progress(rounds)
if max_rounds != 0 and rounds >= max_rounds: break
return pstamp, rounds, found
# The key-mode loop primitive, shared
# between platform variants.
def _loop_key(material, target, stop_check, max_rounds=0):
fh = RNS.Identity.full_hash
generate = RNS.Cryptography.X25519PrivateKey.generate
rounds = 0
found = False
keypair = None
while True:
if stop_check(): break
key = generate()
pub_bytes = key.public_key().public_bytes()
if int.from_bytes(fh(material+pub_bytes), byteorder="big") <= target:
keypair = (key.private_bytes(), pub_bytes)
found = True; break
rounds += 1
if max_rounds != 0 and rounds >= max_rounds: break
return keypair, rounds, found
# Generates a normal, detached, hash-based stamp for any
# bytes material (message_id), which is usually a hash
# of some input material, but can be any sequence of bytes.
def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=False, size=STAMP_SIZE):
if stamp_cost < 0 or stamp_cost > RNS.Transport.STAMP_COST_CAP: raise ValueError("Invalid stamp cost, cannot generate")
workblock = stamp_workblock(message_id, expand_rounds=expand_rounds, no_expand=no_expand)
if message_id in active_jobs: raise RuntimeError(f"Duplicate stamp generation job for {RNS.prettyhexrep(message_id)}")
RNS.log(f"Generating stamp with cost {stamp_cost} for <{RNS.hexrep(message_id[:16], delimit=False)}{'…' if len(message_id) > 16 else ''}>", RNS.LOG_DEBUG)
stamp_cost = cost_to_fp(stamp_cost)
start_time = time.time()
stamp = None
value = None
rounds = 0
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin():
RNS.log(f"Multi-core stamp generation on this platform is not supported, work limited to single CPU core.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id, size)
elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id, size)
else:
if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST_FP else job_simple(stamp_cost, workblock, message_id, size)
else: stamp, rounds = job_linux(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST_FP else job_simple(stamp_cost, workblock, message_id, size)
duration = max(time.time()-start_time, 1E-6)
speed = rounds/duration
if stamp != None: value = stamp_value(workblock, stamp)
if value != None: RNS.log(f"Stamp with value {value} generated in {RNS.prettytime(duration)}, {rounds} rounds, {RNS.prettysize(speed, suffix='rps')}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
else: RNS.log(f"Stamp generation aborted or failed after {RNS.prettytime(duration)}, {rounds} rounds", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return stamp, value
# Generates a new, internally stamped X25519 keypair. The
# public key carries the stamp value itself, so no detached
# stamp is created. Unlike the detached stamp generator, this
# method returns the private+public X25519 keypair, and the
# private key can be loaded normally to create a full, stamped
# Reticulum Identity. The stamp value is verifiable to anyone
# knowing the public key (and the optional input material).
#
# For a transparently verifiable stamped identity, you should
# generally always use the provided `RNS.Identity(value=n)`
# primitive, and not generate raw stamped keys manually. Using
# the API primitive correctly binds the full identity keyset
# together with the stamped portion of the key. Situations do
# exist where you may want a bare, no-material stamped key,
# but this should be considered hazardous materials territory!
#
# For custom stamped keys, where the stamped key is intended
# to be tied to a specific domain, for example a link request
# or other one-time use, the `material` argument must be the
# relevant uniquely identifying nonce of such use that the
# stamped key is bound to.
#
# Key stamp validation uses the normal validation methods,
# simply provide the public key bytes directly *as* the stamp,
# and any domain material as the workblock, for example:
#
# - stamp_valid(public_key_bytes, required_cost, material)
# - stamp_value(material, public_key_bytes)
#
# WARNING: Never attempt to use this for Ed25519 *signature* key
# creation, is NOT a valid approach. The keypairs emitted by
# this function must ONLY be used as X25519 keys.
def generate_stamped_key(material, stamp_cost):
if material == None: material = b""
if material in active_jobs: raise RuntimeError(f"Duplicate stamped key generation job for {RNS.prettyhexrep(material)}")
if stamp_cost < 0 or stamp_cost > RNS.Transport.STAMP_COST_CAP: raise ValueError("Invalid stamp cost, cannot generate")
for_str = f" for <{RNS.hexrep(material[:16], delimit=False)}{'…' if len(material) > 16 else ''}>" if material else ""
RNS.log(f"Generating stamped key with cost {stamp_cost}{for_str}", RNS.LOG_DEBUG)
stamp_cost = cost_to_fp(stamp_cost)
start_time = time.time()
keypair = None
value = None
rounds = 0
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin():
RNS.log(f"Multi-core stamp generation on this platform is not supported, work limited to single CPU core.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): keypair, rounds = job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
elif RNS.vendor.platformutils.is_android(): keypair, rounds = job_android(stamp_cost, material, material, 0, MODE_KEYSTAMP)
else:
if USE_WORKER_MANAGER: keypair, rounds = job_linux_managed(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD_FP else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
else: keypair, rounds = job_linux(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD_FP else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
duration = max(time.time()-start_time, 1E-6)
speed = rounds/duration
if keypair != None: value = stamp_value(material, keypair[1])
if value != None: RNS.log(f"Stamped key with value {value} generated in {RNS.prettyshorttime(duration)}, {rounds} rounds, {RNS.prettysize(speed, suffix='rps')}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
else: RNS.log(f"Stamp generation aborted or failed after {RNS.prettytime(duration)}, {rounds} rounds", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return keypair, value
def cancel_work(message_id):
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin():
try:
if message_id in active_jobs:
active_jobs[message_id] = True
except Exception as e:
RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
elif RNS.vendor.platformutils.is_android():
try:
if message_id in active_jobs:
active_jobs[message_id] = True
except Exception as e:
RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
else:
try:
job_entry = active_jobs.get(message_id, None)
if job_entry != None:
if type(job_entry) != list: active_jobs[message_id] = True
else:
stop_event = job_entry[0]
result_queue = job_entry[1]
stop_event.set()
result_queue.put(None)
active_jobs.pop(message_id)
except Exception as e:
RNS.log("Error while terminating stamp generation workers: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
def job_simple(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
# A simple, single-process stamp generator.
# should work on any platform, and is used
# as a fall-back, in case of limited multi-
# processing and/or acceleration support.
#
# In key mode, the workblock argument is
# used for the stamp material, and the
# size argument is ignored.
st = time.time()
target = target_from_fp(stamp_cost)
active_jobs[message_id] = False
def stop_check(): return active_jobs[message_id]
def progress(r):
speed = r / max(time.time()-st, 1E-6)
RNS.log(f"Stamp generation running. {r} rounds completed so far, {RNS.prettysize(speed, suffix='')} rounds per second", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
if mode == MODE_STAMP: result, rounds, _ = _loop_hash(workblock, target, size, stop_check, progress)
else: result, rounds, _ = _loop_key(workblock, target, stop_check)
cancelled = (active_jobs[message_id] == True)
active_jobs.pop(message_id)
return (None if cancelled else result), rounds
def job_linux_managed(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
ctx = multiprocessing.get_context("fork")
cores = multiprocessing.cpu_count()
jobs = cores if cores <= 12 else int(cores/2)
allow_kill = True
stamp = None
total_rounds = 0
with worker_context(ctx, stamp_cost, workblock, message_id) as (stop_event, result_queue, rounds_queue, job_procs):
RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG)
def job(stop_event, sc, wb, pn, mode):
target = target_from_fp(sc)
if mode == MODE_STAMP: result, rounds, _ = _loop_hash(wb, target, size, stop_event.is_set)
else: result, rounds, _ = _loop_key(wb, target, stop_event.is_set)
if not stop_event.is_set():
stop_event.set()
try: result_queue.put_nowait(result)
except: pass
try: rounds_queue.put_nowait(rounds)
except: pass
for jpn in range(jobs):
p = ctx.Process(target=job, args=(stop_event, stamp_cost, workblock, jpn, mode), daemon=True)
job_procs.append(p)
p.start()
try: stamp = result_queue.get(timeout=None)
except Exception as e:
RNS.log(f"Failed to get result from workers: {e}", RNS.LOG_ERROR)
stamp = None
# Collect any potential spurious
# results from worker queue.
try:
while True: result_queue.get_nowait()
except: pass
for j in range(jobs):
nrounds = 0
try:
nrounds = rounds_queue.get(timeout=2)
except Exception as e:
RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR)
total_rounds += nrounds
all_exited = False
exit_timeout = time.time() + 5
while time.time() < exit_timeout:
if not any(p.is_alive() for p in job_procs):
all_exited = True
break
time.sleep(0.005)
if not all_exited:
RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR)
if allow_kill:
for j in range(jobs):
process = job_procs[j]
process.kill()
else:
return None
else:
for j in range(jobs):
process = job_procs[j]
process.join()
# RNS.log(f"Joined {j} / {process}", RNS.LOG_DEBUG) # TODO: Remove
return stamp, total_rounds
def job_linux(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
allow_kill = True
stamp = None
total_rounds = 0
cores = multiprocessing.cpu_count()
jobs = cores if cores <= 12 else int(cores/2)
stop_event = multiprocessing.Event()
result_queue = multiprocessing.Queue(1)
rounds_queue = multiprocessing.Queue()
def job(stop_event, pn, sc, wb, mode):
target = target_from_fp(sc)
if mode == MODE_STAMP: result, rounds, _ = _loop_hash(wb, target, size, stop_event.is_set)
else: result, rounds, _ = _loop_key(wb, target, stop_event.is_set)
if not stop_event.is_set():
stop_event.set()
result_queue.put(result)
rounds_queue.put(rounds)
job_procs = []
RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG)
for jpn in range(jobs):
process = multiprocessing.get_context("fork").Process(target=job, kwargs={"stop_event": stop_event, "pn": jpn, "sc": stamp_cost, "wb": workblock, "mode": mode}, daemon=True)
job_procs.append(process)
process.start()
active_jobs[message_id] = [stop_event, result_queue]
try:
stamp = result_queue.get()
# Collect any potential spurious
# results from worker queue.
try:
while True: result_queue.get_nowait()
except: pass
for j in range(jobs):
nrounds = 0
try:
nrounds = rounds_queue.get(timeout=2)
except Exception as e:
RNS.log(f"Failed to get round stats part {j}: {e}", RNS.LOG_ERROR)
total_rounds += nrounds
all_exited = False
exit_timeout = time.time() + 5
while time.time() < exit_timeout:
if not any(p.is_alive() for p in job_procs):
all_exited = True
break
time.sleep(0.005)
if not all_exited:
RNS.log("Stamp generation IPC timeout, possible worker deadlock. Terminating remaining processes.", RNS.LOG_ERROR)
if allow_kill:
for j in range(jobs):
process = job_procs[j]
process.kill()
else:
return None
else:
for j in range(jobs):
process = job_procs[j]
process.join()
except Exception as e: RNS.log(f"Error while getting stamp generation result: {e}", RNS.LOG_ERROR)
finally: active_jobs.pop(message_id, None)
return stamp, total_rounds
ANDROID_PREFER_NACL = True
ANDROID_USE_PERSISTENT = True
def job_android(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
if ANDROID_USE_PERSISTENT: return job_android_persistent(stamp_cost, workblock, message_id, size, mode)
else: return job_android_legacy(stamp_cost, workblock, message_id, size, mode)
def job_android_legacy(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
# Semaphore support is flaky to non-existent on
# Android, so we need to manually dispatch and
# manage workloads here, while periodically
# checking in on the progress.
stamp = None
start_time = time.time()
total_rounds = 0
rounds_per_worker = 1000
use_nacl = False
try:
if ANDROID_PREFER_NACL:
import nacl.encoding
import nacl.hash
use_nacl = True
except: pass
if use_nacl:
def full_hash(m): return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder)
else:
def full_hash(m): return RNS.Identity.full_hash(m)
wm = multiprocessing.Manager()
jobs = multiprocessing.cpu_count()
def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None, mode=MODE_STAMP):
try:
found_result = None
target = target_from_fp(sc)
if mode == MODE_STAMP: result, rounds, found = _loop_hash(wb, target, size, _never, None, jr, full_hash)
else: result, rounds, found = _loop_key(wb, target, _never, jr)
if found: found_result = result
results_dict[procnum] = [found_result, rounds]
except Exception as e:
RNS.log(f"Stamp generation worker error: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
active_jobs[message_id] = False;
RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
results_dict = wm.dict()
while stamp == None and active_jobs[message_id] == False:
job_procs = []
try:
for pnum in range(jobs):
pargs = {"procnum":pnum, "results_dict": results_dict, "wb": workblock, "sc":stamp_cost, "jr":rounds_per_worker, "mode":mode}
process = multiprocessing.Process(target=job, kwargs=pargs)
job_procs.append(process)
process.start()
for process in job_procs:
process.join()
for j in results_dict:
r = results_dict[j]
total_rounds += r[1]
if r[0] != None:
stamp = r[0]
if stamp == None:
elapsed = max(time.time()-start_time, 1E-6)
speed = total_rounds/elapsed
RNS.log(f"Stamp generation running. {total_rounds} rounds completed so far, {RNS.prettysize(speed, suffix='rps')}", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
except Exception as e:
RNS.log(f"Stamp generation job error: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
active_jobs.pop(message_id)
return stamp, total_rounds
def job_android_persistent(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
stamp = None
start_time = time.time()
total_rounds = 0
use_nacl = False
try:
if ANDROID_PREFER_NACL:
import nacl.encoding
import nacl.hash
use_nacl = True
except: pass
if use_nacl:
def full_hash(m): return nacl.hash.sha256(m, encoder=nacl.encoding.RawEncoder)
else:
def full_hash(m): return RNS.Identity.full_hash(m)
ctx = multiprocessing.get_context("fork")
wm = ctx.Manager()
try: jobs = len(os.sched_getaffinity(0))
except Exception: jobs = multiprocessing.cpu_count()
results_dict = wm.dict()
stop_flag = mmap.mmap(-1, 1, flags=mmap.MAP_SHARED | getattr(mmap, "MAP_ANONYMOUS", 0))
active_jobs[message_id] = False
RNS.log(f"Dispatching {jobs} workers for stamp generation...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
def job(worker_id):
rounds = 0
found_result = None
try:
target = target_from_fp(stamp_cost)
stop_check = lambda: stop_flag[0] != 0
if mode == MODE_STAMP: result, rounds, found = _loop_hash(workblock, target, size, stop_check, None, 0, full_hash)
else: result, rounds, found = _loop_key(workblock, target, stop_check, 0)
if found:
found_result = result
results_dict["found"] = [found_result, rounds]
stop_flag[0] = 1
results_dict[worker_id] = [found_result, rounds]
except Exception as e:
RNS.log(f"Stamp generation worker error: {e}", RNS.LOG_ERROR)
RNS.trace_exception(e)
try: results_dict[worker_id] = [None, rounds]
except Exception: pass
job_procs = []
try:
for pnum in range(jobs):
process = ctx.Process(target=job, kwargs={"worker_id": pnum}, daemon=True)
job_procs.append(process)
process.start()
last_progress = time.time()
while stamp == None and active_jobs.get(message_id) == False:
if "found" in results_dict: stamp = results_dict["found"][0]
if stamp == None:
if not any(p.is_alive() for p in job_procs): break
if time.time()-last_progress > 10:
elapsed = max(time.time()-start_time, 1E-6)
RNS.log(f"Stamp ({cost_from_fp(stamp_cost)}) generation running, {RNS.prettyshorttime(elapsed)} elapsed", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
last_progress = time.time()
time.sleep(0.05)
finally:
stop_flag[0] = 1
for process in job_procs:
try: process.join(timeout=2)
except Exception: pass
for worker_id in list(results_dict.keys()):
try:
if worker_id == "found": continue
r = results_dict[worker_id]
if r != None: total_rounds += r[1]
except Exception: pass
active_jobs.pop(message_id, None)
try: wm.shutdown()
except Exception: pass
return stamp, total_rounds
#######################
# Fixed-point Helpers #
#######################
# Fixed-point stamp target mantissa table
# TAB[f] = floor(2^33 * 2^(f/256)), f = 0..255
TAB_DIGEST = "55ba3ce1df224895b0dfdc102eb4359fc2f5a7efefee68df0cc90c8a8ac24f48"
PRM_DIGEST = "21436fc98e76a12cad7c52ebeb6ab76c2ae93b4cd768949df7fdd4e2f98c378d"
MANTISSA_BITS = 33
COST_FP_BITS = 8
COST_FP = 1 << COST_FP_BITS
# These values are authoritative. DO NOT attempt to
# invent your own, calculate them at load time, or
# generate them on the fly. You have been warned.
TAB = (
0x200000000, 0x201635f4b, 0x202c7b53f, 0x2042d0287, 0x2059347ce, 0x206fa85c2,
0x20862bd0d, 0x209cbee5e, 0x20b361a62, 0x20ca141c7, 0x20e0d653b, 0x20f7a856e,
0x210e8a30e, 0x21257becc, 0x213c7d958, 0x21538f363, 0x216ab0d9f, 0x2181e28bc,
0x21992456e, 0x21b076467, 0x21c7d865a, 0x21df4abfb, 0x21f6cd5ff, 0x220e6051a,
0x222603a02, 0x223db756b, 0x22557b80d, 0x226d5029e, 0x2285355d5, 0x229d2b269,
0x22b531914, 0x22cd48a8e, 0x22e57078f, 0x22fda90d2, 0x2315f2711, 0x232e4cb06,
0x2346b7d6d, 0x235f33f02, 0x2377c1080, 0x23905f2a5, 0x23a90e62d, 0x23c1cebd6,
0x23daa045f, 0x23f383087, 0x240c7710c, 0x24257c6af, 0x243e9322f, 0x2457bb44f,
0x2470f4dce, 0x248a3ff70, 0x24a39c9f6, 0x24bd0ae23, 0x24d68acbc, 0x24f01c683,
0x2509bfc3e, 0x252374eb2, 0x253d3bea3, 0x255714cda, 0x2570ffa1b, 0x258afc72e,
0x25a50b4dc, 0x25bf2c3ec, 0x25d95f527, 0x25f3a4957, 0x260dfc146, 0x262865dbd,
0x2642e1f89, 0x265d70775, 0x26781164c, 0x2692c4cdc, 0x26ad8abf2, 0x26c86345b,
0x26e34e6e7, 0x26fe4c463, 0x27195cda0, 0x27348036e, 0x274fb669c, 0x276aff7fd,
0x27865b862, 0x27a1ca89d, 0x27bd4c982, 0x27d8e1be3, 0x27f48a095, 0x28104586c,
0x282c1443e, 0x2847f64e1, 0x2863ebb2a, 0x287ff47f0, 0x289c10c0c, 0x28b840854,
0x28d483da3, 0x28f0dacd1, 0x290d456b8, 0x2929c3c32, 0x294655e1a, 0x2962fbd4d,
0x297fb5aa6, 0x299c83702, 0x29b96533f, 0x29d65b03b, 0x29f364ed3, 0x2a1082fe8,
0x2a2db5459, 0x2a4afbd07, 0x2a6856ad3, 0x2a85c5e9e, 0x2aa34994b, 0x2ac0e1bbd,
0x2ade8e6d6, 0x2afc4fb7c, 0x2b1a25a92, 0x2b38104ff, 0x2b560fba9, 0x2b7423f75,
0x2b924d14b, 0x2bb08b213, 0x2bcede2b5, 0x2bed4641b, 0x2c0bc372e, 0x2c2a55cd9,
0x2c48fd607, 0x2c67ba3a3, 0x2c868c699, 0x2ca573fd7, 0x2cc47104a, 0x2ce3838e1,
0x2d02aba89, 0x2d21e9633, 0x2d413cccf, 0x2d60a5f4e, 0x2d8024ea1, 0x2d9fb9bba,
0x2dbf6478c, 0x2ddf2530b, 0x2dfefbf2a, 0x2e1ee8cde, 0x2e3eebd1d, 0x2e5f050dd,
0x2e7f34914, 0x2e9f7a6ba, 0x2ebfd6ac8, 0x2ee049635, 0x2f00d29fb, 0x2f2172715,
0x2f4228e7d, 0x2f62f612e, 0x2f83da026, 0x2fa4d4c5f, 0x2fc5e66d9, 0x2fe70f092,
0x30084ea87, 0x3029a55ba, 0x304b13329, 0x306c983d7, 0x308e348c4, 0x30afe82f3,
0x30d1b3368, 0x30f395b26, 0x31158fb31, 0x3137a148f, 0x3159ca845, 0x317c0b75a,
0x319e642d6, 0x31c0d4bc1, 0x31e35d322, 0x3205fda05, 0x3228b6172, 0x324b86a75,
0x326e6f619, 0x32917056b, 0x32b489979, 0x32d7bb34e, 0x32fb053fb, 0x331e67c8f,
0x3341e2e19, 0x3365769aa, 0x338923054, 0x33ace8329, 0x33d0c633c, 0x33f4bd1a0,
0x3418ccf6b, 0x343cf5db1, 0x346137d89, 0x348593008, 0x34aa07647, 0x34ce9515e,
0x34f33c266, 0x3517fca78, 0x353cd6aaf, 0x3561ca426, 0x3586d77fa, 0x35abfe747,
0x35d13f32b, 0x35f699cc4, 0x361c0e531, 0x36419cd93, 0x366745709, 0x368d082b6,
0x36b2e51bc, 0x36d8dc53e, 0x36feede5f, 0x372519e44, 0x374b60614, 0x3771c16f3,
0x37983d209, 0x37bed387e, 0x37e584b7a, 0x380c50c28, 0x383337bb0, 0x385a39b3f,
0x388156bff, 0x38a88ef1f, 0x38cfe25ca, 0x38f751131, 0x391edb280, 0x394680aea,
0x396e41b9d, 0x39961e5cd, 0x39be16aab, 0x39e62ab6b, 0x3a0e5a940, 0x3a36a6561,
0x3a5f0e101, 0x3a8791d59, 0x3ab031b9f, 0x3ad8edd0c, 0x3b01c62d9, 0x3b2abae3f,
0x3b53cc07b, 0x3b7cf9ac7, 0x3ba643e60, 0x3bcfaac83, 0x3bf92e66f, 0x3c22ced63,
0x3c4c8c29e, 0x3c7666762, 0x3ca05dcf1, 0x3cca7248c, 0x3cf4a3f78, 0x3d1ef2ef9,
0x3d495f454, 0x3d73e90cf, 0x3d9e905b1, 0x3dc955443, 0x3df437dcc, 0x3e1f38396,
0x3e4a566ed, 0x3e759291b, 0x3ea0ecb6d, 0x3ecc64f31, 0x3ef7fb5b3, 0x3f23b0044,
0x3f4f83033, 0x3f7b746d2, 0x3fa784571, 0x3fd3b2d65 )
assert len(TAB) == COST_FP
assert TAB[0] == 1 << MANTISSA_BITS
assert all(b > a for a, b in zip(TAB, TAB[1:]))
assert sha256(",".join(str(v) for v in TAB).encode("ascii")) == bytes.fromhex(TAB_DIGEST)
assert sha256(f"master|{TAB_DIGEST}|{MANTISSA_BITS}|{len(TAB)}|{COST_FP_BITS}|{COST_FP}|".encode("utf-8")) == bytes.fromhex(PRM_DIGEST)
def cost_to_fp(cost):
v = math.floor(float(cost) * COST_FP + 0.5)
return max(0, min(v, RNS.Transport.STAMP_COST_CAP * COST_FP))
def fp_to_bytes(fp):
return bytes(((fp >> COST_FP_BITS) & 0xFF, fp & 0xFF))
def cost_to_bytes(cost):
return fp_to_bytes(cost_to_fp(cost))
def fp_from_bytes(b):
return (b[0] << COST_FP_BITS) | b[1]
def cost_from_fp(fp):
return fp/COST_FP
def cost_from_bytes(b):
return cost_from_fp(fp_from_bytes(b))
def quantize_cost(cost):
return cost_from_fp(cost_to_fp(cost))
def target_from_fp(fp):
assert 0 <= fp <= RNS.Transport.STAMP_COST_CAP * COST_FP
# Extract the integer and fractional parts
# by modular division, here equivalent to:
# ((256*COST_FP) // COST_FP, (256*COST_FP) % COST_FP)
# i = quotient f = remainder
i, f = divmod(256*COST_FP - fp, COST_FP)
# Then, construct the numerical representation
# by getting the quantized fractional mantissa
# from TAB, and raising it to i (the exponent)
# and right-shifting it by MANTISSA_BITS.
return (TAB[f] << i) >> MANTISSA_BITS
def target_from_cost(cost):
return target_from_fp(cost_to_fp(cost))
def cost_int(fp): return fp >> COST_FP_BITS # Legacy integer display representation
def cost_str(fp): return f"{fp / COST_FP:g}" # Display only, remember, the floats are only in your head
###########################
# Persistence and loading #
###########################
DESTINATION_STAMP_EXTENSION = ".dstamp"
STAMP_FILE_VERSION = 1
def stamps_dir(create=False):
storagepath = RNS.Reticulum.storagepath
if not storagepath: return None
path = os.path.join(storagepath, "stamps")
if create and not os.path.isdir(path):
try: os.makedirs(path, exist_ok=True)
except Exception as e:
RNS.log(f"An error occurred while creating destination stamp storage path: {e}", RNS.LOG_WARNING)
return None
return path
def destination_stamp_file(destination_hash, create=False):
dir_path = stamps_dir(create=create)
if dir_path == None: return None
return os.path.join(dir_path, RNS.hexrep(destination_hash, delimit=False)+DESTINATION_STAMP_EXTENSION)
def store_destination_stamp(destination_hash, stamp, stamp_value):
file_path = destination_stamp_file(destination_hash, create=True)
if file_path == None:
RNS.log("No storage path available, destination stamp not persisted", RNS.LOG_WARNING)
return False
try:
data = msgpack.packb({"v": STAMP_FILE_VERSION, "stamp": stamp,
"value": stamp_value, "generated": time.time()})
temp_path = file_path+f".{os.getpid()}.tmp"
with open(temp_path, "wb") as temp_file: temp_file.write(data)
os.replace(temp_path, file_path)
return True
except Exception as e:
RNS.log(f"An error occurred while persisting destination stamp: {e}", RNS.LOG_WARNING)
return False
def load_destination_stamp(destination_hash, material, required_value):
file_path = destination_stamp_file(destination_hash)
if file_path == None or not os.path.isfile(file_path): return None
try:
with open(file_path, "rb") as stamp_file: entry = msgpack.unpackb(stamp_file.read())
if entry.get("v") != STAMP_FILE_VERSION: return None
workblock = stamp_workblock(material, expand_rounds=0)
stamp = entry["stamp"]
if not isinstance(stamp, bytes) or len(stamp) != RNS.Identity.STAMPLENGTH:
RNS.log(f"Stored stamp for {RNS.prettyhexrep(destination_hash)} has invalid format. Regenerating on next announce.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
value = stamp_value(workblock, stamp)
if not stamp_valid(stamp, required_value, workblock):
RNS.log(f"Stored stamp for {RNS.prettyhexrep(destination_hash)} has value {value}, below required {required_value}. Regenerating on next announce.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
return (stamp, value)
except Exception as e:
RNS.log(f"Ignoring unreadable destination stamp file for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
return None
#########################
# Pooling Configuration #
#########################
POOLING_THRESHOLD_COST_FP = POOLING_THRESHOLD_COST*COST_FP
KEYSTAMP_POOLING_THRESHOLD_FP = KEYSTAMP_POOLING_THRESHOLD*COST_FP