Implemented fractional-value stamps

This commit is contained in:
Mark Qvist 2026-10-29 16:49:39 +01:00
parent c0fe08c08d
commit 78fb90ea6f
No known key found for this signature in database
3 changed files with 692 additions and 19 deletions

View file

@ -30,6 +30,7 @@
import RNS
import RNS.vendor.umsgpack as msgpack
from RNS.Cryptography import sha256
import os
import sys
@ -39,15 +40,15 @@ import mmap
import itertools
import contextlib
import multiprocessing
from bisect import bisect_right
WORKBLOCK_EXPAND_ROUNDS = 3000
STAMP_SIZE = 32
PN_VALIDATION_POOL_MIN_SIZE = 256
USE_WORKER_MANAGER = False
POOLING_THRESHOLD_COST = 15
STOP_CHECK_INTERVAL = 64
KEYSTAMP_POOLING_THRESHOLD = 12
STOP_CHECK_INTERVAL = 64
MODE_STAMP = 0x00
MODE_KEYSTAMP = 0x01
@ -94,14 +95,101 @@ def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=F
return workblock
def stamp_value(workblock, stamp):
def stamp_value_legacy(workblock, stamp):
if not workblock: workblock = b""
material = RNS.Identity.full_hash(workblock+stamp)
return 256 - int.from_bytes(material, byteorder="big").bit_length()
stamp_product = RNS.Identity.full_hash(workblock+stamp)
return 256 - int.from_bytes(stamp_product, byteorder="big").bit_length()
def stamp_value(workblock, stamp):
# This function is very little code, and a whole lot of
# comments. The actual method employed here is relatively
# simple, but without proper explanation, it is also very
# opaque, and could induce a lot of head-scratching.
if not workblock: workblock = b""
stamp_product = RNS.Identity.full_hash(workblock+stamp) # Bytes representation
hn = int.from_bytes(stamp_product, byteorder="big") # Numerical representation
if hn == 0: return 256 # Yeah, but *if* it happens, you know...
nb = MANTISSA_BITS+1 # Normalized mantissa bit count
bl = hn.bit_length() # Bit length of the stamp product
lz = 256 - bl # Leading zeros of the stamp product
##############################################################
# #
# Simplified mantissa extraction example with #
# smaller numbers; 32-bit stamp product and 8 #
# mantissa bits. In the actual version, it is #
# 256-bit stamp products and 33-bit mantissa. #
# #
# mb = 8 nb = 9 (8+1) #
# bl = 27 lz = 5 (32-27) #
# #
# shift = bl - nb = 18 #
# #
# hn = 0b00000111101101101110111010110111 #
# >>18 0b00000000000000000000000111101101|101110111010110111 #
# | \_____ gone #
# The leading one | #
# 8 significand mantissa bits #
# #
# mn = 0b111101101 (normalized with the leading 1) #
# #
##############################################################
# Normalize mantissa to the scale of TAB: Total of 34
# bits, 33 mantissa bits. The leading one required to
# match TAB entries is guaranteed by the construction:
# Since we're shifting by the *difference* between the
# normalized mantissa bit length and the stamp product
# bit length, we always end up with a leading one, and
# then the required number of *actual* mantissa bits.
# See the example above for a simplified illustration.
if bl > nb: mn = (hn >> (bl - nb))
else: mn = (hn << (nb - bl)) # Again, better safe than sorry. You may notice
# here that in this case, we will have a more
# coarse mantissa part, which is inherent to
# the methodology, but in practice the best thing
# we can do. Additionally, this can only happen
# at absolutely extreme values, far above the
# stamp cap anyway. It is only here to handle the
# absurd case where a stamp product randomly lands
# on an astronomically improbable leading zero count.
# We can now construct the fixed-point value representation:
#
# The leading zero count is equivalent to the legacy value, an
# integer representation. Thus, (lz+1)<<COST_FP_BITS gives us
# the fixed-point representation of a value one greater than
# what we currently have. Since the mantissa is normalized, the
# real value always lies somewhere in between, at the ceiling
# minus however far the mantissa has climbed into this range
#
# Right-bisecting TAB with the normalized mantissa returns the
# number of grid points at or below it, and since the table
# values *are* the normalized mantissas of the grid points in
# our 256-step quantization, this count is the climb, quantized
# in 1/256th steps. The *count* always overshoots the *actual*
# climb (one full step on an exact match), so subtracting it
# from the ceiling *floors* the value onto the grid, so we
# never over-value the stamp.
#
# Note that for a mantissa *past* (or equal to) the last table
# entry, there is no special case handling required, since the
# bisect_right count returned in that case is 256 (the largest
# it can ever return). Thus, the arithmetic itself here folds
# correctly: The entire "added one" (lz+1) is consumed, and can
# never over-consume. Remember: In the fixed-point representation
# a value of 1.0 *is* 256 (and, as a side note, this is one of
# the reasons that len(TAB)==COST_FP is asserted as invariant).
value_fp = ((lz + 1) << COST_FP_BITS) - bisect_right(TAB, mn)
# Then, convert to decimal representation and return it
return cost_from_fp(value_fp)
def stamp_valid(stamp, target_cost, workblock):
if not workblock: workblock = b""
target = 0b1 << 256-target_cost
target = target_from_fp(cost_to_fp(target_cost))
result = RNS.Identity.full_hash(workblock+stamp)
if int.from_bytes(result, byteorder="big") > target: return False
else: return True
@ -163,13 +251,13 @@ def _loop_key(material, target, stop_check, max_rounds=0):
# bytes material (message_id), which is usually a hash
# of some input material, but can be any sequence of bytes.
def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=False, size=STAMP_SIZE):
stamp_cost = int(stamp_cost)
if stamp_cost < 0 or stamp_cost > RNS.Transport.STAMP_COST_CAP: raise ValueError("Invalid stamp cost, cannot generate")
workblock = stamp_workblock(message_id, expand_rounds=expand_rounds, no_expand=no_expand)
if message_id in active_jobs: raise RuntimeError(f"Duplicate stamp generation job for {RNS.prettyhexrep(message_id)}")
RNS.log(f"Generating stamp with cost {stamp_cost} for <{RNS.hexrep(message_id[:16], delimit=False)}{'…' if len(message_id) > 16 else ''}>", RNS.LOG_DEBUG)
stamp_cost = cost_to_fp(stamp_cost)
start_time = time.time()
stamp = None
rounds = 0
@ -181,8 +269,8 @@ def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id, size)
elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id, size)
else:
if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST else job_simple(stamp_cost, workblock, message_id, size)
else: stamp, rounds = job_linux(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST else job_simple(stamp_cost, workblock, message_id, size)
if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST_FP else job_simple(stamp_cost, workblock, message_id, size)
else: stamp, rounds = job_linux(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST_FP else job_simple(stamp_cost, workblock, message_id, size)
duration = max(time.time()-start_time, 1E-6)
speed = rounds/duration
@ -226,12 +314,12 @@ def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS
# this function must ONLY be used as X25519 keys.
def generate_stamped_key(material, stamp_cost):
if material == None: material = b""
stamp_cost = int(stamp_cost)
if material in active_jobs: raise RuntimeError(f"Duplicate stamped key generation job for {RNS.prettyhexrep(material)}")
if stamp_cost < 0 or stamp_cost > RNS.Transport.STAMP_COST_CAP: raise ValueError("Invalid stamp cost, cannot generate")
for_str = f" for <{RNS.hexrep(material[:16], delimit=False)}{'…' if len(material) > 16 else ''}>" if material else ""
RNS.log(f"Generating stamped key with cost {stamp_cost}{for_str}", RNS.LOG_DEBUG)
stamp_cost = cost_to_fp(stamp_cost)
start_time = time.time()
keypair = None
rounds = 0
@ -243,8 +331,8 @@ def generate_stamped_key(material, stamp_cost):
if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): keypair, rounds = job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
elif RNS.vendor.platformutils.is_android(): keypair, rounds = job_android(stamp_cost, material, material, 0, MODE_KEYSTAMP)
else:
if USE_WORKER_MANAGER: keypair, rounds = job_linux_managed(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
else: keypair, rounds = job_linux(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
if USE_WORKER_MANAGER: keypair, rounds = job_linux_managed(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD_FP else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
else: keypair, rounds = job_linux(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD_FP else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP)
duration = max(time.time()-start_time, 1E-6)
speed = rounds/duration
@ -300,7 +388,7 @@ def job_simple(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
# size argument is ignored.
st = time.time()
target = 0b1<<256-stamp_cost
target = target_from_fp(stamp_cost)
active_jobs[message_id] = False
@ -329,7 +417,7 @@ def job_linux_managed(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG)
def job(stop_event, sc, wb, pn, mode):
target = 0b1<<256 - sc
target = target_from_fp(sc)
if mode == MODE_STAMP: result, rounds, _ = _loop_hash(wb, target, size, stop_event.is_set)
else: result, rounds, _ = _loop_key(wb, target, stop_event.is_set)
@ -402,7 +490,7 @@ def job_linux(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
rounds_queue = multiprocessing.Queue()
def job(stop_event, pn, sc, wb, mode):
target = 0b1<<256 - sc
target = target_from_fp(sc)
if mode == MODE_STAMP: result, rounds, _ = _loop_hash(wb, target, size, stop_event.is_set)
else: result, rounds, _ = _loop_key(wb, target, stop_event.is_set)
@ -495,7 +583,7 @@ def job_android(stamp_cost, workblock, message_id, size, mode=MODE_STAMP):
def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None, mode=MODE_STAMP):
try:
found_result = None
target = 0b1<<256-sc
target = target_from_fp(sc)
if mode == MODE_STAMP: result, rounds, found = _loop_hash(wb, target, size, _never, None, jr, full_hash)
else: result, rounds, found = _loop_key(wb, target, _never, jr)
@ -575,7 +663,7 @@ def job_android_persistent(stamp_cost, workblock, message_id, size, mode=MODE_ST
rounds = 0
found_result = None
try:
target = 0b1<<256-stamp_cost
target = target_from_fp(stamp_cost)
stop_check = lambda: stop_flag[0] != 0
if mode == MODE_STAMP: result, rounds, found = _loop_hash(workblock, target, size, stop_check, None, 0, full_hash)
@ -634,6 +722,111 @@ def job_android_persistent(stamp_cost, workblock, message_id, size, mode=MODE_ST
return stamp, total_rounds
#######################
# Fixed-point Helpers #
#######################
# Fixed-point stamp target mantissa table
# TAB[f] = floor(2^33 * 2^(f/256)), f = 0..255
TAB_DIGEST = "55ba3ce1df224895b0dfdc102eb4359fc2f5a7efefee68df0cc90c8a8ac24f48"
PRM_DIGEST = "21436fc98e76a12cad7c52ebeb6ab76c2ae93b4cd768949df7fdd4e2f98c378d"
MANTISSA_BITS = 33
COST_FP_BITS = 8
COST_FP = 1 << COST_FP_BITS
# These values are authoritative. DO NOT attempt to
# invent your own, calculate them at load time, or
# generate them on the fly. You have been warned.
TAB = (
0x200000000, 0x201635f4b, 0x202c7b53f, 0x2042d0287, 0x2059347ce, 0x206fa85c2,
0x20862bd0d, 0x209cbee5e, 0x20b361a62, 0x20ca141c7, 0x20e0d653b, 0x20f7a856e,
0x210e8a30e, 0x21257becc, 0x213c7d958, 0x21538f363, 0x216ab0d9f, 0x2181e28bc,
0x21992456e, 0x21b076467, 0x21c7d865a, 0x21df4abfb, 0x21f6cd5ff, 0x220e6051a,
0x222603a02, 0x223db756b, 0x22557b80d, 0x226d5029e, 0x2285355d5, 0x229d2b269,
0x22b531914, 0x22cd48a8e, 0x22e57078f, 0x22fda90d2, 0x2315f2711, 0x232e4cb06,
0x2346b7d6d, 0x235f33f02, 0x2377c1080, 0x23905f2a5, 0x23a90e62d, 0x23c1cebd6,
0x23daa045f, 0x23f383087, 0x240c7710c, 0x24257c6af, 0x243e9322f, 0x2457bb44f,
0x2470f4dce, 0x248a3ff70, 0x24a39c9f6, 0x24bd0ae23, 0x24d68acbc, 0x24f01c683,
0x2509bfc3e, 0x252374eb2, 0x253d3bea3, 0x255714cda, 0x2570ffa1b, 0x258afc72e,
0x25a50b4dc, 0x25bf2c3ec, 0x25d95f527, 0x25f3a4957, 0x260dfc146, 0x262865dbd,
0x2642e1f89, 0x265d70775, 0x26781164c, 0x2692c4cdc, 0x26ad8abf2, 0x26c86345b,
0x26e34e6e7, 0x26fe4c463, 0x27195cda0, 0x27348036e, 0x274fb669c, 0x276aff7fd,
0x27865b862, 0x27a1ca89d, 0x27bd4c982, 0x27d8e1be3, 0x27f48a095, 0x28104586c,
0x282c1443e, 0x2847f64e1, 0x2863ebb2a, 0x287ff47f0, 0x289c10c0c, 0x28b840854,
0x28d483da3, 0x28f0dacd1, 0x290d456b8, 0x2929c3c32, 0x294655e1a, 0x2962fbd4d,
0x297fb5aa6, 0x299c83702, 0x29b96533f, 0x29d65b03b, 0x29f364ed3, 0x2a1082fe8,
0x2a2db5459, 0x2a4afbd07, 0x2a6856ad3, 0x2a85c5e9e, 0x2aa34994b, 0x2ac0e1bbd,
0x2ade8e6d6, 0x2afc4fb7c, 0x2b1a25a92, 0x2b38104ff, 0x2b560fba9, 0x2b7423f75,
0x2b924d14b, 0x2bb08b213, 0x2bcede2b5, 0x2bed4641b, 0x2c0bc372e, 0x2c2a55cd9,
0x2c48fd607, 0x2c67ba3a3, 0x2c868c699, 0x2ca573fd7, 0x2cc47104a, 0x2ce3838e1,
0x2d02aba89, 0x2d21e9633, 0x2d413cccf, 0x2d60a5f4e, 0x2d8024ea1, 0x2d9fb9bba,
0x2dbf6478c, 0x2ddf2530b, 0x2dfefbf2a, 0x2e1ee8cde, 0x2e3eebd1d, 0x2e5f050dd,
0x2e7f34914, 0x2e9f7a6ba, 0x2ebfd6ac8, 0x2ee049635, 0x2f00d29fb, 0x2f2172715,
0x2f4228e7d, 0x2f62f612e, 0x2f83da026, 0x2fa4d4c5f, 0x2fc5e66d9, 0x2fe70f092,
0x30084ea87, 0x3029a55ba, 0x304b13329, 0x306c983d7, 0x308e348c4, 0x30afe82f3,
0x30d1b3368, 0x30f395b26, 0x31158fb31, 0x3137a148f, 0x3159ca845, 0x317c0b75a,
0x319e642d6, 0x31c0d4bc1, 0x31e35d322, 0x3205fda05, 0x3228b6172, 0x324b86a75,
0x326e6f619, 0x32917056b, 0x32b489979, 0x32d7bb34e, 0x32fb053fb, 0x331e67c8f,
0x3341e2e19, 0x3365769aa, 0x338923054, 0x33ace8329, 0x33d0c633c, 0x33f4bd1a0,
0x3418ccf6b, 0x343cf5db1, 0x346137d89, 0x348593008, 0x34aa07647, 0x34ce9515e,
0x34f33c266, 0x3517fca78, 0x353cd6aaf, 0x3561ca426, 0x3586d77fa, 0x35abfe747,
0x35d13f32b, 0x35f699cc4, 0x361c0e531, 0x36419cd93, 0x366745709, 0x368d082b6,
0x36b2e51bc, 0x36d8dc53e, 0x36feede5f, 0x372519e44, 0x374b60614, 0x3771c16f3,
0x37983d209, 0x37bed387e, 0x37e584b7a, 0x380c50c28, 0x383337bb0, 0x385a39b3f,
0x388156bff, 0x38a88ef1f, 0x38cfe25ca, 0x38f751131, 0x391edb280, 0x394680aea,
0x396e41b9d, 0x39961e5cd, 0x39be16aab, 0x39e62ab6b, 0x3a0e5a940, 0x3a36a6561,
0x3a5f0e101, 0x3a8791d59, 0x3ab031b9f, 0x3ad8edd0c, 0x3b01c62d9, 0x3b2abae3f,
0x3b53cc07b, 0x3b7cf9ac7, 0x3ba643e60, 0x3bcfaac83, 0x3bf92e66f, 0x3c22ced63,
0x3c4c8c29e, 0x3c7666762, 0x3ca05dcf1, 0x3cca7248c, 0x3cf4a3f78, 0x3d1ef2ef9,
0x3d495f454, 0x3d73e90cf, 0x3d9e905b1, 0x3dc955443, 0x3df437dcc, 0x3e1f38396,
0x3e4a566ed, 0x3e759291b, 0x3ea0ecb6d, 0x3ecc64f31, 0x3ef7fb5b3, 0x3f23b0044,
0x3f4f83033, 0x3f7b746d2, 0x3fa784571, 0x3fd3b2d65 )
assert len(TAB) == COST_FP
assert TAB[0] == 1 << MANTISSA_BITS
assert all(b > a for a, b in zip(TAB, TAB[1:]))
assert sha256(",".join(str(v) for v in TAB).encode("ascii")) == bytes.fromhex(TAB_DIGEST)
assert sha256(f"master|{TAB_DIGEST}|{MANTISSA_BITS}|{len(TAB)}|{COST_FP_BITS}|{COST_FP}|".encode("utf-8")) == bytes.fromhex(PRM_DIGEST)
def cost_to_fp(cost):
v = math.floor(float(cost) * COST_FP + 0.5)
return max(0, min(v, RNS.Transport.STAMP_COST_CAP * COST_FP))
def fp_to_bytes(fp):
return bytes(((fp >> COST_FP_BITS) & 0xFF, fp & 0xFF))
def cost_to_bytes(cost):
return fp_to_bytes(cost_to_fp(cost))
def fp_from_bytes(b):
return (b[0] << COST_FP_BITS) | b[1]
def cost_from_fp(fp):
return fp/COST_FP
def cost_from_bytes(b):
return cost_from_fp(fp_from_bytes(b))
def target_from_fp(fp):
assert 0 <= fp <= RNS.Transport.STAMP_COST_CAP * COST_FP
# Extract the integer and fractional parts
# by modular division, here equivalent to:
# ((256*COST_FP) // COST_FP, (256*COST_FP) % COST_FP)
# i = quotient f = remainder
i, f = divmod(256*COST_FP - fp, COST_FP)
# Then, construct the numerical representation
# by getting the quantized fractional mantissa
# from TAB, and raising it to i (the exponent)
# and right-shifting it by MANTISSA_BITS.
return (TAB[f] << i) >> MANTISSA_BITS
def target_from_cost(cost):
return target_from_fp(cost_to_fp(cost))
def cost_int(fp): return fp >> COST_FP_BITS # Legacy integer display representation
def cost_str(fp): return f"{fp / COST_FP:g}" # Display only, remember, the floats are only in your head
###########################
# Persistence and loading #
###########################
@ -691,7 +884,7 @@ def load_destination_stamp(destination_hash, material, required_value):
return None
value = stamp_value(workblock, stamp)
if value < required_value:
if not stamp_valid(stamp, required_value, workblock):
RNS.log(f"Stored stamp for {RNS.prettyhexrep(destination_hash)} has value {value}, below required {required_value}. Regenerating on next announce.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None
return None
@ -700,3 +893,10 @@ def load_destination_stamp(destination_hash, material, required_value):
except Exception as e:
RNS.log(f"Ignoring unreadable destination stamp file for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING)
return None
#########################
# Pooling Configuration #
#########################
POOLING_THRESHOLD_COST_FP = POOLING_THRESHOLD_COST*COST_FP
KEYSTAMP_POOLING_THRESHOLD_FP = KEYSTAMP_POOLING_THRESHOLD*COST_FP

View file

@ -7,6 +7,10 @@ from .link import TestLink
from .channel import TestChannel
from .hkdf import TestHKDF
from .ifac import TestIFAC
from .stamps import TestStampFixedPoint
from .stamps import TestStampWorkblock
from .stamps import TestStampGeneration
from .stamps import TestStampPersistence
from .coalesced_transmit import TestTransmitBuffer
from .hdlc import TestReceiveBuffer
from .egress import TestHWMLimiter

469
tests/stamps.py Normal file
View file

@ -0,0 +1,469 @@
# Generated with lc/glm5.3_flash_q6_k_xl
# Reviewed and approved by Mark Qvist
import unittest
import os
import time
import math
import random
import shutil
import tempfile
import threading
from unittest import skipIf
import RNS
from RNS import Stamps
import RNS.vendor.umsgpack as msgpack
# The stamper itself is intentionally seeded from os.urandom, but the
# value and target layers are pure deterministic math. Seeded test
# vectors therefore make the statistical tests exact and repeatable
# on any platform and any hash backend.
rng = random.Random(28411)
def rb(n):
return bytes(rng.getrandbits(8) for _ in range(n))
def fp_of(value):
# stamp_value returns cost_from_fp(value_fp), and division by
# COST_FP (a power of two) is exact in binary floating point,
# so the fixed-point representation can be recovered losslessly.
fp = round(value * Stamps.COST_FP)
assert fp == int(value * Stamps.COST_FP)
return fp
class TestStampFixedPoint(unittest.TestCase):
@classmethod
def setUpClass(cls):
# A shared pool of deterministic workblock/stamp samples,
# used to lock in the value layer invariants.
cls.samples = []
for _ in range(3000):
wb = rb(rng.randint(0, 48))
stamp = rb(rng.choice((RNS.Identity.STAMPLENGTH, Stamps.STAMP_SIZE)))
hn = int.from_bytes(RNS.Identity.full_hash(wb+stamp), byteorder="big")
vleg = Stamps.stamp_value_legacy(wb, stamp)
v = Stamps.stamp_value(wb, stamp)
exact = 256 - math.log2(hn)
cls.samples.append((wb, stamp, hn, vleg, v, exact))
def test_00_fixed_point_conversions(self):
cap = RNS.Transport.STAMP_COST_CAP
for cost in range(0, cap+1):
self.assertEqual(Stamps.cost_to_fp(cost), cost*Stamps.COST_FP)
self.assertEqual(Stamps.cost_from_fp(Stamps.cost_to_fp(cost)), cost)
self.assertEqual(Stamps.cost_to_fp(8.5), 8.5*Stamps.COST_FP)
self.assertEqual(Stamps.cost_from_fp(8.5*Stamps.COST_FP), 8.5)
self.assertEqual(Stamps.cost_to_fp(0.5), 128)
self.assertEqual(Stamps.cost_to_fp(-1), 0)
self.assertEqual(Stamps.cost_to_fp(cap+1), cap*Stamps.COST_FP)
for fp in list(range(0, 65536, 97)) + [0, 255, 256, 65535]:
packed = Stamps.fp_to_bytes(fp)
self.assertEqual(len(packed), 2)
self.assertEqual(Stamps.fp_from_bytes(packed), fp)
for cost in (0, 1, 4, 8.5, 64, 100):
self.assertEqual(Stamps.cost_from_bytes(Stamps.cost_to_bytes(cost)), cost)
self.assertEqual(Stamps.cost_int(8*Stamps.COST_FP + 128), 8)
self.assertEqual(Stamps.cost_str(8*Stamps.COST_FP), "8")
self.assertEqual(Stamps.cost_str(8*Stamps.COST_FP + 128), "8.5")
self.assertEqual(Stamps.cost_str(0), "0")
def test_01_target_for_integer_costs(self):
cap = RNS.Transport.STAMP_COST_CAP
for cost in range(0, cap+1):
self.assertEqual(Stamps.target_from_cost(cost), 2**(256-cost))
self.assertEqual(Stamps.target_from_fp(0), 2**256)
self.assertRaises(AssertionError, Stamps.target_from_fp, -1)
self.assertRaises(AssertionError, Stamps.target_from_fp, cap*Stamps.COST_FP + 1)
def test_02_target_monotonic_and_exact(self):
# Targets must strictly decrease as cost increases
cap = RNS.Transport.STAMP_COST_CAP
last = None
for fp in range(0, cap*Stamps.COST_FP + 1):
target = Stamps.target_from_fp(fp)
if last != None:
self.assertLess(target, last)
last = target
# Independent verification of the whole mantissa/shift
# construction against floor(2^(256 - fp/256)), computed
# with exact integer arithmetic: The 256th root of
# 2^(33*256+f), obtained by a Newton iteration, is exactly
# floor(2^(33+f/256)), and shifting by the integer part of
# the exponent yields the exact target for every step.
def iroot(n, k):
x = 1 << ((n.bit_length() + k - 1)//k)
while True:
y = ((k-1)*x + n//x**(k-1))//k
if y >= x:
return x
x = y
roots = [iroot(1 << (Stamps.MANTISSA_BITS*Stamps.COST_FP + f), Stamps.COST_FP) for f in range(Stamps.COST_FP)]
for fp in range(0, cap*Stamps.COST_FP + 1):
i, f = divmod(256*Stamps.COST_FP - fp, Stamps.COST_FP)
self.assertEqual(Stamps.target_from_fp(fp), roots[f] << (i - Stamps.MANTISSA_BITS))
def test_03_value_matches_legacy_integer_value(self):
# The compatibility invariant: old clients parse fractional
# stamps at their integer value. The legacy value function
# must therefore always be the integer floor of the new one.
for wb, stamp, hn, vleg, v, exact in self.samples:
self.assertEqual(vleg, 256 - hn.bit_length())
self.assertLessEqual(vleg, v)
self.assertLess(v, vleg + 1)
self.assertEqual(fp_of(v) // Stamps.COST_FP, vleg)
# Empty and None workblocks are handled identically
wb, stamp, hn, vleg, v, exact = self.samples[0]
self.assertEqual(Stamps.stamp_value(None, stamp), Stamps.stamp_value(b"", stamp))
self.assertEqual(Stamps.stamp_value_legacy(None, stamp), Stamps.stamp_value_legacy(b"", stamp))
self.assertTrue(Stamps.stamp_valid(stamp, 0, None))
def test_04_value_never_overreports(self):
# The reported value must never exceed the exact value of
# the stamp product, and must be within one quantisation
# step (1/256th) below it.
step = 1.0 / Stamps.COST_FP
for wb, stamp, hn, vleg, v, exact in self.samples:
self.assertLessEqual(v, exact + 1e-9)
self.assertLessEqual(exact, v + step + 1e-9)
def test_05_value_monotone_in_stamp_product(self):
# A smaller stamp product (numerically) always means a more
# rare hash, and must never yield a smaller value.
ordered = sorted(self.samples, key=lambda sample: sample[2])
last = None
for wb, stamp, hn, vleg, v, exact in ordered:
fp = fp_of(v)
if last != None:
self.assertLessEqual(fp, last)
last = fp
def test_06_validity_boundaries(self):
cap = RNS.Transport.STAMP_COST_CAP
for wb, stamp, hn, vleg, v, exact in self.samples:
# Any stamp is always valid at its legacy integer value,
# and always invalid at exactly one step above it. This
# locks stamp_valid and stamp_value into one scheme.
if vleg <= cap:
self.assertTrue(Stamps.stamp_valid(stamp, vleg, wb))
if vleg + 1 <= cap:
self.assertFalse(Stamps.stamp_valid(stamp, vleg + 1, wb))
# Directional consistency between value and validity
for wb, stamp, hn, vleg, v, exact in self.samples[:500]:
fp = fp_of(v)
for cost in (1, 2, 3):
if fp >= cost * Stamps.COST_FP:
self.assertTrue(Stamps.stamp_valid(stamp, cost, wb))
if Stamps.stamp_valid(stamp, cost, wb):
self.assertGreaterEqual(fp, (cost - 1) * Stamps.COST_FP)
def test_07_target_probability_calibration(self):
# The defining semantic of cost: a random stamp is valid for
# cost c with probability exactly 2^-c.
for cost in (2, 4):
trials = 20000
wb = rb(32)
valid = sum(1 for _ in range(trials) if Stamps.stamp_valid(rb(14), cost, wb))
fraction = valid / trials
expected = 2.0**-cost
sigma = math.sqrt(expected * (1-expected) / trials)
self.assertLess(abs(fraction - expected), 5*sigma)
def test_08_value_deterministic(self):
for wb, stamp, hn, vleg, v, exact in self.samples[:100]:
self.assertEqual(Stamps.stamp_value(wb, stamp), v)
self.assertEqual(Stamps.stamp_value_legacy(wb, stamp), vleg)
class TestStampWorkblock(unittest.TestCase):
def test_00_no_expand(self):
material = rb(64)
self.assertIs(Stamps.stamp_workblock(material, no_expand=True), material)
self.assertIs(Stamps.stamp_workblock(material, expand_rounds=0), material)
self.assertEqual(Stamps.stamp_workblock(material, expand_rounds=16, no_expand=True), material)
def test_01_expansion(self):
material = rb(32)
wb1 = Stamps.stamp_workblock(material, expand_rounds=16)
wb2 = Stamps.stamp_workblock(material, expand_rounds=16)
self.assertEqual(len(wb1), 16*256)
self.assertEqual(wb1, wb2)
self.assertNotEqual(wb1, Stamps.stamp_workblock(rb(32), expand_rounds=16))
def test_02_default_expansion(self):
material = rb(32)
wb = Stamps.stamp_workblock(material)
self.assertEqual(len(wb), Stamps.WORKBLOCK_EXPAND_ROUNDS*256)
self.assertEqual(wb, Stamps.stamp_workblock(material))
class TestStampGeneration(unittest.TestCase):
def _wait_registered(self, mid, timeout=15):
for _ in range(int(timeout/0.05)):
if mid in Stamps.active_jobs:
return True
time.sleep(0.05)
return False
def test_00_generate_stamp_low_costs(self):
for cost in (0, 4, 8):
mid = rb(32)
stamp, value = Stamps.generate_stamp(mid, cost, no_expand=True)
self.assertNotEqual(stamp, None)
self.assertIsInstance(stamp, bytes)
self.assertEqual(len(stamp), Stamps.STAMP_SIZE)
# The generation contract: the stamp must validate at
# the requested cost, and the reported value must be
# within one quantisation step of the requested cost.
self.assertTrue(Stamps.stamp_valid(stamp, cost, mid))
self.assertGreaterEqual(fp_of(value), (cost - 1) * Stamps.COST_FP)
# Old clients parse the fractional value at its integer
# value, so that must always be at least cost - 1, and
# must match the legacy value function exactly.
vleg = Stamps.stamp_value_legacy(mid, stamp)
self.assertGreaterEqual(vleg, cost - 1)
self.assertEqual(fp_of(value) // Stamps.COST_FP, vleg)
self.assertLess(value, 256)
self.assertEqual(value, Stamps.stamp_value(mid, stamp))
def test_01_generate_stamp_expanded_workblock(self):
mid = rb(32)
stamp, value = Stamps.generate_stamp(mid, 4, expand_rounds=16)
wb = Stamps.stamp_workblock(mid, expand_rounds=16)
self.assertEqual(len(wb), 16*256)
self.assertTrue(Stamps.stamp_valid(stamp, 4, wb))
self.assertEqual(value, Stamps.stamp_value(wb, stamp))
def test_02_generate_stamp_default_workblock(self):
mid = rb(32)
stamp, value = Stamps.generate_stamp(mid, 4)
wb = Stamps.stamp_workblock(mid)
self.assertEqual(len(wb), Stamps.WORKBLOCK_EXPAND_ROUNDS*256)
self.assertTrue(Stamps.stamp_valid(stamp, 4, wb))
def test_03_generate_stamped_key(self):
material = rb(32)
keypair, value = Stamps.generate_stamped_key(material, 4)
private, public = keypair
self.assertEqual(len(private), 32)
self.assertEqual(len(public), 32)
self.assertEqual(RNS.Cryptography.X25519PrivateKey.from_private_bytes(private).public_key().public_bytes(), public)
# Keystamp validation uses the public key bytes as the stamp
self.assertTrue(Stamps.stamp_valid(public, 4, material))
self.assertEqual(value, Stamps.stamp_value(material, public))
self.assertGreaterEqual(fp_of(value), 3 * Stamps.COST_FP)
def test_04_generate_stamped_key_empty_material(self):
keypair, value = Stamps.generate_stamped_key(None, 2)
private, public = keypair
self.assertTrue(Stamps.stamp_valid(public, 2, b""))
self.assertEqual(value, Stamps.stamp_value(b"", public))
def test_05_invalid_costs_rejected(self):
cap = RNS.Transport.STAMP_COST_CAP
mid = rb(16)
for cost in (-1, cap + 1):
with self.assertRaises(ValueError):
Stamps.generate_stamp(mid, cost, no_expand=True)
with self.assertRaises(ValueError):
Stamps.generate_stamped_key(mid, cost)
def test_06_duplicate_job_rejected(self):
mid = rb(16)
Stamps.active_jobs[mid] = False
try:
with self.assertRaises(RuntimeError):
Stamps.generate_stamp(mid, 4, no_expand=True)
with self.assertRaises(RuntimeError):
Stamps.generate_stamped_key(mid, 4)
finally:
del Stamps.active_jobs[mid]
def test_07_cancel_unknown_job(self):
# Cancelling a job that was never started must be a safe no-op
Stamps.cancel_work(rb(16))
def test_08_cancel_simple_job(self):
mid = rb(16)
wb = rb(32)
result = {}
def work():
result["r"] = Stamps.job_simple(30*Stamps.COST_FP, wb, mid, Stamps.STAMP_SIZE)
worker = threading.Thread(target=work, daemon=True)
worker.start()
self.assertTrue(self._wait_registered(mid))
time.sleep(0.2)
self.assertIs(Stamps.active_jobs[mid], False) # Still running
Stamps.cancel_work(mid)
worker.join(timeout=10)
self.assertFalse(worker.is_alive())
# Cost 30 can never complete, so a timely return with no
# stamp proves cancellation worked, and that work was done.
self.assertEqual(result["r"][0], None)
self.assertGreater(result["r"][1], 0)
self.assertNotIn(mid, Stamps.active_jobs)
def test_09_cancel_generation_job(self):
mid = rb(16)
result = {}
def work():
result["r"] = Stamps.generate_stamp(mid, 30, no_expand=True)
worker = threading.Thread(target=work, daemon=True)
worker.start()
self.assertTrue(self._wait_registered(mid))
time.sleep(0.3)
Stamps.cancel_work(mid)
worker.join(timeout=30)
self.assertFalse(worker.is_alive())
stamp, value = result["r"]
self.assertEqual(stamp, None)
self.assertNotIn(mid, Stamps.active_jobs)
@skipIf(not RNS.vendor.platformutils.is_linux(), "Direct multiprocess dispatchers are Linux-only paths")
def test_10_direct_multiprocess_dispatch(self):
# generate_stamp only routes to the multiprocess dispatchers
# at cost >= POOLING_THRESHOLD_COST, so they are exercised
# directly here at a trivial cost. This covers both the
# classic and managed worker mechanisms regardless of the
# Python version in use.
wb = rb(32)
mid1 = rb(16)
mid2 = rb(16)
stamp, rounds = Stamps.job_linux(4*Stamps.COST_FP, wb, mid1, Stamps.STAMP_SIZE)
self.assertNotEqual(stamp, None)
self.assertTrue(Stamps.stamp_valid(stamp, 4, wb))
self.assertGreater(rounds, 0)
self.assertNotIn(mid1, Stamps.active_jobs)
stamp, rounds = Stamps.job_linux_managed(5*Stamps.COST_FP, wb, mid2, Stamps.STAMP_SIZE)
self.assertNotEqual(stamp, None)
self.assertTrue(Stamps.stamp_valid(stamp, 5, wb))
self.assertGreater(rounds, 0)
self.assertNotIn(mid2, Stamps.active_jobs)
class TestStampPersistence(unittest.TestCase):
@classmethod
def setUpClass(cls):
cls.tmpdir = None
cls.owns_instance = False
# The full test suite shares one Reticulum instance, created
# by whichever test needed it first. If none is running, a
# temporary instance is created for this class only, and
# cleanly detached again afterwards.
if not RNS.Reticulum.storagepath:
cls.tmpdir = tempfile.mkdtemp(prefix="rns_stamp_tests_")
RNS.Reticulum(configdir=cls.tmpdir, loglevel=RNS.LOG_CRITICAL)
cls.owns_instance = True
@classmethod
def tearDownClass(cls):
if cls.owns_instance:
RNS.Reticulum._Reticulum__instance = None
RNS.Reticulum.configpath = ""
RNS.Reticulum.storagepath = ""
RNS.Reticulum.cachepath = ""
RNS.Reticulum.interfacepath = ""
if cls.tmpdir:
shutil.rmtree(cls.tmpdir, ignore_errors=True)
def _make_stamp(self, dest, cost=4):
# Destination stamps are STAMPLENGTH bytes, and are generated
# without workblock expansion, since the persistence layer
# recomputes the workblock as the unexpanded material.
return Stamps.generate_stamp(dest, cost, no_expand=True, size=RNS.Identity.STAMPLENGTH)
def _cleanup(self, dest):
path = Stamps.destination_stamp_file(dest)
if path and os.path.isfile(path):
os.remove(path)
def test_00_store_load_roundtrip(self):
dest = rb(16)
self.addCleanup(self._cleanup, dest)
stamp, value = self._make_stamp(dest)
self.assertTrue(Stamps.store_destination_stamp(dest, stamp, value))
loaded = Stamps.load_destination_stamp(dest, dest, 4)
self.assertNotEqual(loaded, None)
lstamp, lvalue = loaded
self.assertEqual(lstamp, stamp)
self.assertEqual(lvalue, value)
def test_01_load_value_boundaries(self):
dest = rb(16)
self.addCleanup(self._cleanup, dest)
stamp, value = self._make_stamp(dest)
Stamps.store_destination_stamp(dest, stamp, value)
# A stamp is always valid at its legacy integer value
vleg = Stamps.stamp_value_legacy(dest, stamp)
loaded = Stamps.load_destination_stamp(dest, dest, vleg)
self.assertNotEqual(loaded, None)
self.assertEqual(loaded[0], stamp)
# ... and never at a full step above its own value
self.assertEqual(Stamps.load_destination_stamp(dest, dest, value + 1), None)
def test_02_load_nonexistent(self):
self.assertEqual(Stamps.load_destination_stamp(rb(16), rb(16), 1), None)
def test_03_load_corrupt_file(self):
dest = rb(16)
self.addCleanup(self._cleanup, dest)
stamp, value = self._make_stamp(dest)
Stamps.store_destination_stamp(dest, stamp, value)
with open(Stamps.destination_stamp_file(dest), "wb") as file:
file.write(rb(64))
self.assertEqual(Stamps.load_destination_stamp(dest, dest, 1), None)
def test_04_load_wrong_version(self):
dest = rb(16)
self.addCleanup(self._cleanup, dest)
stamp, value = self._make_stamp(dest)
Stamps.store_destination_stamp(dest, stamp, value)
with open(Stamps.destination_stamp_file(dest), "wb") as file:
file.write(msgpack.packb({"v": 99, "stamp": stamp, "value": value, "generated": time.time()}))
self.assertEqual(Stamps.load_destination_stamp(dest, dest, 1), None)
def test_05_load_wrong_stamp_length(self):
dest = rb(16)
self.addCleanup(self._cleanup, dest)
stamp, value = self._make_stamp(dest)
Stamps.store_destination_stamp(dest, stamp, value)
with open(Stamps.destination_stamp_file(dest), "wb") as file:
file.write(msgpack.packb({"v": Stamps.STAMP_FILE_VERSION, "stamp": rb(8), "value": value, "generated": time.time()}))
self.assertEqual(Stamps.load_destination_stamp(dest, dest, 1), None)
def test_06_stamp_paths(self):
directory = Stamps.stamps_dir(create=True)
self.assertNotEqual(directory, None)
self.assertTrue(os.path.isdir(directory))
dest = rb(16)
path = Stamps.destination_stamp_file(dest)
self.assertEqual(os.path.dirname(path), directory)
self.assertTrue(path.endswith(RNS.hexrep(dest, delimit=False) + Stamps.DESTINATION_STAMP_EXTENSION))
if __name__ == '__main__':
unittest.main(verbosity=2)