From 78fb90ea6f30baf7201a3fb79ec224bb5c1ca3c1 Mon Sep 17 00:00:00 2001 From: Mark Qvist Date: Thu, 29 Oct 2026 16:49:39 +0100 Subject: [PATCH] Implemented fractional-value stamps --- RNS/Stamps.py | 238 ++++++++++++++++++++++-- tests/all.py | 4 + tests/stamps.py | 469 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 692 insertions(+), 19 deletions(-) create mode 100644 tests/stamps.py diff --git a/RNS/Stamps.py b/RNS/Stamps.py index d357c4d6..b873eb69 100644 --- a/RNS/Stamps.py +++ b/RNS/Stamps.py @@ -30,6 +30,7 @@ import RNS import RNS.vendor.umsgpack as msgpack +from RNS.Cryptography import sha256 import os import sys @@ -39,15 +40,15 @@ import mmap import itertools import contextlib import multiprocessing +from bisect import bisect_right WORKBLOCK_EXPAND_ROUNDS = 3000 STAMP_SIZE = 32 PN_VALIDATION_POOL_MIN_SIZE = 256 USE_WORKER_MANAGER = False POOLING_THRESHOLD_COST = 15 -STOP_CHECK_INTERVAL = 64 - KEYSTAMP_POOLING_THRESHOLD = 12 +STOP_CHECK_INTERVAL = 64 MODE_STAMP = 0x00 MODE_KEYSTAMP = 0x01 @@ -94,14 +95,101 @@ def stamp_workblock(material, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=F return workblock -def stamp_value(workblock, stamp): +def stamp_value_legacy(workblock, stamp): if not workblock: workblock = b"" - material = RNS.Identity.full_hash(workblock+stamp) - return 256 - int.from_bytes(material, byteorder="big").bit_length() + stamp_product = RNS.Identity.full_hash(workblock+stamp) + return 256 - int.from_bytes(stamp_product, byteorder="big").bit_length() + +def stamp_value(workblock, stamp): + # This function is very little code, and a whole lot of + # comments. The actual method employed here is relatively + # simple, but without proper explanation, it is also very + # opaque, and could induce a lot of head-scratching. + + if not workblock: workblock = b"" + stamp_product = RNS.Identity.full_hash(workblock+stamp) # Bytes representation + hn = int.from_bytes(stamp_product, byteorder="big") # Numerical representation + if hn == 0: return 256 # Yeah, but *if* it happens, you know... + nb = MANTISSA_BITS+1 # Normalized mantissa bit count + bl = hn.bit_length() # Bit length of the stamp product + lz = 256 - bl # Leading zeros of the stamp product + + ############################################################## + # # + # Simplified mantissa extraction example with # + # smaller numbers; 32-bit stamp product and 8 # + # mantissa bits. In the actual version, it is # + # 256-bit stamp products and 33-bit mantissa. # + # # + # mb = 8 nb = 9 (8+1) # + # bl = 27 lz = 5 (32-27) # + # # + # shift = bl - nb = 18 # + # # + # hn = 0b00000111101101101110111010110111 # + # >>18 0b00000000000000000000000111101101|101110111010110111 # + # | \_____ gone # + # The leading one | # + # 8 significand mantissa bits # + # # + # mn = 0b111101101 (normalized with the leading 1) # + # # + ############################################################## + + # Normalize mantissa to the scale of TAB: Total of 34 + # bits, 33 mantissa bits. The leading one required to + # match TAB entries is guaranteed by the construction: + # Since we're shifting by the *difference* between the + # normalized mantissa bit length and the stamp product + # bit length, we always end up with a leading one, and + # then the required number of *actual* mantissa bits. + # See the example above for a simplified illustration. + if bl > nb: mn = (hn >> (bl - nb)) + else: mn = (hn << (nb - bl)) # Again, better safe than sorry. You may notice + # here that in this case, we will have a more + # coarse mantissa part, which is inherent to + # the methodology, but in practice the best thing + # we can do. Additionally, this can only happen + # at absolutely extreme values, far above the + # stamp cap anyway. It is only here to handle the + # absurd case where a stamp product randomly lands + # on an astronomically improbable leading zero count. + + # We can now construct the fixed-point value representation: + # + # The leading zero count is equivalent to the legacy value, an + # integer representation. Thus, (lz+1)< target: return False else: return True @@ -163,13 +251,13 @@ def _loop_key(material, target, stop_check, max_rounds=0): # bytes material (message_id), which is usually a hash # of some input material, but can be any sequence of bytes. def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS, no_expand=False, size=STAMP_SIZE): - stamp_cost = int(stamp_cost) if stamp_cost < 0 or stamp_cost > RNS.Transport.STAMP_COST_CAP: raise ValueError("Invalid stamp cost, cannot generate") workblock = stamp_workblock(message_id, expand_rounds=expand_rounds, no_expand=no_expand) if message_id in active_jobs: raise RuntimeError(f"Duplicate stamp generation job for {RNS.prettyhexrep(message_id)}") RNS.log(f"Generating stamp with cost {stamp_cost} for <{RNS.hexrep(message_id[:16], delimit=False)}{'…' if len(message_id) > 16 else ''}>", RNS.LOG_DEBUG) - + + stamp_cost = cost_to_fp(stamp_cost) start_time = time.time() stamp = None rounds = 0 @@ -181,8 +269,8 @@ def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): stamp, rounds = job_simple(stamp_cost, workblock, message_id, size) elif RNS.vendor.platformutils.is_android(): stamp, rounds = job_android(stamp_cost, workblock, message_id, size) else: - if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST else job_simple(stamp_cost, workblock, message_id, size) - else: stamp, rounds = job_linux(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST else job_simple(stamp_cost, workblock, message_id, size) + if USE_WORKER_MANAGER: stamp, rounds = job_linux_managed(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST_FP else job_simple(stamp_cost, workblock, message_id, size) + else: stamp, rounds = job_linux(stamp_cost, workblock, message_id, size) if stamp_cost >= POOLING_THRESHOLD_COST_FP else job_simple(stamp_cost, workblock, message_id, size) duration = max(time.time()-start_time, 1E-6) speed = rounds/duration @@ -226,12 +314,12 @@ def generate_stamp(message_id, stamp_cost, expand_rounds=WORKBLOCK_EXPAND_ROUNDS # this function must ONLY be used as X25519 keys. def generate_stamped_key(material, stamp_cost): if material == None: material = b"" - stamp_cost = int(stamp_cost) if material in active_jobs: raise RuntimeError(f"Duplicate stamped key generation job for {RNS.prettyhexrep(material)}") if stamp_cost < 0 or stamp_cost > RNS.Transport.STAMP_COST_CAP: raise ValueError("Invalid stamp cost, cannot generate") for_str = f" for <{RNS.hexrep(material[:16], delimit=False)}{'…' if len(material) > 16 else ''}>" if material else "" RNS.log(f"Generating stamped key with cost {stamp_cost}{for_str}", RNS.LOG_DEBUG) + stamp_cost = cost_to_fp(stamp_cost) start_time = time.time() keypair = None rounds = 0 @@ -243,8 +331,8 @@ def generate_stamped_key(material, stamp_cost): if RNS.vendor.platformutils.is_windows() or RNS.vendor.platformutils.is_darwin(): keypair, rounds = job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP) elif RNS.vendor.platformutils.is_android(): keypair, rounds = job_android(stamp_cost, material, material, 0, MODE_KEYSTAMP) else: - if USE_WORKER_MANAGER: keypair, rounds = job_linux_managed(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP) - else: keypair, rounds = job_linux(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP) + if USE_WORKER_MANAGER: keypair, rounds = job_linux_managed(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD_FP else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP) + else: keypair, rounds = job_linux(stamp_cost, material, material, 0, MODE_KEYSTAMP) if stamp_cost >= KEYSTAMP_POOLING_THRESHOLD_FP else job_simple(stamp_cost, material, material, 0, MODE_KEYSTAMP) duration = max(time.time()-start_time, 1E-6) speed = rounds/duration @@ -300,7 +388,7 @@ def job_simple(stamp_cost, workblock, message_id, size, mode=MODE_STAMP): # size argument is ignored. st = time.time() - target = 0b1<<256-stamp_cost + target = target_from_fp(stamp_cost) active_jobs[message_id] = False @@ -329,7 +417,7 @@ def job_linux_managed(stamp_cost, workblock, message_id, size, mode=MODE_STAMP): RNS.log(f"Starting {jobs} stamp generation workers", RNS.LOG_DEBUG) def job(stop_event, sc, wb, pn, mode): - target = 0b1<<256 - sc + target = target_from_fp(sc) if mode == MODE_STAMP: result, rounds, _ = _loop_hash(wb, target, size, stop_event.is_set) else: result, rounds, _ = _loop_key(wb, target, stop_event.is_set) @@ -402,7 +490,7 @@ def job_linux(stamp_cost, workblock, message_id, size, mode=MODE_STAMP): rounds_queue = multiprocessing.Queue() def job(stop_event, pn, sc, wb, mode): - target = 0b1<<256 - sc + target = target_from_fp(sc) if mode == MODE_STAMP: result, rounds, _ = _loop_hash(wb, target, size, stop_event.is_set) else: result, rounds, _ = _loop_key(wb, target, stop_event.is_set) @@ -495,7 +583,7 @@ def job_android(stamp_cost, workblock, message_id, size, mode=MODE_STAMP): def job(procnum=None, results_dict=None, wb=None, sc=None, jr=None, mode=MODE_STAMP): try: found_result = None - target = 0b1<<256-sc + target = target_from_fp(sc) if mode == MODE_STAMP: result, rounds, found = _loop_hash(wb, target, size, _never, None, jr, full_hash) else: result, rounds, found = _loop_key(wb, target, _never, jr) @@ -575,7 +663,7 @@ def job_android_persistent(stamp_cost, workblock, message_id, size, mode=MODE_ST rounds = 0 found_result = None try: - target = 0b1<<256-stamp_cost + target = target_from_fp(stamp_cost) stop_check = lambda: stop_flag[0] != 0 if mode == MODE_STAMP: result, rounds, found = _loop_hash(workblock, target, size, stop_check, None, 0, full_hash) @@ -634,6 +722,111 @@ def job_android_persistent(stamp_cost, workblock, message_id, size, mode=MODE_ST return stamp, total_rounds +####################### +# Fixed-point Helpers # +####################### + +# Fixed-point stamp target mantissa table +# TAB[f] = floor(2^33 * 2^(f/256)), f = 0..255 +TAB_DIGEST = "55ba3ce1df224895b0dfdc102eb4359fc2f5a7efefee68df0cc90c8a8ac24f48" +PRM_DIGEST = "21436fc98e76a12cad7c52ebeb6ab76c2ae93b4cd768949df7fdd4e2f98c378d" +MANTISSA_BITS = 33 +COST_FP_BITS = 8 +COST_FP = 1 << COST_FP_BITS + +# These values are authoritative. DO NOT attempt to +# invent your own, calculate them at load time, or +# generate them on the fly. You have been warned. +TAB = ( + 0x200000000, 0x201635f4b, 0x202c7b53f, 0x2042d0287, 0x2059347ce, 0x206fa85c2, + 0x20862bd0d, 0x209cbee5e, 0x20b361a62, 0x20ca141c7, 0x20e0d653b, 0x20f7a856e, + 0x210e8a30e, 0x21257becc, 0x213c7d958, 0x21538f363, 0x216ab0d9f, 0x2181e28bc, + 0x21992456e, 0x21b076467, 0x21c7d865a, 0x21df4abfb, 0x21f6cd5ff, 0x220e6051a, + 0x222603a02, 0x223db756b, 0x22557b80d, 0x226d5029e, 0x2285355d5, 0x229d2b269, + 0x22b531914, 0x22cd48a8e, 0x22e57078f, 0x22fda90d2, 0x2315f2711, 0x232e4cb06, + 0x2346b7d6d, 0x235f33f02, 0x2377c1080, 0x23905f2a5, 0x23a90e62d, 0x23c1cebd6, + 0x23daa045f, 0x23f383087, 0x240c7710c, 0x24257c6af, 0x243e9322f, 0x2457bb44f, + 0x2470f4dce, 0x248a3ff70, 0x24a39c9f6, 0x24bd0ae23, 0x24d68acbc, 0x24f01c683, + 0x2509bfc3e, 0x252374eb2, 0x253d3bea3, 0x255714cda, 0x2570ffa1b, 0x258afc72e, + 0x25a50b4dc, 0x25bf2c3ec, 0x25d95f527, 0x25f3a4957, 0x260dfc146, 0x262865dbd, + 0x2642e1f89, 0x265d70775, 0x26781164c, 0x2692c4cdc, 0x26ad8abf2, 0x26c86345b, + 0x26e34e6e7, 0x26fe4c463, 0x27195cda0, 0x27348036e, 0x274fb669c, 0x276aff7fd, + 0x27865b862, 0x27a1ca89d, 0x27bd4c982, 0x27d8e1be3, 0x27f48a095, 0x28104586c, + 0x282c1443e, 0x2847f64e1, 0x2863ebb2a, 0x287ff47f0, 0x289c10c0c, 0x28b840854, + 0x28d483da3, 0x28f0dacd1, 0x290d456b8, 0x2929c3c32, 0x294655e1a, 0x2962fbd4d, + 0x297fb5aa6, 0x299c83702, 0x29b96533f, 0x29d65b03b, 0x29f364ed3, 0x2a1082fe8, + 0x2a2db5459, 0x2a4afbd07, 0x2a6856ad3, 0x2a85c5e9e, 0x2aa34994b, 0x2ac0e1bbd, + 0x2ade8e6d6, 0x2afc4fb7c, 0x2b1a25a92, 0x2b38104ff, 0x2b560fba9, 0x2b7423f75, + 0x2b924d14b, 0x2bb08b213, 0x2bcede2b5, 0x2bed4641b, 0x2c0bc372e, 0x2c2a55cd9, + 0x2c48fd607, 0x2c67ba3a3, 0x2c868c699, 0x2ca573fd7, 0x2cc47104a, 0x2ce3838e1, + 0x2d02aba89, 0x2d21e9633, 0x2d413cccf, 0x2d60a5f4e, 0x2d8024ea1, 0x2d9fb9bba, + 0x2dbf6478c, 0x2ddf2530b, 0x2dfefbf2a, 0x2e1ee8cde, 0x2e3eebd1d, 0x2e5f050dd, + 0x2e7f34914, 0x2e9f7a6ba, 0x2ebfd6ac8, 0x2ee049635, 0x2f00d29fb, 0x2f2172715, + 0x2f4228e7d, 0x2f62f612e, 0x2f83da026, 0x2fa4d4c5f, 0x2fc5e66d9, 0x2fe70f092, + 0x30084ea87, 0x3029a55ba, 0x304b13329, 0x306c983d7, 0x308e348c4, 0x30afe82f3, + 0x30d1b3368, 0x30f395b26, 0x31158fb31, 0x3137a148f, 0x3159ca845, 0x317c0b75a, + 0x319e642d6, 0x31c0d4bc1, 0x31e35d322, 0x3205fda05, 0x3228b6172, 0x324b86a75, + 0x326e6f619, 0x32917056b, 0x32b489979, 0x32d7bb34e, 0x32fb053fb, 0x331e67c8f, + 0x3341e2e19, 0x3365769aa, 0x338923054, 0x33ace8329, 0x33d0c633c, 0x33f4bd1a0, + 0x3418ccf6b, 0x343cf5db1, 0x346137d89, 0x348593008, 0x34aa07647, 0x34ce9515e, + 0x34f33c266, 0x3517fca78, 0x353cd6aaf, 0x3561ca426, 0x3586d77fa, 0x35abfe747, + 0x35d13f32b, 0x35f699cc4, 0x361c0e531, 0x36419cd93, 0x366745709, 0x368d082b6, + 0x36b2e51bc, 0x36d8dc53e, 0x36feede5f, 0x372519e44, 0x374b60614, 0x3771c16f3, + 0x37983d209, 0x37bed387e, 0x37e584b7a, 0x380c50c28, 0x383337bb0, 0x385a39b3f, + 0x388156bff, 0x38a88ef1f, 0x38cfe25ca, 0x38f751131, 0x391edb280, 0x394680aea, + 0x396e41b9d, 0x39961e5cd, 0x39be16aab, 0x39e62ab6b, 0x3a0e5a940, 0x3a36a6561, + 0x3a5f0e101, 0x3a8791d59, 0x3ab031b9f, 0x3ad8edd0c, 0x3b01c62d9, 0x3b2abae3f, + 0x3b53cc07b, 0x3b7cf9ac7, 0x3ba643e60, 0x3bcfaac83, 0x3bf92e66f, 0x3c22ced63, + 0x3c4c8c29e, 0x3c7666762, 0x3ca05dcf1, 0x3cca7248c, 0x3cf4a3f78, 0x3d1ef2ef9, + 0x3d495f454, 0x3d73e90cf, 0x3d9e905b1, 0x3dc955443, 0x3df437dcc, 0x3e1f38396, + 0x3e4a566ed, 0x3e759291b, 0x3ea0ecb6d, 0x3ecc64f31, 0x3ef7fb5b3, 0x3f23b0044, + 0x3f4f83033, 0x3f7b746d2, 0x3fa784571, 0x3fd3b2d65 ) + +assert len(TAB) == COST_FP +assert TAB[0] == 1 << MANTISSA_BITS +assert all(b > a for a, b in zip(TAB, TAB[1:])) +assert sha256(",".join(str(v) for v in TAB).encode("ascii")) == bytes.fromhex(TAB_DIGEST) +assert sha256(f"master|{TAB_DIGEST}|{MANTISSA_BITS}|{len(TAB)}|{COST_FP_BITS}|{COST_FP}|".encode("utf-8")) == bytes.fromhex(PRM_DIGEST) + +def cost_to_fp(cost): + v = math.floor(float(cost) * COST_FP + 0.5) + return max(0, min(v, RNS.Transport.STAMP_COST_CAP * COST_FP)) + +def fp_to_bytes(fp): + return bytes(((fp >> COST_FP_BITS) & 0xFF, fp & 0xFF)) + +def cost_to_bytes(cost): + return fp_to_bytes(cost_to_fp(cost)) + +def fp_from_bytes(b): + return (b[0] << COST_FP_BITS) | b[1] + +def cost_from_fp(fp): + return fp/COST_FP + +def cost_from_bytes(b): + return cost_from_fp(fp_from_bytes(b)) + +def target_from_fp(fp): + assert 0 <= fp <= RNS.Transport.STAMP_COST_CAP * COST_FP + # Extract the integer and fractional parts + # by modular division, here equivalent to: + # ((256*COST_FP) // COST_FP, (256*COST_FP) % COST_FP) + # i = quotient f = remainder + i, f = divmod(256*COST_FP - fp, COST_FP) + + # Then, construct the numerical representation + # by getting the quantized fractional mantissa + # from TAB, and raising it to i (the exponent) + # and right-shifting it by MANTISSA_BITS. + return (TAB[f] << i) >> MANTISSA_BITS + +def target_from_cost(cost): + return target_from_fp(cost_to_fp(cost)) + +def cost_int(fp): return fp >> COST_FP_BITS # Legacy integer display representation +def cost_str(fp): return f"{fp / COST_FP:g}" # Display only, remember, the floats are only in your head + ########################### # Persistence and loading # ########################### @@ -691,7 +884,7 @@ def load_destination_stamp(destination_hash, material, required_value): return None value = stamp_value(workblock, stamp) - if value < required_value: + if not stamp_valid(stamp, required_value, workblock): RNS.log(f"Stored stamp for {RNS.prettyhexrep(destination_hash)} has value {value}, below required {required_value}. Regenerating on next announce.", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None return None @@ -700,3 +893,10 @@ def load_destination_stamp(destination_hash, material, required_value): except Exception as e: RNS.log(f"Ignoring unreadable destination stamp file for {RNS.prettyhexrep(destination_hash)}: {e}", RNS.LOG_WARNING) return None + +######################### +# Pooling Configuration # +######################### + +POOLING_THRESHOLD_COST_FP = POOLING_THRESHOLD_COST*COST_FP +KEYSTAMP_POOLING_THRESHOLD_FP = KEYSTAMP_POOLING_THRESHOLD*COST_FP \ No newline at end of file diff --git a/tests/all.py b/tests/all.py index 17332612..d7539402 100644 --- a/tests/all.py +++ b/tests/all.py @@ -7,6 +7,10 @@ from .link import TestLink from .channel import TestChannel from .hkdf import TestHKDF from .ifac import TestIFAC +from .stamps import TestStampFixedPoint +from .stamps import TestStampWorkblock +from .stamps import TestStampGeneration +from .stamps import TestStampPersistence from .coalesced_transmit import TestTransmitBuffer from .hdlc import TestReceiveBuffer from .egress import TestHWMLimiter diff --git a/tests/stamps.py b/tests/stamps.py new file mode 100644 index 00000000..a63802e8 --- /dev/null +++ b/tests/stamps.py @@ -0,0 +1,469 @@ +# Generated with lc/glm5.3_flash_q6_k_xl +# Reviewed and approved by Mark Qvist + +import unittest +import os +import time +import math +import random +import shutil +import tempfile +import threading +from unittest import skipIf + +import RNS +from RNS import Stamps +import RNS.vendor.umsgpack as msgpack + +# The stamper itself is intentionally seeded from os.urandom, but the +# value and target layers are pure deterministic math. Seeded test +# vectors therefore make the statistical tests exact and repeatable +# on any platform and any hash backend. +rng = random.Random(28411) + +def rb(n): + return bytes(rng.getrandbits(8) for _ in range(n)) + +def fp_of(value): + # stamp_value returns cost_from_fp(value_fp), and division by + # COST_FP (a power of two) is exact in binary floating point, + # so the fixed-point representation can be recovered losslessly. + fp = round(value * Stamps.COST_FP) + assert fp == int(value * Stamps.COST_FP) + return fp + + +class TestStampFixedPoint(unittest.TestCase): + @classmethod + def setUpClass(cls): + # A shared pool of deterministic workblock/stamp samples, + # used to lock in the value layer invariants. + cls.samples = [] + for _ in range(3000): + wb = rb(rng.randint(0, 48)) + stamp = rb(rng.choice((RNS.Identity.STAMPLENGTH, Stamps.STAMP_SIZE))) + hn = int.from_bytes(RNS.Identity.full_hash(wb+stamp), byteorder="big") + vleg = Stamps.stamp_value_legacy(wb, stamp) + v = Stamps.stamp_value(wb, stamp) + exact = 256 - math.log2(hn) + cls.samples.append((wb, stamp, hn, vleg, v, exact)) + + def test_00_fixed_point_conversions(self): + cap = RNS.Transport.STAMP_COST_CAP + for cost in range(0, cap+1): + self.assertEqual(Stamps.cost_to_fp(cost), cost*Stamps.COST_FP) + self.assertEqual(Stamps.cost_from_fp(Stamps.cost_to_fp(cost)), cost) + + self.assertEqual(Stamps.cost_to_fp(8.5), 8.5*Stamps.COST_FP) + self.assertEqual(Stamps.cost_from_fp(8.5*Stamps.COST_FP), 8.5) + self.assertEqual(Stamps.cost_to_fp(0.5), 128) + self.assertEqual(Stamps.cost_to_fp(-1), 0) + self.assertEqual(Stamps.cost_to_fp(cap+1), cap*Stamps.COST_FP) + + for fp in list(range(0, 65536, 97)) + [0, 255, 256, 65535]: + packed = Stamps.fp_to_bytes(fp) + self.assertEqual(len(packed), 2) + self.assertEqual(Stamps.fp_from_bytes(packed), fp) + + for cost in (0, 1, 4, 8.5, 64, 100): + self.assertEqual(Stamps.cost_from_bytes(Stamps.cost_to_bytes(cost)), cost) + + self.assertEqual(Stamps.cost_int(8*Stamps.COST_FP + 128), 8) + self.assertEqual(Stamps.cost_str(8*Stamps.COST_FP), "8") + self.assertEqual(Stamps.cost_str(8*Stamps.COST_FP + 128), "8.5") + self.assertEqual(Stamps.cost_str(0), "0") + + def test_01_target_for_integer_costs(self): + cap = RNS.Transport.STAMP_COST_CAP + for cost in range(0, cap+1): + self.assertEqual(Stamps.target_from_cost(cost), 2**(256-cost)) + + self.assertEqual(Stamps.target_from_fp(0), 2**256) + self.assertRaises(AssertionError, Stamps.target_from_fp, -1) + self.assertRaises(AssertionError, Stamps.target_from_fp, cap*Stamps.COST_FP + 1) + + def test_02_target_monotonic_and_exact(self): + # Targets must strictly decrease as cost increases + cap = RNS.Transport.STAMP_COST_CAP + last = None + for fp in range(0, cap*Stamps.COST_FP + 1): + target = Stamps.target_from_fp(fp) + if last != None: + self.assertLess(target, last) + last = target + + # Independent verification of the whole mantissa/shift + # construction against floor(2^(256 - fp/256)), computed + # with exact integer arithmetic: The 256th root of + # 2^(33*256+f), obtained by a Newton iteration, is exactly + # floor(2^(33+f/256)), and shifting by the integer part of + # the exponent yields the exact target for every step. + def iroot(n, k): + x = 1 << ((n.bit_length() + k - 1)//k) + while True: + y = ((k-1)*x + n//x**(k-1))//k + if y >= x: + return x + x = y + + roots = [iroot(1 << (Stamps.MANTISSA_BITS*Stamps.COST_FP + f), Stamps.COST_FP) for f in range(Stamps.COST_FP)] + for fp in range(0, cap*Stamps.COST_FP + 1): + i, f = divmod(256*Stamps.COST_FP - fp, Stamps.COST_FP) + self.assertEqual(Stamps.target_from_fp(fp), roots[f] << (i - Stamps.MANTISSA_BITS)) + + def test_03_value_matches_legacy_integer_value(self): + # The compatibility invariant: old clients parse fractional + # stamps at their integer value. The legacy value function + # must therefore always be the integer floor of the new one. + for wb, stamp, hn, vleg, v, exact in self.samples: + self.assertEqual(vleg, 256 - hn.bit_length()) + self.assertLessEqual(vleg, v) + self.assertLess(v, vleg + 1) + self.assertEqual(fp_of(v) // Stamps.COST_FP, vleg) + + # Empty and None workblocks are handled identically + wb, stamp, hn, vleg, v, exact = self.samples[0] + self.assertEqual(Stamps.stamp_value(None, stamp), Stamps.stamp_value(b"", stamp)) + self.assertEqual(Stamps.stamp_value_legacy(None, stamp), Stamps.stamp_value_legacy(b"", stamp)) + self.assertTrue(Stamps.stamp_valid(stamp, 0, None)) + + def test_04_value_never_overreports(self): + # The reported value must never exceed the exact value of + # the stamp product, and must be within one quantisation + # step (1/256th) below it. + step = 1.0 / Stamps.COST_FP + for wb, stamp, hn, vleg, v, exact in self.samples: + self.assertLessEqual(v, exact + 1e-9) + self.assertLessEqual(exact, v + step + 1e-9) + + def test_05_value_monotone_in_stamp_product(self): + # A smaller stamp product (numerically) always means a more + # rare hash, and must never yield a smaller value. + ordered = sorted(self.samples, key=lambda sample: sample[2]) + last = None + for wb, stamp, hn, vleg, v, exact in ordered: + fp = fp_of(v) + if last != None: + self.assertLessEqual(fp, last) + last = fp + + def test_06_validity_boundaries(self): + cap = RNS.Transport.STAMP_COST_CAP + for wb, stamp, hn, vleg, v, exact in self.samples: + # Any stamp is always valid at its legacy integer value, + # and always invalid at exactly one step above it. This + # locks stamp_valid and stamp_value into one scheme. + if vleg <= cap: + self.assertTrue(Stamps.stamp_valid(stamp, vleg, wb)) + if vleg + 1 <= cap: + self.assertFalse(Stamps.stamp_valid(stamp, vleg + 1, wb)) + + # Directional consistency between value and validity + for wb, stamp, hn, vleg, v, exact in self.samples[:500]: + fp = fp_of(v) + for cost in (1, 2, 3): + if fp >= cost * Stamps.COST_FP: + self.assertTrue(Stamps.stamp_valid(stamp, cost, wb)) + if Stamps.stamp_valid(stamp, cost, wb): + self.assertGreaterEqual(fp, (cost - 1) * Stamps.COST_FP) + + def test_07_target_probability_calibration(self): + # The defining semantic of cost: a random stamp is valid for + # cost c with probability exactly 2^-c. + for cost in (2, 4): + trials = 20000 + wb = rb(32) + valid = sum(1 for _ in range(trials) if Stamps.stamp_valid(rb(14), cost, wb)) + fraction = valid / trials + expected = 2.0**-cost + sigma = math.sqrt(expected * (1-expected) / trials) + self.assertLess(abs(fraction - expected), 5*sigma) + + def test_08_value_deterministic(self): + for wb, stamp, hn, vleg, v, exact in self.samples[:100]: + self.assertEqual(Stamps.stamp_value(wb, stamp), v) + self.assertEqual(Stamps.stamp_value_legacy(wb, stamp), vleg) + + +class TestStampWorkblock(unittest.TestCase): + def test_00_no_expand(self): + material = rb(64) + self.assertIs(Stamps.stamp_workblock(material, no_expand=True), material) + self.assertIs(Stamps.stamp_workblock(material, expand_rounds=0), material) + self.assertEqual(Stamps.stamp_workblock(material, expand_rounds=16, no_expand=True), material) + + def test_01_expansion(self): + material = rb(32) + wb1 = Stamps.stamp_workblock(material, expand_rounds=16) + wb2 = Stamps.stamp_workblock(material, expand_rounds=16) + self.assertEqual(len(wb1), 16*256) + self.assertEqual(wb1, wb2) + self.assertNotEqual(wb1, Stamps.stamp_workblock(rb(32), expand_rounds=16)) + + def test_02_default_expansion(self): + material = rb(32) + wb = Stamps.stamp_workblock(material) + self.assertEqual(len(wb), Stamps.WORKBLOCK_EXPAND_ROUNDS*256) + self.assertEqual(wb, Stamps.stamp_workblock(material)) + + +class TestStampGeneration(unittest.TestCase): + def _wait_registered(self, mid, timeout=15): + for _ in range(int(timeout/0.05)): + if mid in Stamps.active_jobs: + return True + time.sleep(0.05) + return False + + def test_00_generate_stamp_low_costs(self): + for cost in (0, 4, 8): + mid = rb(32) + stamp, value = Stamps.generate_stamp(mid, cost, no_expand=True) + self.assertNotEqual(stamp, None) + self.assertIsInstance(stamp, bytes) + self.assertEqual(len(stamp), Stamps.STAMP_SIZE) + + # The generation contract: the stamp must validate at + # the requested cost, and the reported value must be + # within one quantisation step of the requested cost. + self.assertTrue(Stamps.stamp_valid(stamp, cost, mid)) + self.assertGreaterEqual(fp_of(value), (cost - 1) * Stamps.COST_FP) + + # Old clients parse the fractional value at its integer + # value, so that must always be at least cost - 1, and + # must match the legacy value function exactly. + vleg = Stamps.stamp_value_legacy(mid, stamp) + self.assertGreaterEqual(vleg, cost - 1) + self.assertEqual(fp_of(value) // Stamps.COST_FP, vleg) + self.assertLess(value, 256) + self.assertEqual(value, Stamps.stamp_value(mid, stamp)) + + def test_01_generate_stamp_expanded_workblock(self): + mid = rb(32) + stamp, value = Stamps.generate_stamp(mid, 4, expand_rounds=16) + wb = Stamps.stamp_workblock(mid, expand_rounds=16) + self.assertEqual(len(wb), 16*256) + self.assertTrue(Stamps.stamp_valid(stamp, 4, wb)) + self.assertEqual(value, Stamps.stamp_value(wb, stamp)) + + def test_02_generate_stamp_default_workblock(self): + mid = rb(32) + stamp, value = Stamps.generate_stamp(mid, 4) + wb = Stamps.stamp_workblock(mid) + self.assertEqual(len(wb), Stamps.WORKBLOCK_EXPAND_ROUNDS*256) + self.assertTrue(Stamps.stamp_valid(stamp, 4, wb)) + + def test_03_generate_stamped_key(self): + material = rb(32) + keypair, value = Stamps.generate_stamped_key(material, 4) + private, public = keypair + self.assertEqual(len(private), 32) + self.assertEqual(len(public), 32) + self.assertEqual(RNS.Cryptography.X25519PrivateKey.from_private_bytes(private).public_key().public_bytes(), public) + + # Keystamp validation uses the public key bytes as the stamp + self.assertTrue(Stamps.stamp_valid(public, 4, material)) + self.assertEqual(value, Stamps.stamp_value(material, public)) + self.assertGreaterEqual(fp_of(value), 3 * Stamps.COST_FP) + + def test_04_generate_stamped_key_empty_material(self): + keypair, value = Stamps.generate_stamped_key(None, 2) + private, public = keypair + self.assertTrue(Stamps.stamp_valid(public, 2, b"")) + self.assertEqual(value, Stamps.stamp_value(b"", public)) + + def test_05_invalid_costs_rejected(self): + cap = RNS.Transport.STAMP_COST_CAP + mid = rb(16) + for cost in (-1, cap + 1): + with self.assertRaises(ValueError): + Stamps.generate_stamp(mid, cost, no_expand=True) + with self.assertRaises(ValueError): + Stamps.generate_stamped_key(mid, cost) + + def test_06_duplicate_job_rejected(self): + mid = rb(16) + Stamps.active_jobs[mid] = False + try: + with self.assertRaises(RuntimeError): + Stamps.generate_stamp(mid, 4, no_expand=True) + with self.assertRaises(RuntimeError): + Stamps.generate_stamped_key(mid, 4) + finally: + del Stamps.active_jobs[mid] + + def test_07_cancel_unknown_job(self): + # Cancelling a job that was never started must be a safe no-op + Stamps.cancel_work(rb(16)) + + def test_08_cancel_simple_job(self): + mid = rb(16) + wb = rb(32) + result = {} + def work(): + result["r"] = Stamps.job_simple(30*Stamps.COST_FP, wb, mid, Stamps.STAMP_SIZE) + + worker = threading.Thread(target=work, daemon=True) + worker.start() + self.assertTrue(self._wait_registered(mid)) + time.sleep(0.2) + self.assertIs(Stamps.active_jobs[mid], False) # Still running + + Stamps.cancel_work(mid) + worker.join(timeout=10) + self.assertFalse(worker.is_alive()) + + # Cost 30 can never complete, so a timely return with no + # stamp proves cancellation worked, and that work was done. + self.assertEqual(result["r"][0], None) + self.assertGreater(result["r"][1], 0) + self.assertNotIn(mid, Stamps.active_jobs) + + def test_09_cancel_generation_job(self): + mid = rb(16) + result = {} + def work(): + result["r"] = Stamps.generate_stamp(mid, 30, no_expand=True) + + worker = threading.Thread(target=work, daemon=True) + worker.start() + self.assertTrue(self._wait_registered(mid)) + time.sleep(0.3) + Stamps.cancel_work(mid) + worker.join(timeout=30) + self.assertFalse(worker.is_alive()) + + stamp, value = result["r"] + self.assertEqual(stamp, None) + self.assertNotIn(mid, Stamps.active_jobs) + + @skipIf(not RNS.vendor.platformutils.is_linux(), "Direct multiprocess dispatchers are Linux-only paths") + def test_10_direct_multiprocess_dispatch(self): + # generate_stamp only routes to the multiprocess dispatchers + # at cost >= POOLING_THRESHOLD_COST, so they are exercised + # directly here at a trivial cost. This covers both the + # classic and managed worker mechanisms regardless of the + # Python version in use. + wb = rb(32) + mid1 = rb(16) + mid2 = rb(16) + + stamp, rounds = Stamps.job_linux(4*Stamps.COST_FP, wb, mid1, Stamps.STAMP_SIZE) + self.assertNotEqual(stamp, None) + self.assertTrue(Stamps.stamp_valid(stamp, 4, wb)) + self.assertGreater(rounds, 0) + self.assertNotIn(mid1, Stamps.active_jobs) + + stamp, rounds = Stamps.job_linux_managed(5*Stamps.COST_FP, wb, mid2, Stamps.STAMP_SIZE) + self.assertNotEqual(stamp, None) + self.assertTrue(Stamps.stamp_valid(stamp, 5, wb)) + self.assertGreater(rounds, 0) + self.assertNotIn(mid2, Stamps.active_jobs) + + +class TestStampPersistence(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.tmpdir = None + cls.owns_instance = False + # The full test suite shares one Reticulum instance, created + # by whichever test needed it first. If none is running, a + # temporary instance is created for this class only, and + # cleanly detached again afterwards. + if not RNS.Reticulum.storagepath: + cls.tmpdir = tempfile.mkdtemp(prefix="rns_stamp_tests_") + RNS.Reticulum(configdir=cls.tmpdir, loglevel=RNS.LOG_CRITICAL) + cls.owns_instance = True + + @classmethod + def tearDownClass(cls): + if cls.owns_instance: + RNS.Reticulum._Reticulum__instance = None + RNS.Reticulum.configpath = "" + RNS.Reticulum.storagepath = "" + RNS.Reticulum.cachepath = "" + RNS.Reticulum.interfacepath = "" + if cls.tmpdir: + shutil.rmtree(cls.tmpdir, ignore_errors=True) + + def _make_stamp(self, dest, cost=4): + # Destination stamps are STAMPLENGTH bytes, and are generated + # without workblock expansion, since the persistence layer + # recomputes the workblock as the unexpanded material. + return Stamps.generate_stamp(dest, cost, no_expand=True, size=RNS.Identity.STAMPLENGTH) + + def _cleanup(self, dest): + path = Stamps.destination_stamp_file(dest) + if path and os.path.isfile(path): + os.remove(path) + + def test_00_store_load_roundtrip(self): + dest = rb(16) + self.addCleanup(self._cleanup, dest) + stamp, value = self._make_stamp(dest) + + self.assertTrue(Stamps.store_destination_stamp(dest, stamp, value)) + loaded = Stamps.load_destination_stamp(dest, dest, 4) + self.assertNotEqual(loaded, None) + lstamp, lvalue = loaded + self.assertEqual(lstamp, stamp) + self.assertEqual(lvalue, value) + + def test_01_load_value_boundaries(self): + dest = rb(16) + self.addCleanup(self._cleanup, dest) + stamp, value = self._make_stamp(dest) + Stamps.store_destination_stamp(dest, stamp, value) + + # A stamp is always valid at its legacy integer value + vleg = Stamps.stamp_value_legacy(dest, stamp) + loaded = Stamps.load_destination_stamp(dest, dest, vleg) + self.assertNotEqual(loaded, None) + self.assertEqual(loaded[0], stamp) + + # ... and never at a full step above its own value + self.assertEqual(Stamps.load_destination_stamp(dest, dest, value + 1), None) + + def test_02_load_nonexistent(self): + self.assertEqual(Stamps.load_destination_stamp(rb(16), rb(16), 1), None) + + def test_03_load_corrupt_file(self): + dest = rb(16) + self.addCleanup(self._cleanup, dest) + stamp, value = self._make_stamp(dest) + Stamps.store_destination_stamp(dest, stamp, value) + with open(Stamps.destination_stamp_file(dest), "wb") as file: + file.write(rb(64)) + self.assertEqual(Stamps.load_destination_stamp(dest, dest, 1), None) + + def test_04_load_wrong_version(self): + dest = rb(16) + self.addCleanup(self._cleanup, dest) + stamp, value = self._make_stamp(dest) + Stamps.store_destination_stamp(dest, stamp, value) + with open(Stamps.destination_stamp_file(dest), "wb") as file: + file.write(msgpack.packb({"v": 99, "stamp": stamp, "value": value, "generated": time.time()})) + self.assertEqual(Stamps.load_destination_stamp(dest, dest, 1), None) + + def test_05_load_wrong_stamp_length(self): + dest = rb(16) + self.addCleanup(self._cleanup, dest) + stamp, value = self._make_stamp(dest) + Stamps.store_destination_stamp(dest, stamp, value) + with open(Stamps.destination_stamp_file(dest), "wb") as file: + file.write(msgpack.packb({"v": Stamps.STAMP_FILE_VERSION, "stamp": rb(8), "value": value, "generated": time.time()})) + self.assertEqual(Stamps.load_destination_stamp(dest, dest, 1), None) + + def test_06_stamp_paths(self): + directory = Stamps.stamps_dir(create=True) + self.assertNotEqual(directory, None) + self.assertTrue(os.path.isdir(directory)) + + dest = rb(16) + path = Stamps.destination_stamp_file(dest) + self.assertEqual(os.path.dirname(path), directory) + self.assertTrue(path.endswith(RNS.hexrep(dest, delimit=False) + Stamps.DESTINATION_STAMP_EXTENSION)) + + +if __name__ == '__main__': + unittest.main(verbosity=2)