diff --git a/Examples/Resource.py b/Examples/Resource.py index 896c936a..2cd55107 100644 --- a/Examples/Resource.py +++ b/Examples/Resource.py @@ -92,6 +92,7 @@ def resource_concluded(resource): if resource.status == RNS.Resource.COMPLETE: RNS.log(f"Resource {resource} received") RNS.log(f"Metadata: {resource.metadata}") + RNS.log(f"Application flag is {'set' if resource.app_flag else 'not set'}") RNS.log(f"Data length: {os.stat(resource.data.name).st_size}") RNS.log(f"Data can be read directly from: {resource.data}") RNS.log(f"Data can be moved or copied from: {resource.data.name}") @@ -192,16 +193,22 @@ def client_loop(): RNS.log(f"First 32 bytes of data: {RNS.hexrep(data[:32])}") # Generate some metadata - metadata = {"text": random_text_generator(), "numbers": [1,2,3,4], "blob": os.urandom(16)} + metadata = {"text": random_text_generator(), + "numbers": [1,2,3,4], "blob": os.urandom(16)} + + # An optional application-specific flag can be set + flag_set = True # Send the resource - resource = RNS.Resource(data, server_link, metadata=metadata, callback=resource_concluded_sending, auto_compress=False) + resource = RNS.Resource(data, server_link, metadata=metadata, app_flag=flag_set, + callback=resource_concluded_sending, auto_compress=False) # Alternatively, you can stream data # directly from an open file descriptor # with open("/path/to/file", "rb") as data_file: - # resource = RNS.Resource(data_file, server_link, metadata=metadata, callback=resource_concluded_sending, auto_compress=False) + # resource = RNS.Resource(data_file, server_link, metadata=metadata, + # callback=resource_concluded_sending, auto_compress=False) except Exception as e: RNS.log("Error while sending resource over the link: "+str(e)) diff --git a/RNS/Link.py b/RNS/Link.py index 019be996..b7dba789 100644 --- a/RNS/Link.py +++ b/RNS/Link.py @@ -70,7 +70,7 @@ class Link: ECPUBSIZE = 32+32 KEYSIZE = 32 - MDU = math.floor((RNS.Reticulum.MTU-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1 + MDU = math.floor((RNS.Reticulum.MTU-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES256_BLOCKSIZE)*RNS.Identity.AES256_BLOCKSIZE - 1 ESTABLISHMENT_TIMEOUT_PER_HOP = RNS.Reticulum.DEFAULT_PER_HOP_TIMEOUT """ @@ -597,7 +597,7 @@ class Link: def update_mdu(self): self.mdu = self.mtu - RNS.Reticulum.HEADER_MAXSIZE - RNS.Reticulum.IFAC_MIN_SIZE - self.mdu = math.floor((self.mtu-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1 + self.mdu = math.floor((self.mtu-RNS.Reticulum.IFAC_MIN_SIZE-RNS.Reticulum.HEADER_MINSIZE-RNS.Identity.TOKEN_OVERHEAD)/RNS.Identity.AES256_BLOCKSIZE)*RNS.Identity.AES256_BLOCKSIZE - 1 def rtt_packet(self, packet): try: diff --git a/RNS/Packet.py b/RNS/Packet.py index 31ddf7fb..b76ad676 100755 --- a/RNS/Packet.py +++ b/RNS/Packet.py @@ -103,7 +103,7 @@ class Packet: # With an MTU of 500, the maximum of data we can # send in a single encrypted packet is given by # the below calculation; 383 bytes. - ENCRYPTED_MDU = math.floor((RNS.Reticulum.MDU-RNS.Identity.TOKEN_OVERHEAD-RNS.Identity.KEYSIZE//16)/RNS.Identity.AES128_BLOCKSIZE)*RNS.Identity.AES128_BLOCKSIZE - 1 + ENCRYPTED_MDU = math.floor((RNS.Reticulum.MDU-RNS.Identity.TOKEN_OVERHEAD-RNS.Identity.KEYSIZE//16)/RNS.Identity.AES256_BLOCKSIZE)*RNS.Identity.AES256_BLOCKSIZE - 1 """ The maximum size of the payload data in a single encrypted packet """ diff --git a/RNS/Resource.py b/RNS/Resource.py index f5db9fb3..43210ebc 100644 --- a/RNS/Resource.py +++ b/RNS/Resource.py @@ -139,6 +139,7 @@ class Resource: HASHMAP_IS_NOT_EXHAUSTED = 0x00 HASHMAP_IS_EXHAUSTED = 0xFF + HASHMAP_IS_EMPTY = 0xFE # Status constants NONE = 0x00 @@ -182,6 +183,7 @@ class Resource: resource.hashmap_raw = adv.m resource.encrypted = True if resource.flags & 0x01 else False resource.compressed = True if resource.flags >> 1 & 0x01 else False + resource.app_flag = True if resource.flags >> 6 & 0x01 else False resource.initiator = False resource.callback = callback resource.__progress_callback = progress_callback @@ -247,7 +249,7 @@ class Resource: # The data passed can be either a bytes-array or a file opened # in binary read mode. def __init__(self, data, link, metadata=None, advertise=True, auto_compress=True, callback=None, progress_callback=None, - timeout = None, segment_index = 1, original_hash = None, request_id = None, is_response = False, sent_metadata_size=0): + timeout=None, segment_index = 1, original_hash=None, request_id=None, is_response=False, app_flag=False, sent_metadata_size=0): data_size = None resource_data = None @@ -257,6 +259,7 @@ class Resource: self.metadata = None self.has_metadata = False self.metadata_size = sent_metadata_size + self.app_flag = app_flag if metadata != None: packed_metadata = umsgpack.packb(metadata) @@ -503,11 +506,12 @@ class Resource: if self.hashmap[i+segment*seg_len] == None: self.hashmap_height += 1 self.hashmap[i+segment*seg_len] = hashmap[i*Resource.MAPHASH_LEN:(i+1)*Resource.MAPHASH_LEN] - if hashes < 1: + if hashes < 1 and segment != 0: RNS.log("Invalid HMU received, cancelling transfer", RNS.LOG_ERROR) self.cancel() else: + if hashes == 0 and segment == 0: RNS.log(f"Received HMU-less resource advertisement, requesting initial hashmap", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None self.waiting_for_hmu = False self.request_next() @@ -535,7 +539,7 @@ class Resource: else: return True def __advertise_job(self): - self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self).pack(), context=RNS.Packet.RESOURCE_ADV) + self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self, mdu=self.link.mdu).pack(), context=RNS.Packet.RESOURCE_ADV) while not self.link.ready_for_new_resource(): self.status = Resource.QUEUED sleep(0.25) @@ -594,7 +598,7 @@ class Resource: RNS.log("No part requests received, retrying resource advertisement...", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None self.retries_left -= 1 if not self.ensure_link(): return - self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self).pack(), context=RNS.Packet.RESOURCE_ADV) + self.advertisement_packet = RNS.Packet(self.link, ResourceAdvertisement(self, mdu=self.link.mdu).pack(), context=RNS.Packet.RESOURCE_ADV) self.advertisement_packet.send() self.last_activity = time.time() self.adv_sent = self.last_activity @@ -789,6 +793,7 @@ class Resource: is_response = self.is_response, advertise = False, auto_compress = self.auto_compress_option, + app_flag = self.app_flag, sent_metadata_size = self.metadata_size) if self.__progress_callback: self.next_segment.progress_callback(self.__progress_callback) @@ -949,23 +954,30 @@ class Resource: hashmap_exhausted = Resource.HASHMAP_IS_NOT_EXHAUSTED requested_hashes = b"" - i = 0; pn = self.consecutive_completed_height+1 - search_start = pn - search_size = self.window - - for part in self.parts[search_start:search_start+search_size]: - if part == None: - part_hash = self.hashmap[pn] - if part_hash != None: - requested_hashes += part_hash - self.outstanding_parts += 1 - i += 1 - else: - hashmap_exhausted = Resource.HASHMAP_IS_EXHAUSTED + # Initial advertisement-carried hashmap was empty, + # we can't request any real parts, and need an + # actual hashmap update first + if self.hashmap[0] == None: + hashmap_exhausted = Resource.HASHMAP_IS_EMPTY - pn += 1 - if i >= self.window or hashmap_exhausted == Resource.HASHMAP_IS_EXHAUSTED: - break + else: + i = 0; pn = self.consecutive_completed_height+1 + search_start = pn + search_size = self.window + + for part in self.parts[search_start:search_start+search_size]: + if part == None: + part_hash = self.hashmap[pn] + if part_hash != None: + requested_hashes += part_hash + self.outstanding_parts += 1 + i += 1 + else: + hashmap_exhausted = Resource.HASHMAP_IS_EXHAUSTED + + pn += 1 + if i >= self.window or hashmap_exhausted == Resource.HASHMAP_IS_EXHAUSTED: + break hmu_part = bytes([hashmap_exhausted]) if hashmap_exhausted == Resource.HASHMAP_IS_EXHAUSTED: @@ -973,6 +985,9 @@ class Resource: hmu_part += last_map_hash self.waiting_for_hmu = True + if hashmap_exhausted == Resource.HASHMAP_IS_EMPTY: + self.waiting_for_hmu = True + request_data = hmu_part + self.hash + requested_hashes request_packet = RNS.Packet(self.link, request_data, context = RNS.Packet.RESOURCE_REQ) @@ -1003,8 +1018,16 @@ class Resource: self.retries_left = self.max_retries - wants_more_hashmap = True if request_data[0] == Resource.HASHMAP_IS_EXHAUSTED else False - pad = 1+Resource.MAPHASH_LEN if wants_more_hashmap else 1 + pad = 1 + hashmap_is_empty = False + wants_more_hashmap = False + if request_data[0] == Resource.HASHMAP_IS_EMPTY: + wants_more_hashmap = True + hashmap_is_empty = True + + if request_data[0] == Resource.HASHMAP_IS_EXHAUSTED: + wants_more_hashmap = True + pad += Resource.MAPHASH_LEN requested_hashes = request_data[pad+RNS.Identity.HASHLENGTH//8:] @@ -1040,15 +1063,15 @@ class Resource: last_map_hash = request_data[1:Resource.MAPHASH_LEN+1] part_index = self.receiver_min_consecutive_height - search_start = part_index - search_end = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE + search_start = part_index if not hashmap_is_empty else 0 + search_end = self.receiver_min_consecutive_height+ResourceAdvertisement.COLLISION_GUARD_SIZE if not hashmap_is_empty else 0 for part in self.parts[search_start:search_end]: part_index += 1 if part.map_hash == last_map_hash: break self.receiver_min_consecutive_height = max(part_index-1-Resource.WINDOW_MAX, 0) - if part_index % ResourceAdvertisement.HASHMAP_MAX_LEN != 0: + if not hashmap_is_empty and part_index % ResourceAdvertisement.HASHMAP_MAX_LEN != 0: RNS.log("Resource sequencing error, cancelling transfer!", RNS.LOG_ERROR) self.cancel() return @@ -1286,8 +1309,9 @@ class ResourceAdvertisement: adv = ResourceAdvertisement.unpack(advertisement_packet.plaintext) return adv.d - def __init__(self, resource=None, request_id=None, is_response=False): + def __init__(self, resource=None, request_id=None, is_response=False, mdu=None): self.link = None + self.mdu = mdu or RNS.Link.MDU if resource != None: self.t = resource.size # Transfer size self.d = resource.total_size # Total uncompressed data size @@ -1298,6 +1322,7 @@ class ResourceAdvertisement: self.m = resource.hashmap # Resource hashmap self.c = resource.compressed # Compression flag self.e = resource.encrypted # Encryption flag + self.a = resource.app_flag # Application flag self.s = resource.split # Split flag self.x = resource.has_metadata # Metadata flag self.i = resource.segment_index # Segment index @@ -1315,7 +1340,7 @@ class ResourceAdvertisement: self.p = True # Flags - self.f = 0x00 | self.x << 5 | self.p << 4 | self.u << 3 | self.s << 2 | self.c << 1 | self.e + self.f = 0x00 | self.a << 6 | self.x << 5 | self.p << 4 | self.u << 3 | self.s << 2 | self.c << 1 | self.e def get_transfer_size(self): return self.t def get_data_size(self): return self.d @@ -1346,7 +1371,13 @@ class ResourceAdvertisement: "f": self.f, # Resource flags "m": hashmap } - return umsgpack.packb(dictionary) + packed_adv = umsgpack.packb(dictionary) + if len(packed_adv) > self.mdu: + RNS.log(f"Resource advertisement ({RNS.prettysize(len(packed_adv))}) exceeds link MDU, deferring initial HMU", RNS.LOG_DEBUG) if RNS.sl(RNS.LOG_DEBUG) else None + dictionary["m"] = b"" + packed_adv = umsgpack.packb(dictionary) + + return packed_adv @staticmethod def unpack(data): @@ -1370,6 +1401,7 @@ class ResourceAdvertisement: adv.u = True if ((adv.f >> 3) & 0x01) == 0x01 else False adv.p = True if ((adv.f >> 4) & 0x01) == 0x01 else False adv.x = True if ((adv.f >> 5) & 0x01) == 0x01 else False + adv.a = True if ((adv.f >> 6) & 0x01) == 0x01 else False if adv.t > Resource.MAX_EFFICIENT_SIZE*3: raise ValueError("Invalid transfer size")