From 472c85a4fb76e93b7fff1a9baa77356c25a43db0 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 30 Aug 2026 23:09:29 +0200 Subject: [PATCH] Centralize canonical SnapRAID block hashing semantics --- cmdline/check.c | 17 +++++++++-------- cmdline/dup.c | 19 +++++++++++++++++-- cmdline/import.c | 11 +++++++---- cmdline/scrub.c | 6 +++--- cmdline/search.c | 20 ++++++++++---------- cmdline/sync.c | 14 +++++++------- cmdline/util.c | 30 ++++++++++++++++++++++++++++++ cmdline/util.h | 22 +++++++++++++++++++++- 8 files changed, 104 insertions(+), 35 deletions(-) diff --git a/cmdline/check.c b/cmdline/check.c index 85ab028a..e12e8f75 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -270,11 +270,11 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl { unsigned char hash[HASH_MAX]; - /* now compute the hash of the valid part */ + /* compute the hash according to the block-hash semantics */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, buffer, pos_size); + memhash_block(state->prevhash, state->prevhashseed, hash, buffer, pos_size, state->block_size); } else { - memhash(state->hash, state->hashseed, hash, buffer, pos_size); + memhash_block(state->hash, state->hashseed, hash, buffer, pos_size, state->block_size); } /* compare the hash */ @@ -283,10 +283,11 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl } /* - * The hash covers only the logical bytes belonging to the file, while RAID + * Legacy hashes cover only the logical bytes belonging to the file, while RAID * operates on the complete zero-padded block. Verify the padding separately: - * otherwise corruption confined to the reconstructed tail could pass the hash - * check and later produce incorrect parity. + * otherwise corruption confined to the reconstructed tail could pass a legacy + * hash check and later produce incorrect parity. This is redundant for MuseAir, + * but also makes the canonical-block invariant explicit. */ if (pos_size < state->block_size) { if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) { @@ -1977,9 +1978,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna /* compute the hash of the block just read */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size); + memhash_block(state->prevhash, state->prevhashseed, hash, buffer[j], read_size, state->block_size); } else { - memhash(state->hash, state->hashseed, hash, buffer[j], read_size); + memhash_block(state->hash, state->hashseed, hash, buffer[j], read_size, state->block_size); } /* compare the hash */ diff --git a/cmdline/dup.c b/cmdline/dup.c index 39e3b66e..4f239bc1 100644 --- a/cmdline/dup.c +++ b/cmdline/dup.c @@ -22,18 +22,31 @@ struct snapraid_hash { tommy_hashdyn_node node; }; +static void hash_size_store(unsigned char* ptr, uint64_t size) +{ + ptr[0] = (unsigned char)(size >> 0); + ptr[1] = (unsigned char)(size >> 8); + ptr[2] = (unsigned char)(size >> 16); + ptr[3] = (unsigned char)(size >> 24); + ptr[4] = (unsigned char)(size >> 32); + ptr[5] = (unsigned char)(size >> 40); + ptr[6] = (unsigned char)(size >> 48); + ptr[7] = (unsigned char)(size >> 56); +} + struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) { struct snapraid_hash* hash; block_off_t i; unsigned char* buf; size_t hash_size = BLOCK_HASH_SIZE; + size_t data_size = file->blockmax * hash_size; hash = malloc_nofail(sizeof(struct snapraid_hash)); hash->disk = disk; hash->file = file; - buf = nalloc_nofail(file->blockmax, hash_size); + buf = malloc_nofail(data_size + sizeof(uint64_t)); /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { @@ -48,7 +61,9 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d } } - memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * hash_size); + hash_size_store(buf + data_size, (uint64_t)file->size); + + memhash(state->besthash, state->hashseed, hash->hash, buf, data_size + sizeof(uint64_t)); free(buf); diff --git a/cmdline/import.c b/cmdline/import.c index d3631aef..6644209f 100644 --- a/cmdline/import.c +++ b/cmdline/import.c @@ -128,17 +128,20 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t count += ret; } while (count < read_size); + if (read_size < block_size) + memset((unsigned char*)buffer + read_size, 0, block_size - read_size); + block->file = file; block->offset = offset; block->size = read_size; - memhash(state->hash, state->hashseed, block->hash, buffer, read_size); + memhash_block(state->hash, state->hashseed, block->hash, buffer, read_size, block_size); tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash)); /* if we are in a rehash state */ if (state->prevhash != HASH_UNDEFINED) { /* compute also the previous hash */ - memhash(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size); + memhash_block(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size, block_size); tommy_hashdyn_insert(&state->previmportset, &block->prevnodeset, block, import_block_hash(block->prevhash)); } @@ -373,9 +376,9 @@ static int state_import_fetch_candidate(struct snapraid_state* state, int rehash /* recheck the hash */ if (rehash) - memhash(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size); + memhash_block(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size, block_size); else - memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size); + memhash_block(state->hash, state->hashseed, buffer_hash, buffer, read_size, block_size); if (memcmp(buffer_hash, hash, BLOCK_HASH_SIZE) != 0) { /* diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 10fe1bc1..8dc3d313 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -701,13 +701,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par * same position may still fail verification. */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size); + memhash_block(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size, state->block_size); /* compute the new hash, and store it */ rehandle[diskcur].block = block; - memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size); + memhash_block(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size, state->block_size); } else { - memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size); + memhash_block(state->hash, state->hashseed, hash, buffer[diskcur], read_size, state->block_size); } /* until now is hash */ diff --git a/cmdline/search.c b/cmdline/search.c index 6d614e36..ed9a6583 100644 --- a/cmdline/search.c +++ b/cmdline/search.c @@ -109,21 +109,21 @@ int search_file_compare(const void* void_arg, const void* void_data) /* LCOV_EXCL_STOP */ } - /* compute the hash */ - if (arg->prevhash) - memhash(state->prevhash, state->prevhashseed, buffer_hash, arg->buffer, arg->read_size); - else - memhash(state->hash, state->hashseed, buffer_hash, arg->buffer, arg->read_size); - - /* check if the hash is matching */ - if (memcmp(buffer_hash, arg->block->hash, BLOCK_HASH_SIZE) != 0) - return -1; - if (arg->read_size != state->block_size) { /* fill the remaining with 0 */ memset(arg->buffer + arg->read_size, 0, state->block_size - arg->read_size); } + /* compute the block hash from the canonical buffer */ + if (arg->prevhash) + memhash_block(state->prevhash, state->prevhashseed, buffer_hash, arg->buffer, arg->read_size, state->block_size); + else + memhash_block(state->hash, state->hashseed, buffer_hash, arg->buffer, arg->read_size, state->block_size); + + /* check if the hash is matching */ + if (memcmp(buffer_hash, arg->block->hash, BLOCK_HASH_SIZE) != 0) + return -1; + return 0; } diff --git a/cmdline/sync.c b/cmdline/sync.c index 71dfc02b..5e1de52c 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -507,9 +507,9 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta /* now compute the hash */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, buffer, read_size); + memhash_block(state->prevhash, state->prevhashseed, hash, buffer, read_size, state->block_size); } else { - memhash(state->hash, state->hashseed, hash, buffer, read_size); + memhash_block(state->hash, state->hashseed, hash, buffer, read_size, state->block_size); } /* until now is hash */ @@ -1219,13 +1219,13 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* now compute the hash */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size); + memhash_block(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size, state->block_size); /* compute the new hash, and store it */ rehandle[diskcur].block = block; - memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size); + memhash_block(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size, state->block_size); } else { - memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size); + memhash_block(state->hash, state->hashseed, hash, buffer[diskcur], read_size, state->block_size); } /* until now is hash */ @@ -1475,9 +1475,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* compute the hash of the recovered block */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, block_buffer, size); + memhash_block(state->prevhash, state->prevhashseed, hash, block_buffer, size, state->block_size); } else { - memhash(state->hash, state->hashseed, hash, block_buffer, size); + memhash_block(state->hash, state->hashseed, hash, block_buffer, size, state->block_size); } /* until now is hash */ diff --git a/cmdline/util.c b/cmdline/util.c index 7e212e12..f018761a 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -775,6 +775,36 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* } } +void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t logical_size, size_t block_size) +{ + assert(logical_size <= block_size); + + switch (kind) { + case HASH_MUSEAIR : + /* + * MuseAir defines the identity of the complete canonical RAID + * block. The caller must have zero-padded the buffer after the + * logical end of the file block. + */ + memhash(kind, seed, digest, src, block_size); + break; + case HASH_MURMUR3 : + case HASH_SPOOKY2 : + /* + * Preserve the historical SnapRAID hash semantics for content + * files using legacy hashes. + */ + memhash(kind, seed, digest, src, logical_size); + break; + default : + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Internal inconsistency in hash function %u\n", kind); + exit(EXIT_FAILURE); + break; + /* LCOV_EXCL_STOP */ + } +} + const char* hash_config_name(unsigned kind) { switch (kind) { diff --git a/cmdline/util.h b/cmdline/util.h index babdcf84..bc43fbeb 100644 --- a/cmdline/util.h +++ b/cmdline/util.h @@ -226,11 +226,31 @@ unsigned membesthash(void); const char* memhashname(unsigned kind); /** - * Compute the HASH of a memory block. + * Compute the HASH of a memory buffer. * Seed is a 128 bit vector. + * + * This is the raw hash primitive. It hashes exactly size bytes and does not + * apply SnapRAID block semantics. */ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size); +/** + * Compute the hash of a SnapRAID data block. + * + * logical_size is the number of bytes logically belonging to the file. + * block_size is the complete zero-padded RAID block size. + * + * Legacy hashes preserve their historical semantics and hash only + * logical_size bytes. + * + * MuseAir hashes the complete canonical RAID block, including the zero + * padding up to block_size. + * + * The caller must guarantee that logical_size <= block_size and, when + * logical_size < block_size, src[logical_size .. block_size-1] == 0. + */ +void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t logical_size, size_t block_size); + /** * Return the hash name. */