Centralize canonical SnapRAID block hashing semantics

cmdline/check.c

@@ -270,11 +270,11 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl
 {
 	unsigned char hash[HASH_MAX];
 
-	/* now compute the hash of the valid part */
+	/* compute the hash according to the block-hash semantics */
 	if (rehash) {
-		memhash(state->prevhash, state->prevhashseed, hash, buffer, pos_size);
+		memhash_block(state->prevhash, state->prevhashseed, hash, buffer, pos_size, state->block_size);
 	} else {
-		memhash(state->hash, state->hashseed, hash, buffer, pos_size);
+		memhash_block(state->hash, state->hashseed, hash, buffer, pos_size, state->block_size);
 	}
 
 	/* compare the hash */
@@ -283,10 +283,11 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl
 	}
 
 	/*
-	 * The hash covers only the logical bytes belonging to the file, while RAID
+	 * Legacy hashes cover only the logical bytes belonging to the file, while RAID
 	 * operates on the complete zero-padded block. Verify the padding separately:
-	 * otherwise corruption confined to the reconstructed tail could pass the hash
+	 * otherwise corruption confined to the reconstructed tail could pass a legacy
-	 * check and later produce incorrect parity.
+	 * hash check and later produce incorrect parity. This is redundant for MuseAir,
+	 * but also makes the canonical-block invariant explicit.
 	 */
 	if (pos_size < state->block_size) {
 		if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) {
@@ -1977,9 +1978,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
 
 			/* compute the hash of the block just read */
 			if (rehash) {
-				memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
+				memhash_block(state->prevhash, state->prevhashseed, hash, buffer[j], read_size, state->block_size);
 			} else {
-				memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
+				memhash_block(state->hash, state->hashseed, hash, buffer[j], read_size, state->block_size);
 			}
 
 			/* compare the hash */

cmdline/import.c

@@ -128,17 +128,20 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
 			count += ret;
 		} while (count < read_size);
 
+		if (read_size < block_size)
+			memset((unsigned char*)buffer + read_size, 0, block_size - read_size);
+
 		block->file = file;
 		block->offset = offset;
 		block->size = read_size;
 
-		memhash(state->hash, state->hashseed, block->hash, buffer, read_size);
+		memhash_block(state->hash, state->hashseed, block->hash, buffer, read_size, block_size);
 		tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash));
 
 		/* if we are in a rehash state */
 		if (state->prevhash != HASH_UNDEFINED) {
 			/* compute also the previous hash */
-			memhash(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size);
+			memhash_block(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size, block_size);
 			tommy_hashdyn_insert(&state->previmportset, &block->prevnodeset, block, import_block_hash(block->prevhash));
 		}
 
@@ -373,9 +376,9 @@ static int state_import_fetch_candidate(struct snapraid_state* state, int rehash
 
 	/* recheck the hash */
 	if (rehash)
-		memhash(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size);
+		memhash_block(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size, block_size);
 	else
-		memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size);
+		memhash_block(state->hash, state->hashseed, buffer_hash, buffer, read_size, block_size);
 
 	if (memcmp(buffer_hash, hash, BLOCK_HASH_SIZE) != 0) {
 		/*

cmdline/scrub.c

@@ -701,13 +701,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 			 * same position may still fail verification.
 			 */
 			if (rehash) {
-				memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
+				memhash_block(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size, state->block_size);
 
 				/* compute the new hash, and store it */
 				rehandle[diskcur].block = block;
-				memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
+				memhash_block(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size, state->block_size);
 			} else {
-				memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
+				memhash_block(state->hash, state->hashseed, hash, buffer[diskcur], read_size, state->block_size);
 			}
 
 			/* until now is hash */

cmdline/search.c

@@ -109,21 +109,21 @@ int search_file_compare(const void* void_arg, const void* void_data)
 		/* LCOV_EXCL_STOP */
 	}
 
-	/* compute the hash */
-	if (arg->prevhash)
-		memhash(state->prevhash, state->prevhashseed, buffer_hash, arg->buffer, arg->read_size);
-	else
-		memhash(state->hash, state->hashseed, buffer_hash, arg->buffer, arg->read_size);
-
-	/* check if the hash is matching */
-	if (memcmp(buffer_hash, arg->block->hash, BLOCK_HASH_SIZE) != 0)
-		return -1;
-
 	if (arg->read_size != state->block_size) {
 		/* fill the remaining with 0 */
 		memset(arg->buffer + arg->read_size, 0, state->block_size - arg->read_size);
 	}
 
+	/* compute the block hash from the canonical buffer */
+	if (arg->prevhash)
+		memhash_block(state->prevhash, state->prevhashseed, buffer_hash, arg->buffer, arg->read_size, state->block_size);
+	else
+		memhash_block(state->hash, state->hashseed, buffer_hash, arg->buffer, arg->read_size, state->block_size);
+
+	/* check if the hash is matching */
+	if (memcmp(buffer_hash, arg->block->hash, BLOCK_HASH_SIZE) != 0)
+		return -1;
+
 	return 0;
 }
 

cmdline/sync.c

@@ -507,9 +507,9 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
 
 			/* now compute the hash */
 			if (rehash) {
-				memhash(state->prevhash, state->prevhashseed, hash, buffer, read_size);
+				memhash_block(state->prevhash, state->prevhashseed, hash, buffer, read_size, state->block_size);
 			} else {
-				memhash(state->hash, state->hashseed, hash, buffer, read_size);
+				memhash_block(state->hash, state->hashseed, hash, buffer, read_size, state->block_size);
 			}
 
 			/* until now is hash */
@@ -1219,13 +1219,13 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 
 			/* now compute the hash */
 			if (rehash) {
-				memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
+				memhash_block(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size, state->block_size);
 
 				/* compute the new hash, and store it */
 				rehandle[diskcur].block = block;
-				memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
+				memhash_block(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size, state->block_size);
 			} else {
-				memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
+				memhash_block(state->hash, state->hashseed, hash, buffer[diskcur], read_size, state->block_size);
 			}
 
 			/* until now is hash */
@@ -1475,9 +1475,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 
 							/* compute the hash of the recovered block */
 							if (rehash) {
-								memhash(state->prevhash, state->prevhashseed, hash, block_buffer, size);
+								memhash_block(state->prevhash, state->prevhashseed, hash, block_buffer, size, state->block_size);
 							} else {
-								memhash(state->hash, state->hashseed, hash, block_buffer, size);
+								memhash_block(state->hash, state->hashseed, hash, block_buffer, size, state->block_size);
 							}
 
 							/* until now is hash */

cmdline/util.c

@@ -775,6 +775,36 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void*
 	}
 }
 
+void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t logical_size, size_t block_size)
+{
+	assert(logical_size <= block_size);
+
+	switch (kind) {
+	case HASH_MUSEAIR :
+		/*
+		 * MuseAir defines the identity of the complete canonical RAID
+		 * block. The caller must have zero-padded the buffer after the
+		 * logical end of the file block.
+		 */
+		memhash(kind, seed, digest, src, block_size);
+		break;
+	case HASH_MURMUR3 :
+	case HASH_SPOOKY2 :
+		/*
+		 * Preserve the historical SnapRAID hash semantics for content
+		 * files using legacy hashes.
+		 */
+		memhash(kind, seed, digest, src, logical_size);
+		break;
+	default :
+		/* LCOV_EXCL_START */
+		log_fatal(EINTERNAL, "Internal inconsistency in hash function %u\n", kind);
+		exit(EXIT_FAILURE);
+		break;
+		/* LCOV_EXCL_STOP */
+	}
+}
+
 const char* hash_config_name(unsigned kind)
 {
 	switch (kind) {

cmdline/util.h

@@ -226,11 +226,31 @@ unsigned membesthash(void);
 const char* memhashname(unsigned kind);
 
 /**
- * Compute the HASH of a memory block.
+ * Compute the HASH of a memory buffer.
  * Seed is a 128 bit vector.
+ *
+ * This is the raw hash primitive. It hashes exactly size bytes and does not
+ * apply SnapRAID block semantics.
  */
 void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size);
 
+/**
+ * Compute the hash of a SnapRAID data block.
+ *
+ * logical_size is the number of bytes logically belonging to the file.
+ * block_size is the complete zero-padded RAID block size.
+ *
+ * Legacy hashes preserve their historical semantics and hash only
+ * logical_size bytes.
+ *
+ * MuseAir hashes the complete canonical RAID block, including the zero
+ * padding up to block_size.
+ *
+ * The caller must guarantee that logical_size <= block_size and, when
+ * logical_size < block_size, src[logical_size .. block_size-1] == 0.
+ */
+void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t logical_size, size_t block_size);
+
 /**
  * Return the hash name.
  */