Adds a random hash seed.

HISTORY

@@ -8,6 +8,7 @@ SnapRAID HISTORY
    To convert you can use the new 'rehash' command.
  * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4.
    They are a little faster than the previous unroll by 2.
+ * Added a hash seed to make hash values unique and not recognizable.
 
 3.1 2013/5
 ==========

check.c

@@ -68,7 +68,7 @@ static int blockcmp(struct snapraid_state* state, struct snapraid_block* block,
 	size = block_file_size(block, state->block_size);
 
 	/* now compute the hash of the valid part */
-	memhash(state->hash, hash, buffer, size);
+	memhash(state->hash, state->hashseed, hash, buffer, size);
 
 	/* compare the hash */
 	if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
@@ -700,7 +700,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 			/* if the block has the hash */
 			if (block_state == BLOCK_STATE_BLK) {
 				/* compute the hash of the block just read */
-				memhash(state->hash, hash, buffer[j], read_size);
+				memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
 
 				/* compare the hash */
 				if (memcmp(hash, block->hash, HASH_SIZE) != 0) {

dup.c

@@ -36,7 +36,7 @@ struct snapraid_hash {
 	tommy_hashdyn_node nodeset;
 };
 
-struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_file* file)
+struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
 {
 	struct snapraid_hash* hash;
 	block_off_t i;
@@ -58,7 +58,7 @@ struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_fil
 		}
 	}
 
-	memhash(HASH_MURMUR3, hash->hash, buf, file->blockmax * HASH_SIZE);
+	memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * HASH_SIZE);
 
 	free(buf);
 
@@ -111,7 +111,7 @@ void state_dup(struct snapraid_state* state)
 			if (file->size == 0)
 				continue;
 
-			hash = hash_alloc(disk, file);
+			hash = hash_alloc(state, disk, file);
 
 			/* if no hash, skip it */
 			if (!hash)

import.c

@@ -50,7 +50,6 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
 	int f;
 	int flags;
 	unsigned block_size = state->block_size;
-	unsigned hash = state->hash;
 
 	file = malloc_nofail(sizeof(struct snapraid_import_file));
 	file->path = strdup_nofail(path);
@@ -98,7 +97,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
 		block->file = file;
 		block->offset = offset;
 		block->size = read_size;
-		memhash(hash, block->hash, buffer, read_size);
+		memhash(state->hash, state->hashseed, block->hash, buffer, read_size);
 
 		tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash));
 
@@ -170,7 +169,7 @@ int state_import_fetch(struct snapraid_state* state, const unsigned char* hash,
 	}
 
 	/* recheck the hash */
-	memhash(state->hash, buffer_hash, buffer, read_size);
+	memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size);
 
 	if (memcmp(buffer_hash, hash, HASH_SIZE) != 0) {
 		fprintf(stderr, "Error in data reading file '%s'.\n", path);

murmurhash3.c

@@ -75,16 +75,16 @@ uint32_t c2 = 0xab0e9789;
 uint32_t c3 = 0x38b34ae5;
 uint32_t c4 = 0xa1e38b93;
 
-void MurmurHash3_x86_128(const void* key, unsigned len, void* out)
+void MurmurHash3_x86_128(const void* key, unsigned len, uint32_t seed, void* out)
 {
 	const uint8_t* data = key;
 	unsigned nblocks = len / 16;
 	unsigned i;
 
-	uint32_t h1 = 0;
-	uint32_t h2 = 0;
-	uint32_t h3 = 0;
-	uint32_t h4 = 0;
+	uint32_t h1 = seed;
+	uint32_t h2 = seed;
+	uint32_t h3 = seed;
+	uint32_t h4 = seed;
 
 	const uint32_t* blocks = (const uint32_t*)data;
 

rehash.c

@@ -41,6 +41,7 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks
 	block_off_t countpos;
 	block_off_t countmax;
 	unsigned error;
+	unsigned char seed[HASH_SIZE];
 
 	handle = handle_map(state, &diskmax);
 
@@ -48,6 +49,9 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks
 
 	error = 0;
 
+	/* set a random seed */
+	randomize(seed, HASH_SIZE);
+
 	/* check that we are in a sync complete state */
 	for(i=blockstart;i<blockmax;++i) {
 		/* for each disk, process the block */
@@ -141,12 +145,12 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks
 			countsize += read_size;
 
 			/* compute the new hash */
-			memhash(state->besthash, hash_new, buffer_aligned, read_size);
+			memhash(state->besthash, seed, hash_new, buffer_aligned, read_size);
 
 			/* compute the old hash */
 			/* note that we intentionally compute the new one before the old */
 			/* to ensure to detect any random bit flip in the memory */
-			memhash(state->hash, hash, buffer_aligned, read_size);
+			memhash(state->hash, state->hashseed, hash, buffer_aligned, read_size);
 
 			/* compare the hash */
 			if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
@@ -170,8 +174,9 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks
 
 	state_progress_end(state, countpos, countmax, countsize);
 
-	/* set the new hash */
+	/* set the new hash and seed */
 	state->hash = state->besthash;
+	memcpy(state->hashseed, seed, HASH_SIZE);
 
 	/* mark the state as needing write */
 	state->need_write = 1;

selftest.c

@@ -699,10 +699,32 @@ static void raid6test(unsigned diskmax, unsigned block_size)
 static void hashtest(void)
 {
 	unsigned i;
+	unsigned char seed[HASH_SIZE];
+
+#if 0
+	seed[0] = 0x5d;
+	seed[1] = 0x79;
+	seed[2] = 0x66;
+	seed[3] = 0xa7;
+	seed[4] = 0x73;
+	seed[5] = 0x27;
+	seed[6] = 0x02;
+	seed[7] = 0x2f;
+	seed[8] = 0x6a;
+	seed[9] = 0xa1;
+	seed[10] = 0x9e;
+	seed[11] = 0xc1;
+	seed[12] = 0x14;
+	seed[13] = 0x8c;
+	seed[14] = 0x9e;
+	seed[15] = 0x43;
+#else
+	memset(seed, 0, HASH_SIZE);
+#endif
 
 	for(i=0;TEST_MURMUR3[i].data;++i) {
 		unsigned char digest[HASH_SIZE];
-		memhash(HASH_MURMUR3, digest, TEST_MURMUR3[i].data, TEST_MURMUR3[i].len);
+		memhash(HASH_MURMUR3, seed, digest, TEST_MURMUR3[i].data, TEST_MURMUR3[i].len);
 		if (memcmp(digest, TEST_MURMUR3[i].digest, HASH_SIZE) != 0) {
 			fprintf(stderr, "Failed Murmur3 test vector\n");
 			exit(EXIT_FAILURE);
@@ -711,7 +733,7 @@ static void hashtest(void)
 
 	for(i=0;TEST_SPOOKY2[i].data;++i) {
 		unsigned char digest[HASH_SIZE];
-		memhash(HASH_SPOOKY2, digest, TEST_SPOOKY2[i].data, TEST_SPOOKY2[i].len);
+		memhash(HASH_SPOOKY2, seed, digest, TEST_SPOOKY2[i].data, TEST_SPOOKY2[i].len);
 		if (memcmp(digest, TEST_SPOOKY2[i].digest, HASH_SIZE) != 0) {
 			fprintf(stderr, "Failed Spooky2 test vector %u\n", i);
 			exit(EXIT_FAILURE);

speed.c

@@ -57,6 +57,7 @@ void speed(void)
 	int64_t dt;
 	unsigned i, j;
 	unsigned char digest[HASH_SIZE];
+	unsigned char seed[HASH_SIZE];
 
 	unsigned count;
 	unsigned delta = 100;
@@ -70,8 +71,10 @@ void speed(void)
 	buffer = malloc_nofail((diskmax + 2) * sizeof(void*));
 	for(i=0;i<diskmax+2;++i) {
 		buffer[i] = buffer_aligned + i * block_size;
-		memset(buffer[i], 0, block_size);
+		memset(buffer[i], i, block_size);
 	}
+	for(i=0;i<HASH_SIZE;++i)
+		seed[i] = i;
 
 	printf(PACKAGE " v" VERSION " by Andrea Mazzoleni, " PACKAGE_URL "\n");
 
@@ -156,7 +159,7 @@ void speed(void)
 
 	SPEED_START {
 		for(j=0;j<diskmax;++j) {
-			memhash(HASH_MURMUR3, digest, buffer[j], block_size);
+			memhash(HASH_MURMUR3, seed, digest, buffer[j], block_size);
 		}
 	} SPEED_STOP
 
@@ -164,7 +167,7 @@ void speed(void)
 
 	SPEED_START {
 		for(j=0;j<diskmax;++j) {
-			memhash(HASH_SPOOKY2, digest, buffer[j], block_size);
+			memhash(HASH_SPOOKY2, seed, digest, buffer[j], block_size);
 		}
 	} SPEED_STOP
 

spookyhash2.c

@@ -108,7 +108,7 @@
 //
 #define sc_const 0xdeadbeefdeadbeefLL
 
-void SpookyHash128(const void* message, size_t length, uint8_t* digest)
+void SpookyHash128(const void* message, size_t length, uint64_t seed0, uint64_t seed1, uint8_t* digest)
 {
 	uint64_t h0, h1, h2, h3, h4, h5, h6, h7, h8, h9, h10, h11;
 	uint64_t buf[sc_numVars];
@@ -120,8 +120,8 @@ void SpookyHash128(const void* message, size_t length, uint8_t* digest)
 	} u;
 	size_t remainder;
 
-	h0 = h3 = h6 = h9  = 0;
-	h1 = h4 = h7 = h10 = 0;
+	h0 = h3 = h6 = h9  = seed0;
+	h1 = h4 = h7 = h10 = seed1;
 	h2 = h5 = h8 = h11 = sc_const;
 
 	u.p8 = message;

state.c

@@ -637,6 +637,9 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
 	/* by default use the best hash */
 	state->hash = state->besthash;
 
+	/* by default use a random hash seed */
+	randomize(state->hashseed, HASH_SIZE);
+
 	if (state->gui) {
 		tommy_node* i;
 		fprintf(stdlog, "blocksize:%u\n", state->block_size);
@@ -884,6 +887,9 @@ void state_read(struct snapraid_state* state)
 	/* it's for compatibility with version 1.0 where MD5 was implicit. */
 	state->hash = HASH_UNDEFINED;
 
+	/* start with a zero seed */
+	memset(state->hashseed, 0, HASH_SIZE);
+
 	disk = 0;
 	file = 0;
 	line = 1;
@@ -1448,6 +1454,15 @@ void state_read(struct snapraid_state* state)
 				fprintf(stderr, "Invalid 'checksum' specification '%s' in '%s' at line %u\n", buffer, path, line);
 				exit(EXIT_FAILURE);
 			}
+		} else if (strcmp(tag, "seed") == 0) {
+			hash = oathash8(hash, 'e');
+
+			ret = sgethex(f, state->hashseed, HASH_SIZE);
+			if (ret < 0) {
+				fprintf(stderr, "Invalid 'seed' specification in '%s' at line %u\n", path, line);
+				exit(EXIT_FAILURE);
+			}
+			hash = oathashm(hash, state->hashseed, HASH_SIZE);
 		} else if (strcmp(tag, "blksize") == 0) {
 			block_off_t blksize;
 
@@ -1676,6 +1691,16 @@ void state_write(struct snapraid_state* state)
 		exit(EXIT_FAILURE);
 	}
 
+	sputsl("seed ", f);
+	hash = oathash8(hash, 'e');
+	sputhex(state->hashseed, HASH_SIZE, f);
+	hash = oathashm(hash, state->hashseed, HASH_SIZE);
+	sputeol(f);
+	if (serror(f)) {
+		fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
+		exit(EXIT_FAILURE);
+	}
+
 	/* for each map */
 	for(i=state->maplist;i!=0;i=i->next) {
 		struct snapraid_map* map = i->data;

state.h

@@ -50,6 +50,7 @@ struct snapraid_state {
 	uint64_t qarity_device; /**< Device identifier of the qarity. */
 	char pool[PATH_MAX]; /**< Path of the pool tree. */
 	uint64_t pool_device; /**< Device identifier of the pool. */
+	unsigned char hashseed[HASH_SIZE]; /**< Hash seed. */
 	unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */
 	unsigned hash; /**< Hash kind used. */
 	unsigned besthash; /**< Best hash suggested. */

sync.c

@@ -244,7 +244,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 			countsize += read_size;
 
 			/* now compute the hash */
-			memhash(state->hash, hash, buffer[j], read_size);
+			memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
 
 			if (block_has_hash(block)) {
 				/* compare the hash */

util.c

@@ -836,14 +836,26 @@ static inline uint64_t swap64(uint64_t v)
 #include "murmurhash3.c"
 #include "spookyhash2.c"
 
-void memhash(unsigned kind, void* digest, const void* src, unsigned size)
+void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, unsigned size)
 {
+	uint32_t seed32;
+	uint64_t seed64[2];
 	switch (kind) {
 	case HASH_MURMUR3 :
-		MurmurHash3_x86_128(src, size, digest);
+		seed32 = ((uint32_t*)seed)[0] ^ ((uint32_t*)seed)[1] ^ ((uint32_t*)seed)[2] ^ ((uint32_t*)seed)[3];
+#if WORDS_BIGENDIAN
+		seed32 = swap32(seed32);
+#endif
+		MurmurHash3_x86_128(src, size, seed32, digest);
 		break;
 	case HASH_SPOOKY2 :
-		SpookyHash128(src, size, digest);
+		seed64[0] = ((uint64_t*)seed)[0];
+		seed64[1] = ((uint64_t*)seed)[1];
+#if WORDS_BIGENDIAN
+		seed64[0] = swap64(seed64[0]);
+		seed64[1] = swap64(seed64[1]);
+#endif
+		SpookyHash128(src, size, seed64[0], seed64[1], digest);
 		break;
 	default:
 		fprintf(stderr, "Internal inconsistency in hash function %u\n", kind);
@@ -852,3 +864,17 @@ void memhash(unsigned kind, void* digest, const void* src, unsigned size)
 	}
 }
 
+/****************************************************************************/
+/* random */
+
+void randomize(void* void_ptr, unsigned size)
+{
+	unsigned char* ptr = void_ptr;
+	unsigned i;
+
+	srand(time(0));
+
+	for(i=0;i<size;++i)
+		ptr[i] = rand();
+}
+

util.h

@@ -394,8 +394,17 @@ char* strdup_nofail(const char* str);
 
 /**
  * Computes the HASH of a memory block.
+ * Seed is a 128 bit vector.
  */
-void memhash(unsigned kind, void* digest, const void* src, unsigned size);
+void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, unsigned size);
+
+/****************************************************************************/
+/* random */
+
+/**
+ * Write random values.
+ */
+void randomize(void* ptr, unsigned size);
 
 /****************************************************************************/
 /* sax */