diff --git a/HISTORY b/HISTORY index acb71464..356b5ff4 100644 --- a/HISTORY +++ b/HISTORY @@ -8,6 +8,7 @@ SnapRAID HISTORY To convert you can use the new 'rehash' command. * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4. They are a little faster than the previous unroll by 2. + * Added a hash seed to make hash values unique and not recognizable. 3.1 2013/5 ========== diff --git a/check.c b/check.c index e8d11913..43078534 100644 --- a/check.c +++ b/check.c @@ -68,7 +68,7 @@ static int blockcmp(struct snapraid_state* state, struct snapraid_block* block, size = block_file_size(block, state->block_size); /* now compute the hash of the valid part */ - memhash(state->hash, hash, buffer, size); + memhash(state->hash, state->hashseed, hash, buffer, size); /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { @@ -700,7 +700,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* if the block has the hash */ if (block_state == BLOCK_STATE_BLK) { /* compute the hash of the block just read */ - memhash(state->hash, hash, buffer[j], read_size); + memhash(state->hash, state->hashseed, hash, buffer[j], read_size); /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { diff --git a/dup.c b/dup.c index 7d130889..1480967c 100644 --- a/dup.c +++ b/dup.c @@ -36,7 +36,7 @@ struct snapraid_hash { tommy_hashdyn_node nodeset; }; -struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_file* file) +struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) { struct snapraid_hash* hash; block_off_t i; @@ -58,7 +58,7 @@ struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_fil } } - memhash(HASH_MURMUR3, hash->hash, buf, file->blockmax * HASH_SIZE); + memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * HASH_SIZE); free(buf); @@ -111,7 +111,7 @@ void state_dup(struct snapraid_state* state) if (file->size == 0) continue; - hash = hash_alloc(disk, file); + hash = hash_alloc(state, disk, file); /* if no hash, skip it */ if (!hash) diff --git a/import.c b/import.c index c35c8570..1f9e56ff 100644 --- a/import.c +++ b/import.c @@ -50,7 +50,6 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t int f; int flags; unsigned block_size = state->block_size; - unsigned hash = state->hash; file = malloc_nofail(sizeof(struct snapraid_import_file)); file->path = strdup_nofail(path); @@ -98,7 +97,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t block->file = file; block->offset = offset; block->size = read_size; - memhash(hash, block->hash, buffer, read_size); + memhash(state->hash, state->hashseed, block->hash, buffer, read_size); tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash)); @@ -170,7 +169,7 @@ int state_import_fetch(struct snapraid_state* state, const unsigned char* hash, } /* recheck the hash */ - memhash(state->hash, buffer_hash, buffer, read_size); + memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size); if (memcmp(buffer_hash, hash, HASH_SIZE) != 0) { fprintf(stderr, "Error in data reading file '%s'.\n", path); diff --git a/murmurhash3.c b/murmurhash3.c index 2de9fc67..e7e46267 100644 --- a/murmurhash3.c +++ b/murmurhash3.c @@ -75,16 +75,16 @@ uint32_t c2 = 0xab0e9789; uint32_t c3 = 0x38b34ae5; uint32_t c4 = 0xa1e38b93; -void MurmurHash3_x86_128(const void* key, unsigned len, void* out) +void MurmurHash3_x86_128(const void* key, unsigned len, uint32_t seed, void* out) { const uint8_t* data = key; unsigned nblocks = len / 16; unsigned i; - uint32_t h1 = 0; - uint32_t h2 = 0; - uint32_t h3 = 0; - uint32_t h4 = 0; + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; const uint32_t* blocks = (const uint32_t*)data; diff --git a/rehash.c b/rehash.c index 38397196..e1edd7d3 100644 --- a/rehash.c +++ b/rehash.c @@ -41,6 +41,7 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks block_off_t countpos; block_off_t countmax; unsigned error; + unsigned char seed[HASH_SIZE]; handle = handle_map(state, &diskmax); @@ -48,6 +49,9 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks error = 0; + /* set a random seed */ + randomize(seed, HASH_SIZE); + /* check that we are in a sync complete state */ for(i=blockstart;ibesthash, hash_new, buffer_aligned, read_size); + memhash(state->besthash, seed, hash_new, buffer_aligned, read_size); /* compute the old hash */ /* note that we intentionally compute the new one before the old */ /* to ensure to detect any random bit flip in the memory */ - memhash(state->hash, hash, buffer_aligned, read_size); + memhash(state->hash, state->hashseed, hash, buffer_aligned, read_size); /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { @@ -170,8 +174,9 @@ static int state_rehash_process(struct snapraid_state* state, block_off_t blocks state_progress_end(state, countpos, countmax, countsize); - /* set the new hash */ + /* set the new hash and seed */ state->hash = state->besthash; + memcpy(state->hashseed, seed, HASH_SIZE); /* mark the state as needing write */ state->need_write = 1; diff --git a/selftest.c b/selftest.c index 8383c8b1..f214387f 100644 --- a/selftest.c +++ b/selftest.c @@ -699,10 +699,32 @@ static void raid6test(unsigned diskmax, unsigned block_size) static void hashtest(void) { unsigned i; + unsigned char seed[HASH_SIZE]; + +#if 0 + seed[0] = 0x5d; + seed[1] = 0x79; + seed[2] = 0x66; + seed[3] = 0xa7; + seed[4] = 0x73; + seed[5] = 0x27; + seed[6] = 0x02; + seed[7] = 0x2f; + seed[8] = 0x6a; + seed[9] = 0xa1; + seed[10] = 0x9e; + seed[11] = 0xc1; + seed[12] = 0x14; + seed[13] = 0x8c; + seed[14] = 0x9e; + seed[15] = 0x43; +#else + memset(seed, 0, HASH_SIZE); +#endif for(i=0;TEST_MURMUR3[i].data;++i) { unsigned char digest[HASH_SIZE]; - memhash(HASH_MURMUR3, digest, TEST_MURMUR3[i].data, TEST_MURMUR3[i].len); + memhash(HASH_MURMUR3, seed, digest, TEST_MURMUR3[i].data, TEST_MURMUR3[i].len); if (memcmp(digest, TEST_MURMUR3[i].digest, HASH_SIZE) != 0) { fprintf(stderr, "Failed Murmur3 test vector\n"); exit(EXIT_FAILURE); @@ -711,7 +733,7 @@ static void hashtest(void) for(i=0;TEST_SPOOKY2[i].data;++i) { unsigned char digest[HASH_SIZE]; - memhash(HASH_SPOOKY2, digest, TEST_SPOOKY2[i].data, TEST_SPOOKY2[i].len); + memhash(HASH_SPOOKY2, seed, digest, TEST_SPOOKY2[i].data, TEST_SPOOKY2[i].len); if (memcmp(digest, TEST_SPOOKY2[i].digest, HASH_SIZE) != 0) { fprintf(stderr, "Failed Spooky2 test vector %u\n", i); exit(EXIT_FAILURE); diff --git a/speed.c b/speed.c index 08b4fb36..47bd89b6 100644 --- a/speed.c +++ b/speed.c @@ -57,6 +57,7 @@ void speed(void) int64_t dt; unsigned i, j; unsigned char digest[HASH_SIZE]; + unsigned char seed[HASH_SIZE]; unsigned count; unsigned delta = 100; @@ -70,8 +71,10 @@ void speed(void) buffer = malloc_nofail((diskmax + 2) * sizeof(void*)); for(i=0;ihash = state->besthash; + /* by default use a random hash seed */ + randomize(state->hashseed, HASH_SIZE); + if (state->gui) { tommy_node* i; fprintf(stdlog, "blocksize:%u\n", state->block_size); @@ -884,6 +887,9 @@ void state_read(struct snapraid_state* state) /* it's for compatibility with version 1.0 where MD5 was implicit. */ state->hash = HASH_UNDEFINED; + /* start with a zero seed */ + memset(state->hashseed, 0, HASH_SIZE); + disk = 0; file = 0; line = 1; @@ -1448,6 +1454,15 @@ void state_read(struct snapraid_state* state) fprintf(stderr, "Invalid 'checksum' specification '%s' in '%s' at line %u\n", buffer, path, line); exit(EXIT_FAILURE); } + } else if (strcmp(tag, "seed") == 0) { + hash = oathash8(hash, 'e'); + + ret = sgethex(f, state->hashseed, HASH_SIZE); + if (ret < 0) { + fprintf(stderr, "Invalid 'seed' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + hash = oathashm(hash, state->hashseed, HASH_SIZE); } else if (strcmp(tag, "blksize") == 0) { block_off_t blksize; @@ -1676,6 +1691,16 @@ void state_write(struct snapraid_state* state) exit(EXIT_FAILURE); } + sputsl("seed ", f); + hash = oathash8(hash, 'e'); + sputhex(state->hashseed, HASH_SIZE, f); + hash = oathashm(hash, state->hashseed, HASH_SIZE); + sputeol(f); + if (serror(f)) { + fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); + exit(EXIT_FAILURE); + } + /* for each map */ for(i=state->maplist;i!=0;i=i->next) { struct snapraid_map* map = i->data; diff --git a/state.h b/state.h index b8c40026..4184ef00 100644 --- a/state.h +++ b/state.h @@ -50,6 +50,7 @@ struct snapraid_state { uint64_t qarity_device; /**< Device identifier of the qarity. */ char pool[PATH_MAX]; /**< Path of the pool tree. */ uint64_t pool_device; /**< Device identifier of the pool. */ + unsigned char hashseed[HASH_SIZE]; /**< Hash seed. */ unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */ unsigned hash; /**< Hash kind used. */ unsigned besthash; /**< Best hash suggested. */ diff --git a/sync.c b/sync.c index 127801bb..65245ba9 100644 --- a/sync.c +++ b/sync.c @@ -244,7 +244,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari countsize += read_size; /* now compute the hash */ - memhash(state->hash, hash, buffer[j], read_size); + memhash(state->hash, state->hashseed, hash, buffer[j], read_size); if (block_has_hash(block)) { /* compare the hash */ diff --git a/util.c b/util.c index 7e50e9d3..43c8bd58 100644 --- a/util.c +++ b/util.c @@ -836,14 +836,26 @@ static inline uint64_t swap64(uint64_t v) #include "murmurhash3.c" #include "spookyhash2.c" -void memhash(unsigned kind, void* digest, const void* src, unsigned size) +void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, unsigned size) { + uint32_t seed32; + uint64_t seed64[2]; switch (kind) { case HASH_MURMUR3 : - MurmurHash3_x86_128(src, size, digest); + seed32 = ((uint32_t*)seed)[0] ^ ((uint32_t*)seed)[1] ^ ((uint32_t*)seed)[2] ^ ((uint32_t*)seed)[3]; +#if WORDS_BIGENDIAN + seed32 = swap32(seed32); +#endif + MurmurHash3_x86_128(src, size, seed32, digest); break; case HASH_SPOOKY2 : - SpookyHash128(src, size, digest); + seed64[0] = ((uint64_t*)seed)[0]; + seed64[1] = ((uint64_t*)seed)[1]; +#if WORDS_BIGENDIAN + seed64[0] = swap64(seed64[0]); + seed64[1] = swap64(seed64[1]); +#endif + SpookyHash128(src, size, seed64[0], seed64[1], digest); break; default: fprintf(stderr, "Internal inconsistency in hash function %u\n", kind); @@ -852,3 +864,17 @@ void memhash(unsigned kind, void* digest, const void* src, unsigned size) } } +/****************************************************************************/ +/* random */ + +void randomize(void* void_ptr, unsigned size) +{ + unsigned char* ptr = void_ptr; + unsigned i; + + srand(time(0)); + + for(i=0;i