rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Centralize canonical SnapRAID block hashing semantics

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-30 23:09:29 +02:00
parent 327c0e7aac
commit 472c85a4fb
8 changed files with 104 additions and 35 deletions
Download patch file Download diff file Expand all files Collapse all files

17
cmdline/check.c
View file

@ -270,11 +270,11 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl
{
unsigned char hash[HASH_MAX];
/* now compute the hash of the valid part */
/* compute the hash according to the block-hash semantics */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer, pos_size);
memhash_block(state->prevhash, state->prevhashseed, hash, buffer, pos_size, state->block_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer, pos_size);
memhash_block(state->hash, state->hashseed, hash, buffer, pos_size, state->block_size);
}
/* compare the hash */
@ -283,10 +283,11 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl
}
/*
* The hash covers only the logical bytes belonging to the file, while RAID
* Legacy hashes cover only the logical bytes belonging to the file, while RAID
* operates on the complete zero-padded block. Verify the padding separately:
* otherwise corruption confined to the reconstructed tail could pass the hash
* check and later produce incorrect parity.
* otherwise corruption confined to the reconstructed tail could pass a legacy
* hash check and later produce incorrect parity. This is redundant for MuseAir,
* but also makes the canonical-block invariant explicit.
*/
if (pos_size < state->block_size) {
if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) {
@ -1977,9 +1978,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* compute the hash of the block just read */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
memhash_block(state->prevhash, state->prevhashseed, hash, buffer[j], read_size, state->block_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
memhash_block(state->hash, state->hashseed, hash, buffer[j], read_size, state->block_size);
}
/* compare the hash */

19
cmdline/dup.c
View file

@ -22,18 +22,31 @@ struct snapraid_hash {
tommy_hashdyn_node node;
};
static void hash_size_store(unsigned char* ptr, uint64_t size)
{
ptr[0] = (unsigned char)(size >> 0);
ptr[1] = (unsigned char)(size >> 8);
ptr[2] = (unsigned char)(size >> 16);
ptr[3] = (unsigned char)(size >> 24);
ptr[4] = (unsigned char)(size >> 32);
ptr[5] = (unsigned char)(size >> 40);
ptr[6] = (unsigned char)(size >> 48);
ptr[7] = (unsigned char)(size >> 56);
}
struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file)
{
struct snapraid_hash* hash;
block_off_t i;
unsigned char* buf;
size_t hash_size = BLOCK_HASH_SIZE;
size_t data_size = file->blockmax * hash_size;
hash = malloc_nofail(sizeof(struct snapraid_hash));
hash->disk = disk;
hash->file = file;
buf = nalloc_nofail(file->blockmax, hash_size);
buf = malloc_nofail(data_size + sizeof(uint64_t));
/* set the back pointer */
for (i = 0; i < file->blockmax; ++i) {
@ -48,7 +61,9 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d
}
}
memhash(state->besthash, state->hashseed, hash->hash, buf, file->blockmax * hash_size);
hash_size_store(buf + data_size, (uint64_t)file->size);
memhash(state->besthash, state->hashseed, hash->hash, buf, data_size + sizeof(uint64_t));
free(buf);

11
cmdline/import.c
View file

@ -128,17 +128,20 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
count += ret;
} while (count < read_size);
if (read_size < block_size)
memset((unsigned char*)buffer + read_size, 0, block_size - read_size);
block->file = file;
block->offset = offset;
block->size = read_size;
memhash(state->hash, state->hashseed, block->hash, buffer, read_size);
memhash_block(state->hash, state->hashseed, block->hash, buffer, read_size, block_size);
tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash));
/* if we are in a rehash state */
if (state->prevhash != HASH_UNDEFINED) {
/* compute also the previous hash */
memhash(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size);
memhash_block(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size, block_size);
tommy_hashdyn_insert(&state->previmportset, &block->prevnodeset, block, import_block_hash(block->prevhash));
}
@ -373,9 +376,9 @@ static int state_import_fetch_candidate(struct snapraid_state* state, int rehash
/* recheck the hash */
if (rehash)
memhash(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size);
memhash_block(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size, block_size);
else
memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size);
memhash_block(state->hash, state->hashseed, buffer_hash, buffer, read_size, block_size);
if (memcmp(buffer_hash, hash, BLOCK_HASH_SIZE) != 0) {
/*

6
cmdline/scrub.c
View file

@ -701,13 +701,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
* same position may still fail verification.
*/
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
memhash_block(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size, state->block_size);
/* compute the new hash, and store it */
rehandle[diskcur].block = block;
memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
memhash_block(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size, state->block_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
memhash_block(state->hash, state->hashseed, hash, buffer[diskcur], read_size, state->block_size);
}
/* until now is hash */

20
cmdline/search.c
View file

@ -109,21 +109,21 @@ int search_file_compare(const void* void_arg, const void* void_data)
/* LCOV_EXCL_STOP */
}
/* compute the hash */
if (arg->prevhash)
memhash(state->prevhash, state->prevhashseed, buffer_hash, arg->buffer, arg->read_size);
else
memhash(state->hash, state->hashseed, buffer_hash, arg->buffer, arg->read_size);
/* check if the hash is matching */
if (memcmp(buffer_hash, arg->block->hash, BLOCK_HASH_SIZE) != 0)
return -1;
if (arg->read_size != state->block_size) {
/* fill the remaining with 0 */
memset(arg->buffer + arg->read_size, 0, state->block_size - arg->read_size);
}
/* compute the block hash from the canonical buffer */
if (arg->prevhash)
memhash_block(state->prevhash, state->prevhashseed, buffer_hash, arg->buffer, arg->read_size, state->block_size);
else
memhash_block(state->hash, state->hashseed, buffer_hash, arg->buffer, arg->read_size, state->block_size);
/* check if the hash is matching */
if (memcmp(buffer_hash, arg->block->hash, BLOCK_HASH_SIZE) != 0)
return -1;
return 0;
}

14
cmdline/sync.c
View file

@ -507,9 +507,9 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta
/* now compute the hash */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer, read_size);
memhash_block(state->prevhash, state->prevhashseed, hash, buffer, read_size, state->block_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer, read_size);
memhash_block(state->hash, state->hashseed, hash, buffer, read_size, state->block_size);
}
/* until now is hash */
@ -1219,13 +1219,13 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* now compute the hash */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size);
memhash_block(state->prevhash, state->prevhashseed, hash, buffer[diskcur], read_size, state->block_size);
/* compute the new hash, and store it */
rehandle[diskcur].block = block;
memhash(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size);
memhash_block(state->hash, state->hashseed, rehandle[diskcur].hash, buffer[diskcur], read_size, state->block_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[diskcur], read_size);
memhash_block(state->hash, state->hashseed, hash, buffer[diskcur], read_size, state->block_size);
}
/* until now is hash */
@ -1475,9 +1475,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* compute the hash of the recovered block */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, block_buffer, size);
memhash_block(state->prevhash, state->prevhashseed, hash, block_buffer, size, state->block_size);
} else {
memhash(state->hash, state->hashseed, hash, block_buffer, size);
memhash_block(state->hash, state->hashseed, hash, block_buffer, size, state->block_size);
}
/* until now is hash */

30
cmdline/util.c
View file

@ -775,6 +775,36 @@ void memhash(unsigned kind, const unsigned char* seed, void* digest, const void*
}
}
void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t logical_size, size_t block_size)
{
assert(logical_size <= block_size);
switch (kind) {
case HASH_MUSEAIR :
/*
* MuseAir defines the identity of the complete canonical RAID
* block. The caller must have zero-padded the buffer after the
* logical end of the file block.
*/
memhash(kind, seed, digest, src, block_size);
break;
case HASH_MURMUR3 :
case HASH_SPOOKY2 :
/*
* Preserve the historical SnapRAID hash semantics for content
* files using legacy hashes.
*/
memhash(kind, seed, digest, src, logical_size);
break;
default :
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Internal inconsistency in hash function %u\n", kind);
exit(EXIT_FAILURE);
break;
/* LCOV_EXCL_STOP */
}
}
const char* hash_config_name(unsigned kind)
{
switch (kind) {

22
cmdline/util.h
View file

@ -226,11 +226,31 @@ unsigned membesthash(void);
const char* memhashname(unsigned kind);
/**
* Compute the HASH of a memory block.
* Compute the HASH of a memory buffer.
* Seed is a 128 bit vector.
*
* This is the raw hash primitive. It hashes exactly size bytes and does not
* apply SnapRAID block semantics.
*/
void memhash(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t size);
/**
* Compute the hash of a SnapRAID data block.
*
* logical_size is the number of bytes logically belonging to the file.
* block_size is the complete zero-padded RAID block size.
*
* Legacy hashes preserve their historical semantics and hash only
* logical_size bytes.
*
* MuseAir hashes the complete canonical RAID block, including the zero
* padding up to block_size.
*
* The caller must guarantee that logical_size <= block_size and, when
* logical_size < block_size, src[logical_size .. block_size-1] == 0.
*/
void memhash_block(unsigned kind, const unsigned char* seed, void* digest, const void* src, size_t logical_size, size_t block_size);
/**
* Return the hash name.
*/