rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implements an incremental rehash.

Browse source
This commit is contained in:
Andrea Mazzoleni 2013-07-01 23:08:25 +02:00
parent 2fdcecf350
commit 275ef2ed3c
17 changed files with 473 additions and 338 deletions
Download patch file Download diff file Expand all files Collapse all files

10
Makefile.am
View file

@ -163,7 +163,7 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf --test-force-scrub 1000 scrub
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf --test-force-scrub 10000 scrub
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf --test-force-scrub 100000 scrub
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf status
echo --- Make a hole in the disk array
mv test/disk2 test/disk2.old
@ -196,9 +196,6 @@ endif
mv test/disk3/7* test/disk3/_a/
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Rehash to spooky2
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-force-spooky2 rehash
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete one disk, fix and check with raid5
rm -r test/disk3
mkdir test/disk3
@ -234,6 +231,11 @@ endif
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check -l test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Rehash to spooky2 and rehash even blocks
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-force-spooky2 rehash
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf --test-force-scrub-even scrub
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf status
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete files from three disks and fix with import in RAID6
rm -r test/disk1/_a
rm -r test/disk1/_b

15
TODO
View file

@ -5,21 +5,6 @@ This is the list of TODO items for SnapRAID.
- v4.0
* In the content file, for each position in the parity store the last time
the hash/parity was checked for all the blocks with sync.
We can share the same time info for all the blocks at a given position.
It will allow to define a new "scrub" command, checking only aged blocks
and skipping recently synched ones.
The "scrub" command won't change data in the disks, but it will update the content
file.
The check command should be maintaned independent, and not saving the content file.
The format could be:
tim BLOCK TIME
tir FROM_BLOCK TO_BLOCK TIME
For missing block, we can use 0 as initialization value. TIME is the UNIX time.
- v4.x
* In the import list, uses also all the blocks in the array.
But we must cover the case of bad blocks. Likely we can just check the
hash after reading, and in case, skip it, and retry with another copy.

42
check.c
View file

@ -60,7 +60,7 @@ struct failed_struct {
* Checks if a block hash matches the specified buffer.
* Return ==0 if equal
*/
static int blockcmp(struct snapraid_state* state, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero)
static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero)
{
unsigned char hash[HASH_SIZE];
unsigned size;
@ -68,7 +68,11 @@ static int blockcmp(struct snapraid_state* state, struct snapraid_block* block,
size = block_file_size(block, state->block_size);
/* now compute the hash of the valid part */
memhash(state->hash, state->hashseed, hash, buffer, size);
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer, size);
} else {
memhash(state->hash, state->hashseed, hash, buffer, size);
}
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
@ -90,7 +94,7 @@ static int blockcmp(struct snapraid_state* state, struct snapraid_block* block,
* Returns <0 if failure for missing strategy, >0 if data is wrong and we cannot rebuild correctly, 0 on success.
* If success, the parity and qarity are computed in the buffer variable.
*/
static int repair_step(struct snapraid_state* state, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
{
unsigned j;
int error = 0;
@ -126,7 +130,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
for(j=0;j<1;++j) {
if (block_has_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
}
}
@ -156,7 +160,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
for(j=0;j<1;++j) {
if (block_has_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
}
}
@ -206,7 +210,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
for(j=0;j<2;++j) {
if (block_has_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
}
}
@ -229,7 +233,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk
return -1;
}
static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
{
int ret;
int error;
@ -263,7 +267,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
if (block_state == BLOCK_STATE_BLK) { /* if we have the hash for it */
/* try to fetch the block using the hash */
if (state_import_fetch(state, failed[j].block->hash, buffer[failed[j].index]) == 0) {
if (state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* we have corrected it! */
} else {
/* otherwise try to recover it */
@ -293,7 +297,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
return 0;
}
ret = repair_step(state, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret == 0) {
/* reprocess the blocks for NEW and CHG ones, for which we don't have a hash to check */
/* if they were BAD we have to use some euristics to ensure that we have recovered */
@ -327,7 +331,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
/* instead, if the hash matches, it could be either that the */
/* block after the sync has this hash, or that */
/* we restored the block before the 'sync'. */
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) {
if (blockcmp(state, rehash, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
}
@ -365,7 +369,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
/* we are not really interseted in DELETED and CHG (old version) ones. */
/* try to fetch the old block using the old hash */
if (state_import_fetch(state, failed[j].block->hash, buffer[failed[j].index]) == 0) {
if (state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
} else {
@ -410,7 +414,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax,
/* if nothing to fix, we just don't try */
if (something_to_recover) {
ret = repair_step(state, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret == 0) {
/* we alreay marked as outdated NEW and CHG blocks, we don't need to do it again */
return 0;
@ -502,6 +506,8 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
unsigned failed_count;
int one_tocheck;
int valid_parity;
snapraid_info info;
int rehash;
/* for each disk */
one_tocheck = 0;
@ -534,6 +540,10 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* keep track of the number of failed blocks */
failed_count = 0;
/* if we have to use the old hash */
info = info_get(&state->infoarr, i);
rehash = info_get_rehash(info);
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
@ -700,7 +710,11 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* if the block has the hash */
if (block_state == BLOCK_STATE_BLK) {
/* compute the hash of the block just read */
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
@ -764,7 +778,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
}
/* try all the recovering strategies */
ret = repair(state, i, diskmax, failed, failed_map, failed_count, buffer, buffer_parity, buffer_qarity, buffer_zero);
ret = repair(state, rehash, i, diskmax, failed, failed_map, failed_count, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret != 0) {
/* increment the number of errors */
if (ret > 0)

32
elem.h
View file

@ -698,12 +698,14 @@ void map_free(struct snapraid_map* map);
/**
* Makes an info.
*/
static inline snapraid_info info_make(time_t last_access, int error)
static inline snapraid_info info_make(time_t last_access, int error, int rehash)
{
/* clear the lowest bits as reserved for other information */
snapraid_info info = last_access & ~0x1;
snapraid_info info = last_access & ~0x3;
if (error != 0)
info |= 1;
info |= 0x1;
if (rehash != 0)
info |= 0x2;
return info;
}
@ -713,26 +715,44 @@ static inline snapraid_info info_make(time_t last_access, int error)
*/
static inline time_t info_get_time(snapraid_info info)
{
return info & ~0x1;
return info & ~0x3;
}
/**
* Extract the error information.
* This report if the block address is reporting to have some problem.
*/
static inline int info_get_error(snapraid_info info)
static inline int info_get_bad(snapraid_info info)
{
return (info & 0x1) != 0;
}
/**
* Extract the error information.
* This report if the block address is reporting to have some problem.
*/
static inline int info_get_rehash(snapraid_info info)
{
return (info & 0x2) != 0;
}
/**
* Mark the block address as with error.
*/
static inline snapraid_info info_set_error(snapraid_info info)
static inline snapraid_info info_set_bad(snapraid_info info)
{
return info | 0x1;
}
/**
* Mark the block address as with rehash.
*/
static inline snapraid_info info_set_rehash(snapraid_info info)
{
return info | 0x2;
}
/**
* Sets the info at the specified position.
* The position is allocated if not yet done.

32
import.c
View file

@ -25,15 +25,23 @@
/**
* Compares the hash of two import blocks.
*/
int import_block_compare(const void* void_arg, const void* void_data)
int import_block_hash_compare(const void* void_arg, const void* void_data)
{
const unsigned char* arg = void_arg;
const struct snapraid_import_block* block = void_data;
return memcmp(arg, block->hash, HASH_SIZE);
}
int import_block_prevhash_compare(const void* void_arg, const void* void_data)
{
const unsigned char* arg = void_arg;
const struct snapraid_import_block* block = void_data;
return memcmp(arg, block->prevhash, HASH_SIZE);
}
/**
* Computes the hash of the hash for an import block.
* We just use the first 32 bit of the hash itself.
*/
static inline tommy_uint32_t import_block_hash(const unsigned char* hash)
{
@ -97,10 +105,17 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t
block->file = file;
block->offset = offset;
block->size = read_size;
memhash(state->hash, state->hashseed, block->hash, buffer, read_size);
memhash(state->hash, state->hashseed, block->hash, buffer, read_size);
tommy_hashdyn_insert(&state->importset, &block->nodeset, block, import_block_hash(block->hash));
/* if we are in a rehash state */
if (state->prevhash != HASH_UNDEFINED) {
/* compute also the previous hash */
memhash(state->prevhash, state->prevhashseed, block->prevhash, buffer, read_size);
tommy_hashdyn_insert(&state->previmportset, &block->prevnodeset, block, import_block_hash(block->prevhash));
}
offset += read_size;
size -= read_size;
}
@ -123,7 +138,7 @@ void import_file_free(struct snapraid_import_file* file)
free(file);
}
int state_import_fetch(struct snapraid_state* state, const unsigned char* hash, unsigned char* buffer)
int state_import_fetch(struct snapraid_state* state, int rehash, const unsigned char* hash, unsigned char* buffer)
{
struct snapraid_import_block* block;
int ret;
@ -133,7 +148,11 @@ int state_import_fetch(struct snapraid_state* state, const unsigned char* hash,
unsigned char buffer_hash[HASH_SIZE];
const char* path;
block = tommy_hashdyn_search(&state->importset, import_block_compare, hash, import_block_hash(hash));
if (rehash) {
block = tommy_hashdyn_search(&state->previmportset, import_block_prevhash_compare, hash, import_block_hash(hash));
} else {
block = tommy_hashdyn_search(&state->importset, import_block_hash_compare, hash, import_block_hash(hash));
}
if (!block)
return -1;
@ -169,7 +188,10 @@ int state_import_fetch(struct snapraid_state* state, const unsigned char* hash,
}
/* recheck the hash */
memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size);
if (rehash)
memhash(state->prevhash, state->prevhashseed, buffer_hash, buffer, read_size);
else
memhash(state->hash, state->hashseed, buffer_hash, buffer, read_size);
if (memcmp(buffer_hash, hash, HASH_SIZE) != 0) {
fprintf(stderr, "Error in data reading file '%s'.\n", path);

4
import.h
View file

@ -33,9 +33,11 @@ struct snapraid_import_block {
unsigned size; /**< Size of the block. */
data_off_t offset; /**< Position of the block in the file. */
unsigned char hash[HASH_SIZE]; /**< Hash of the block. */
unsigned char prevhash[HASH_SIZE]; /**< Previous hash of the block. Valid only if we are in rehash state. */
/* nodes for data structures */
tommy_hashdyn_node nodeset;
tommy_hashdyn_node prevnodeset;
};
/**
@ -61,7 +63,7 @@ void import_file_free(struct snapraid_import_file* file);
* Fetch a block from the specified hash.
* Return ==0 if the block is found, and copied into buffer.
*/
int state_import_fetch(struct snapraid_state* state, const unsigned char* hash, unsigned char* buffer);
int state_import_fetch(struct snapraid_state* state, int prevhash, const unsigned char* hash, unsigned char* buffer);
/**
* Import files from the specified directory.

244
rehash.c
View file

@ -28,211 +28,61 @@
/****************************************************************************/
/* rehash */
static int state_rehash_process(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_handle* handle;
unsigned diskmax;
block_off_t i;
unsigned j;
void* buffer_alloc;
unsigned char* buffer_aligned;
int ret;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
unsigned error;
unsigned char seed[HASH_SIZE];
handle = handle_map(state, &diskmax);
buffer_aligned = malloc_nofail_align(state->block_size, &buffer_alloc);
error = 0;
/* set a random seed */
randomize(seed, HASH_SIZE);
/* check that we are in a sync complete state */
for(i=blockstart;i<blockmax;++i) {
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block;
unsigned block_state;
/* if the disk position is not used */
if (!handle[j].disk) {
/* skip it */
continue;
}
/* if the disk block is not used */
block = disk_block_get(handle[j].disk, i);
if (block == BLOCK_EMPTY) {
/* skip it */
continue;
}
/* get the state of the block */
block_state = block_state_get(block);
/* if the block is deleted */
if (block_state != BLOCK_STATE_BLK) {
fprintf(stderr, "You can rehash only after a complete sync.\n");
++error;
goto bail;
}
}
}
/* rehash all the blocks in files */
countmax = blockmax - blockstart;
countsize = 0;
countpos = 0;
state_progress_begin(state, blockstart, blockmax, countmax);
for(i=blockstart;i<blockmax;++i) {
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
unsigned char hash[HASH_SIZE];
unsigned char hash_new[HASH_SIZE];
struct snapraid_block* block;
/* if the disk position is not used */
if (!handle[j].disk) {
/* skip it */
continue;
}
/* if the disk block is not used */
block = disk_block_get(handle[j].disk, i);
if (block == BLOCK_EMPTY) {
/* skip it */
continue;
}
/* here we can have only BLOCK_STATE_BLK */
/* if the file is closed or different than the current one */
if (handle[j].file == 0 || handle[j].file != block_file_get(block)) {
/* close the old one, if any */
ret = handle_close(&handle[j]);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to rehash.\n");
printf("Stopping at block %u\n", i);
++error;
goto bail;
}
/* open the file only for reading */
ret = handle_open(&handle[j], block_file_get(block), stdlog, state->opt.skip_sequential);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected open error in a data disk, it isn't possible to rehash.\n");
printf("Stopping at block %u\n", i);
++error;
goto bail;
}
}
/* read from the file */
read_size = handle_read(&handle[j], block, buffer_aligned, state->block_size, stdlog);
if (read_size == -1) {
fprintf(stderr, "DANGER! Unexpected read error in a data disk, it isn't possible to rehash.\n");
printf("Stopping at block %u\n", i);
++error;
goto bail;
}
countsize += read_size;
/* compute the new hash */
memhash(state->besthash, seed, hash_new, buffer_aligned, read_size);
/* compute the old hash */
/* note that we intentionally compute the new one before the old */
/* to ensure to detect any random bit flip in the memory */
memhash(state->hash, state->hashseed, hash, buffer_aligned, read_size);
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
fprintf(stderr, "DANGER! Unexpected data error in a data disk, it isn't possible to rehash.\n");
printf("Stopping at block %u\n", i);
++error;
}
/* store the new hash */
memcpy(block->hash, hash_new, HASH_SIZE);
}
/* count the number of processed block */
++countpos;
/* progress */
if (state_progress(state, i, countpos, countmax, countsize)) {
break;
}
}
state_progress_end(state, countpos, countmax, countsize);
/* set the new hash and seed */
state->hash = state->besthash;
memcpy(state->hashseed, seed, HASH_SIZE);
/* mark the state as needing write */
state->need_write = 1;
bail:
/* close all the files left open */
for(j=0;j<diskmax;++j) {
ret = handle_close(&handle[j]);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in a data disk.\n");
++error;
/* continue, as we are already exiting */
}
}
if (error) {
printf("Rehash aborted\n");
} else {
printf("No error\n");
}
free(handle);
free(buffer_alloc);
if (error != 0)
return -1;
return 0;
}
void state_rehash(struct snapraid_state* state)
{
block_off_t blockmax;
int ret;
unsigned error;
if (state->hash == state->besthash) {
fprintf(stderr, "You don't need to rehash because you are already using the best hash.\n");
exit(EXIT_FAILURE);
}
printf("Initializing...\n");
block_off_t i;
blockmax = parity_size(state);
printf("Rehashing...\n");
error = 0;
ret = state_rehash_process(state, 0, blockmax);
if (ret == -1) {
++error;
/* continue, as we are already exiting */
/* check if a rehash is already in progress */
if (state->prevhash != HASH_UNDEFINED) {
fprintf(stderr, "You already have a rehash in progress.\n");
exit(EXIT_FAILURE);
}
/* abort if errors are present */
if (error != 0)
if (state->hash == state->besthash) {
fprintf(stderr, "You are already using the best hash for your platform.\n");
exit(EXIT_FAILURE);
}
/* copy the present hash as previous one */
state->prevhash = state->hash;
memcpy(state->prevhashseed, state->hashseed, HASH_SIZE);
/* set the new hash and seed */
state->hash = state->besthash;
randomize(state->hashseed, HASH_SIZE);
/* mark all the block for rehashing */
for(i=0;i<blockmax;++i) {
snapraid_info info;
/* if it's unused */
info = info_get(&state->infoarr, i);
if (info == 0) {
/* skip it */
continue;
}
if (info_get_rehash(info)) {
fprintf(stderr, "Internal incosistency for a rehash already in progress\n");
exit(EXIT_FAILURE);
}
/* enable the rehash */
info = info_set_rehash(info);
/* save it */
info_set(&state->infoarr, i, info);
}
/* save the new content file */
state->need_write = 1;
printf(
"A rehash is now scheduled. It will take place in the next 'sync' and 'scrub'\n"
"commands in a transparent way. You can check the rehash progress status using\n"
"the 'status' commands.\n");
}

67
scrub.c
View file

@ -27,9 +27,19 @@
/****************************************************************************/
/* scrub */
/**
* Buffer for storing the new hashes.
*/
struct snapraid_rehash {
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
};
static int state_scrub_process(struct snapraid_state* state, struct snapraid_parity* parity, struct snapraid_parity* qarity, block_off_t blockstart, block_off_t blockmax, time_t timelimit, block_off_t countlimit, time_t now)
{
struct snapraid_handle* handle;
void* rehandle_alloc;
struct snapraid_rehash* rehandle;
unsigned diskmax;
block_off_t i;
unsigned j;
@ -49,6 +59,9 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
/* maps the disks to handles */
handle = handle_map(state, &diskmax);
/* rehash buffers */
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
/* we need disk + 2 for each parity level buffers */
buffermax = diskmax + state->level * 2;
@ -80,6 +93,12 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
continue;
}
/* skip odd blocks, used only for testing */
if (state->opt.force_scrub_even && (i % 2) != 0) {
/* skip it */
continue;
}
++countmax;
if (countmax >= countlimit) {
@ -105,6 +124,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
int error_on_this_block;
int silent_error_on_this_block;
int ret;
int rehash;
/* if it's unused */
info = info_get(&state->infoarr, i);
@ -120,6 +140,12 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
continue;
}
/* skip odd blocks, used only for testing */
if (state->opt.force_scrub_even && (i % 2) != 0) {
/* skip it */
continue;
}
/* one more block processed for autosave */
++autosavedone;
--autosavemissing;
@ -128,12 +154,18 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
error_on_this_block = 0;
silent_error_on_this_block = 0;
/* if we have to use the old hash */
rehash = info_get_rehash(info);
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
/* by default no rehash in case of "continue" */
rehandle[j].block = 0;
/* if the disk position is not used */
if (!handle[j].disk) {
/* use an empty block */
@ -181,7 +213,15 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
countsize += read_size;
/* now compute the hash */
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
/* compute the new hash, and store it */
rehandle[j].block = block;
memhash(state->hash, state->hashseed, rehandle[j].hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
if (block_has_hash(block)) {
/* compare the hash */
@ -250,13 +290,21 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
/* do nothing, as this is a generic error */
/* maybe just caused by a not synched array */
} else if (silent_error_on_this_block) {
/* set the error status keeping the existing time */
snapraid_info info = info_get(&state->infoarr, i);
info_set(&state->infoarr, i, info_set_error(info));
/* set the error status keeping the existing time and hash */
info_set(&state->infoarr, i, info_set_bad(info));
} else {
/* if rehash is neeed */
if (rehash) {
/* store all the new hash already computed */
for(j=0;j<diskmax;++j) {
if (rehandle[j].block)
memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
}
}
/* update the time info of the block */
info_set(&state->infoarr, i, info_make(now, 0));
/* and clear any other flag */
info_set(&state->infoarr, i, info_make(now, 0, 0));
}
/* mark the state as needing write */
@ -301,6 +349,7 @@ bail:
free(handle);
free(buffer_alloc);
free(buffer);
free(rehandle_alloc);
if (error != 0)
return -1;
@ -331,7 +380,11 @@ int state_scrub(struct snapraid_state* state)
blockmax = parity_size(state);
if (state->opt.force_scrub) {
if (state->opt.force_scrub_even) {
/* no limit */
countlimit = blockmax;
recentlimit = now;
} else if (state->opt.force_scrub) {
/* scrub the specified amount of blocks */
countlimit = state->opt.force_scrub;
recentlimit = now;

17
snapraid.1
View file

@ -30,7 +30,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG]
.PD 0
.PP
.PD
sync|status|scrub|diff|dup|pool|check|fix
sync|status|scrub|diff|dup|pool|check|fix|rehash
.PD 0
.PP
.PD
@ -429,6 +429,21 @@ the \(a".unrecoverable\(a" extension.
The \(a"content\(a" file is NOT modified.
The \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary.
The files in the array are modified if necessary.
.SS rehash
Schedules a rehash of the whole array.
.PP
This option can be used to change the hash kind used,
typically when upgrading from a 32 bits system to a 64
bits one to switch from the murmur3 to the faster spooky2.
.PP
The rehash isn\(cqt done immediately, but it will take place
incrementally during the \(a"sync\(a" and \(a"scrub\(a" commands.
.PP
You can get the rehash status using the \(a"status\(a" command.
.PP
During the rehash, SnapRAID maintains full functionality,
with the only expection of the \(cqdup\(cq command not able to detect
duplicated files using a different hash.
.SH OPTIONS
SnapRAID provides the following options:
.TP

7
snapraid.c
View file

@ -92,6 +92,7 @@ void usage(void)
#define OPT_TEST_FORCE_ORDER_ALPHA 269
#define OPT_TEST_FORCE_ORDER_DIR 270
#define OPT_TEST_FORCE_SCRUB 271
#define OPT_TEST_FORCE_SCRUB_EVEN 272
#if HAVE_GETOPT_LONG
struct option long_options[] = {
@ -158,6 +159,9 @@ struct option long_options[] = {
/* Force scrub of the specified number of blocks */
{ "test-force-scrub", 1, 0, OPT_TEST_FORCE_SCRUB },
/* Force scrub of all the even blocks. This is really for testing, don't try it. */
{ "test-force-scrub-even", 0, 0, OPT_TEST_FORCE_SCRUB_EVEN },
{ 0, 0, 0, 0 }
};
#endif
@ -358,6 +362,9 @@ int main(int argc, char* argv[])
case OPT_TEST_FORCE_SCRUB :
opt.force_scrub = atoi(optarg);
break;
case OPT_TEST_FORCE_SCRUB_EVEN :
opt.force_scrub_even = 1;
break;
default:
fprintf(stderr, "Unknown option '%c'\n", (char)c);
exit(EXIT_FAILURE);

18
snapraid.d
View file

@ -9,7 +9,7 @@ Synopsis
: [-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid]
: [-s, --start BLKSTART] [-t, --count BLKCOUNT]
: [-v, --verbose] [-l, --log FILE]
: sync|status|scrub|diff|dup|pool|check|fix
: sync|status|scrub|diff|dup|pool|check|fix|rehash
:snapraid [-V, --version] [-h, --help]
@ -288,6 +288,22 @@ Commands
The "parity" and "q-parity" files are modified if necessary.
The files in the array are modified if necessary.
rehash
Schedules a rehash of the whole array.
This option can be used to change the hash kind used,
typically when upgrading from a 32 bits system to a 64
bits one to switch from the murmur3 to the faster spooky2.
The rehash isn't done immediately, but it will take place
incrementally during the "sync" and "scrub" commands.
You can get the rehash status using the "status" command.
During the rehash, SnapRAID maintains full functionality,
with the only expection of the 'dup' command not able to detect
duplicated files using a different hash.
Options
SnapRAID provides the following options:

20
snapraid.txt
View file

@ -13,7 +13,7 @@ snapraid [-c, --conf CONFIG]
[-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid]
[-s, --start BLKSTART] [-t, --count BLKCOUNT]
[-v, --verbose] [-l, --log FILE]
sync|status|scrub|diff|dup|pool|check|fix
sync|status|scrub|diff|dup|pool|check|fix|rehash
snapraid [-V, --version] [-h, --help]
@ -324,6 +324,24 @@ The "content" file is NOT modified.
The "parity" and "q-parity" files are modified if necessary.
The files in the array are modified if necessary.
5.9 rehash
----------
Schedules a rehash of the whole array.
This option can be used to change the hash kind used,
typically when upgrading from a 32 bits system to a 64
bits one to switch from the murmur3 to the faster spooky2.
The rehash isn't done immediately, but it will take place
incrementally during the "sync" and "scrub" commands.
You can get the rehash status using the "status" command.
During the rehash, SnapRAID maintains full functionality,
with the only expection of the 'dup' command not able to detect
duplicated files using a different hash.
6 OPTIONS
=========

216
state.c
View file

@ -21,6 +21,7 @@
#include "import.h"
#include "state.h"
#include "util.h"
#include "parity.h"
#include "cpu.h"
void state_init(struct snapraid_state* state)
@ -45,6 +46,7 @@ void state_init(struct snapraid_state* state)
tommy_list_init(&state->filterlist);
tommy_list_init(&state->importlist);
tommy_hashdyn_init(&state->importset);
tommy_hashdyn_init(&state->previmportset);
tommy_arrayof_init(&state->infoarr, sizeof(snapraid_info));
}
@ -56,6 +58,7 @@ void state_done(struct snapraid_state* state)
tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
tommy_hashdyn_done(&state->importset);
tommy_hashdyn_done(&state->previmportset);
tommy_arrayof_done(&state->infoarr);
}
@ -630,6 +633,11 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
/* by default use a random hash seed */
randomize(state->hashseed, HASH_SIZE);
/* no previous hash by default */
state->prevhash = HASH_UNDEFINED;
/* intentionally not set the prevhashseed */
if (state->opt.gui) {
tommy_node* i;
fprintf(stdlog, "blocksize:%u\n", state->block_size);
@ -889,6 +897,10 @@ void state_read(struct snapraid_state* state)
/* start with a zero seed, it was the default in old versions */
memset(state->hashseed, 0, HASH_SIZE);
/* previous hash, start with an undefined value */
state->prevhash = HASH_UNDEFINED;
memset(state->prevhashseed, 0, HASH_SIZE);
disk = 0;
file = 0;
line = 1;
@ -1002,7 +1014,7 @@ void state_read(struct snapraid_state* state)
/* set a fake info block, in case of upgrading from an old version */
/* the real block info will overwrite this */
info = info_make(st.st_mtime, 0);
info = info_make(st.st_mtime, 0, 0);
/* insert the block in the block array */
info_set(&state->infoarr, v_pos, info);
@ -1013,6 +1025,8 @@ void state_read(struct snapraid_state* state)
/* "inf" command */
block_off_t v_pos;
snapraid_info info;
int rehash;
int bad;
uint32_t t;
hash = oathash8(hash, 'i');
@ -1037,7 +1051,33 @@ void state_read(struct snapraid_state* state)
}
hash = oathash32(hash, t);
info = info_make(t, 0);
/* read extra tags if present */
rehash = 0;
bad = 0;
c = sgetc(f);
while (c == ' ') {
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'inf' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
if (strcmp(buffer, "bad") == 0) {
bad = 1;
hash = oathash8(hash, 'x');
} else if (strcmp(buffer, "rehash") == 0) {
rehash = 1;
hash = oathash8(hash, 'y');
} else {
fprintf(stderr, "Invalid 'inf' specification '%s' in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
}
c = sgetc(f);
}
sungetc(c, f);
info = info_make(t, bad, rehash);
/* insert the block in the block array */
info_set(&state->infoarr, v_pos, info);
@ -1471,38 +1511,6 @@ void state_read(struct snapraid_state* state)
/* stat */
++count_dir;
} else if (strcmp(tag, "bad") == 0) {
/* "bad" command */
block_off_t v_pos;
snapraid_info info;
uint32_t t;
hash = oathash8(hash, 'x');
ret = sgetu32(f, &v_pos);
if (ret < 0) {
fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, v_pos);
c = sgetc(f);
if (c != ' ') {
fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
ret = sgetu32(f, &t);
if (ret < 0) {
fprintf(stderr, "Invalid 'bad' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, t);
info = info_make(t, 1);
/* insert the block in the block array */
info_set(&state->infoarr, v_pos, info);
} else if (strcmp(tag, "checksum") == 0) {
hash = oathash8(hash, 'c');
@ -1512,10 +1520,7 @@ void state_read(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
if (strcmp(buffer, "md5") == 0) {
state->hash = HASH_MD5;
hash = oathash8(hash, 'm');
} else if (strcmp(buffer, "murmur3") == 0) {
if (strcmp(buffer, "murmur3") == 0) {
state->hash = HASH_MURMUR3;
hash = oathash8(hash, 'u');
} else if (strcmp(buffer, "spooky2") == 0) {
@ -1539,6 +1544,39 @@ void state_read(struct snapraid_state* state)
sungetc(c, f);
memset(state->hashseed, 0, HASH_SIZE);
}
} else if (strcmp(tag, "prevchecksum") == 0) {
hash = oathash8(hash, 'C');
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'prevchecksum' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
if (strcmp(buffer, "murmur3") == 0) {
state->prevhash = HASH_MURMUR3;
hash = oathash8(hash, 'u');
} else if (strcmp(buffer, "spooky2") == 0) {
state->prevhash = HASH_SPOOKY2;
hash = oathash8(hash, 'k');
} else {
fprintf(stderr, "Invalid 'prevchecksum' specification '%s' in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
}
c = sgetc(f);
if (c != ' ') {
fprintf(stderr, "Invalid 'prevchecksum' specification '%s' in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
}
/* read the seed */
ret = sgethex(f, state->prevhashseed, HASH_SIZE);
if (ret < 0) {
fprintf(stderr, "Invalid 'seed' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashm(hash, state->prevhashseed, HASH_SIZE);
} else if (strcmp(tag, "blksize") == 0) {
block_off_t blksize;
@ -1661,12 +1699,6 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "deleting all the content and parity files.\n");
exit(EXIT_FAILURE);
}
if (state->hash == HASH_MD5) {
fprintf(stderr, "The MD5 checksum is not supported anymore.\n");
fprintf(stderr, "To use the SnapRAID 2.x branch, you must restart from scratch,\n");
fprintf(stderr, "deleting all the content and parity files.\n");
exit(EXIT_FAILURE);
}
state->loaded_blockmax = blockmax;
@ -1695,7 +1727,7 @@ void state_write(struct snapraid_state* state)
unsigned count_content;
tommy_node* i;
block_off_t b;
block_off_t infomax;
block_off_t blockmax;
unsigned k;
oathash_t hash;
@ -1705,7 +1737,9 @@ void state_write(struct snapraid_state* state)
count_symlink = 0;
count_dir = 0;
hash = 0;
infomax = 0; /* index of the maximum info saved from all the disks */
/* blocks of all array */
blockmax = parity_size(state);
/* count the content files */
count_content = 0;
@ -1748,11 +1782,7 @@ void state_write(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
if (state->hash == HASH_MD5) {
sputsl("checksum md5", f);
hash = oathash8(hash, 'c');
hash = oathash8(hash, 'm');
} else if (state->hash == HASH_MURMUR3) {
if (state->hash == HASH_MURMUR3) {
sputsl("checksum murmur3", f);
hash = oathash8(hash, 'c');
hash = oathash8(hash, 'u');
@ -1773,6 +1803,51 @@ void state_write(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
/* previous hash only if present */
if (state->prevhash != HASH_UNDEFINED) {
/* check if the previous hash is still needed */
for(b=0;b<blockmax;++b) {
snapraid_info info;
info = info_get(&state->infoarr, b);
if (info == 0) {
/* skip it */
continue;
}
if (info_get_rehash(info)) {
/* stop the process */
break;
}
}
/* if at least one rehash tag found, we have to save the previous hash */
if (b < blockmax) {
if (state->prevhash == HASH_MURMUR3) {
sputsl("prevchecksum murmur3", f);
hash = oathash8(hash, 'C');
hash = oathash8(hash, 'u');
} else if (state->hash == HASH_SPOOKY2) {
sputsl("prevchecksum spooky2", f);
hash = oathash8(hash, 'C');
hash = oathash8(hash, 'k');
} else {
fprintf(stderr, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
exit(EXIT_FAILURE);
}
sputc(' ', f);
sputhex(state->prevhashseed, HASH_SIZE, f);
hash = oathashm(hash, state->prevhashseed, HASH_SIZE);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
}
}
}
/* for each map */
for(i=state->maplist;i!=0;i=i->next) {
struct snapraid_map* map = i->data;
@ -1801,8 +1876,8 @@ void state_write(struct snapraid_state* state)
for(i=state->disklist;i!=0;i=i->next) {
tommy_node* j;
struct snapraid_disk* disk = i->data;
block_off_t blockmax;
int first_deleted;
block_off_t blockdiskmax;
/* for each file */
for(j=disk->filelist;j!=0;j=j->next) {
@ -1868,10 +1943,6 @@ void state_write(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
/* keep track of the maximum block */
if (block->parity_pos > infomax)
infomax = block->parity_pos;
sputu32(block->parity_pos, f);
hash = oathash32(hash, block->parity_pos);
@ -1946,16 +2017,14 @@ void state_write(struct snapraid_state* state)
++count_dir;
}
/* maximum block */
blockmax = tommy_array_size(&disk->blockarr);
/* keep track of the maximum block for all disk */
if (blockmax > infomax)
infomax = blockmax;
/* maximum block for this disk */
blockdiskmax = tommy_array_size(&disk->blockarr);
if (blockdiskmax > blockmax)
blockdiskmax = blockmax;
/* for each deleted block in the disk */
first_deleted = 1;
for(b=0;b<blockmax;++b) {
for(b=0;b<blockdiskmax;++b) {
struct snapraid_block* block = tommy_array_get(&disk->blockarr, b);
if (block_state_get(block) == BLOCK_STATE_DELETED) {
if (first_deleted) {
@ -1992,7 +2061,7 @@ void state_write(struct snapraid_state* state)
}
/* for each block */
for(b=0;b<infomax;++b) {
for(b=0;b<blockmax;++b) {
snapraid_info info;
info = info_get(&state->infoarr, b);
@ -2014,13 +2083,8 @@ void state_write(struct snapraid_state* state)
/* save only stuffs different than 0 */
if (info != 0) {
if (info_get_error(info)) {
sputsl("bad ", f);
hash = oathash8(hash, 'x');
} else {
sputsl("inf ", f);
hash = oathash8(hash, 'i');
}
sputsl("inf ", f);
hash = oathash8(hash, 'i');
sputu32(b, f);
hash = oathash32(hash, b);
@ -2030,6 +2094,16 @@ void state_write(struct snapraid_state* state)
sputu32(info, f);
hash = oathash32(hash, info);
if (info_get_bad(info)) {
sputsl(" bad", f);
hash = oathash8(hash, 'x');
}
if (info_get_rehash(info)) {
sputsl(" rehash", f);
hash = oathash8(hash, 'y');
}
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));

4
state.h
View file

@ -55,6 +55,7 @@ struct snapraid_option {
int force_spooky2; /**< Force Spooky2 choice. */
int force_order; /**< Force sorting order. One of the SORT_* defines. */
unsigned force_scrub; /**< Force scrub for the specified number of blocks. */
int force_scrub_even; /**< Force scrub of all the even blocks. */
};
struct snapraid_state {
@ -70,9 +71,11 @@ struct snapraid_state {
char pool[PATH_MAX]; /**< Path of the pool tree. */
uint64_t pool_device; /**< Device identifier of the pool. */
unsigned char hashseed[HASH_SIZE]; /**< Hash seed. Just after a uint64 to provide a minimal alignment. */
unsigned char prevhashseed[HASH_SIZE]; /**< Previous hash seed. In case of rehash. */
char lockfile[PATH_MAX]; /**< Path of the lock file to use. */
unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */
unsigned hash; /**< Hash kind used. */
unsigned prevhash; /**< Previous hash kind used. In case of rehash. */
unsigned besthash; /**< Best hash suggested. */
const char* command; /**< Command running. */
tommy_list contentlist; /**< List of content files. */
@ -81,6 +84,7 @@ struct snapraid_state {
tommy_list filterlist; /**< List of inclusion/exclusion. */
tommy_list importlist; /**< List of import file. */
tommy_hashdyn importset; /**< Hashtable by hash of all the import blocks. */
tommy_hashdyn previmportset; /**< Hashtable by prevhash of all the import blocks. Valid only if we are in a rehash state. */
tommy_arrayof infoarr; /**< Block information array. */
block_off_t loaded_blockmax; /**< Previous size of the parity file, computed from the loaded state. */
time_t progress_start; /**< Start of processing for progress visualization. */

26
status.c
View file

@ -41,8 +41,9 @@ int state_status(struct snapraid_state* state)
block_off_t i;
snapraid_info* infomap;
time_t now;
unsigned error;
unsigned count;
block_off_t bad;
block_off_t rehash;
block_off_t count;
unsigned dayoldest, daymedian, daynewest;
unsigned bar[GRAPH_COLUMN];
unsigned barpos;
@ -117,8 +118,9 @@ int state_status(struct snapraid_state* state)
infomap = malloc_nofail(blockmax * sizeof(snapraid_info));
/* copy the info in the temp vector */
error = 0;
bad = 0;
count = 0;
rehash = 0;
for(i=0;i<blockmax;++i) {
snapraid_info info = info_get(&state->infoarr, i);
@ -126,8 +128,11 @@ int state_status(struct snapraid_state* state)
if (info == 0)
continue;
if (info_get_error(info))
++error;
if (info_get_bad(info))
++bad;
if (info_get_rehash(info))
++rehash;
infomap[count++] = info;
}
@ -138,6 +143,11 @@ int state_status(struct snapraid_state* state)
return 0;
}
if (rehash) {
printf("You have a rehash scheduled at %u%%.\n", rehash * 100 / count);
printf("\n");
}
/* sort it */
qsort(infomap, count, sizeof(snapraid_info), info_time_compare);
@ -211,8 +221,8 @@ int state_status(struct snapraid_state* state)
printf("\n");
if (error) {
printf("DANGER! In the array there are %u failing blocks!\n", error);
if (bad) {
printf("DANGER! In the array there are %u failing blocks!\n", bad);
printf("They are:");
@ -224,7 +234,7 @@ int state_status(struct snapraid_state* state)
if (info == 0)
continue;
if (info_get_error(info))
if (info_get_bad(info))
printf(" %u", i);
}

52
sync.c
View file

@ -27,9 +27,19 @@
/****************************************************************************/
/* sync */
/**
* Buffer for storing the new hashes.
*/
struct snapraid_rehash {
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
};
static int state_sync_process(struct snapraid_state* state, struct snapraid_parity* parity, struct snapraid_parity* qarity, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_handle* handle;
void* rehandle_alloc;
struct snapraid_rehash* rehandle;
unsigned diskmax;
block_off_t i;
unsigned j;
@ -53,6 +63,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* maps the disks to handles */
handle = handle_map(state, &diskmax);
/* rehash buffers */
rehandle = malloc_nofail_align(diskmax * sizeof(struct snapraid_rehash), &rehandle_alloc);
/* we need disk + 1 for each parity level buffers */
buffermax = diskmax + state->level;
@ -109,6 +122,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
int one_valid;
int skip_this_block;
int ret;
snapraid_info info;
int rehash;
/* for each disk */
one_invalid = 0;
@ -155,12 +170,19 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* by default process the block, and skip it if something go wrong */
skip_this_block = 0;
/* if we have to use the old hash */
info = info_get(&state->infoarr, i);
rehash = info_get_rehash(info);
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
/* by default no rehash in case of "continue" */
rehandle[j].block = 0;
/* if the disk position is not used */
if (!handle[j].disk) {
/* use an empty block */
@ -248,7 +270,15 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
countsize += read_size;
/* now compute the hash */
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
/* compute the new hash, and store it */
rehandle[j].block = block;
memhash(state->hash, state->hashseed, rehandle[j].hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
if (block_has_hash(block)) {
/* compare the hash */
@ -260,9 +290,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
goto bail;
}
} else {
/* copy the hash, but doesn't mark the block as hashed */
/* this allow on error to do not save the failed computation */
/* copy the hash in the block, but doesn't mark the block as hashed */
/* this allow in case of skipped block to do not save the failed computation */
memcpy(block->hash, hash, HASH_SIZE);
/* note that in case of rehash, this is the wrong hash, */
/* but it will be overwritten later */
}
}
@ -314,8 +347,18 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
block_state_set(block, BLOCK_STATE_BLK);
}
/* if rehash is neeed */
if (rehash) {
/* store all the new hash already computed */
for(j=0;j<diskmax;++j) {
if (rehandle[j].block)
memcpy(rehandle[j].block->hash, rehandle[j].hash, HASH_SIZE);
}
}
/* update the time info of the block */
info_set(&state->infoarr, i, info_make(now, 0));
/* we are also clearing any previous bad and rehash flag */
info_set(&state->infoarr, i, info_make(now, 0, 0));
}
/* mark the state as needing write */
@ -360,6 +403,7 @@ bail:
free(handle);
free(buffer_alloc);
free(buffer);
free(rehandle_alloc);
if (unrecoverable_error != 0)
return -1;

5
util.h
View file

@ -394,9 +394,8 @@ char* strdup_nofail(const char* str);
* Hash kinds.
*/
#define HASH_UNDEFINED 0
#define HASH_MD5 1
#define HASH_MURMUR3 2
#define HASH_SPOOKY2 3
#define HASH_MURMUR3 1
#define HASH_SPOOKY2 2
/**
* Computes the HASH of a memory block.