rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Merges NEW and CHG blocks and introduces REP blocks.

Browse source 
This is a preparation for using REP blocks for containing hashes
of new data.
This commit is contained in:
Andrea Mazzoleni 2014-02-01 17:57:26 +01:00
parent e14966eef1
commit ba1d6ed2b6
6 changed files with 376 additions and 235 deletions
Download patch file Download diff file Expand all files Collapse all files

209
check.c
View file

@ -34,7 +34,8 @@
*/
struct failed_struct {
/**
* If the block needs to be recovered and rewritten to the disk.
* If we know for sure that the block is garbage or missing
* and it needs to be recovered and rewritten to the disk.
*/
int is_bad;
@ -102,7 +103,11 @@ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned d
/* check if the recovered blocks are OK */
for(j=0;j<failed_count;++j) {
if (block_has_updated_hash(failed[failed_map[j]].block)) { /* if the block can be checked */
/* if we are expected to recover this block */
if (!failed[failed_map[j]].is_outofdate
/* if the block has a hash to check */
&& block_has_updated_hash(failed[failed_map[j]].block)
) {
hash_checked = 1;
if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
@ -173,7 +178,11 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
/* if there isn't, we have to sacrifice a parity block to check that the result is correct */
has_hash = 0;
for(i=0;i<failed_count;++i) {
if (block_has_updated_hash(failed[failed_map[i]].block)) /* if the block can be checked */
/* if we are expected to recover this block */
if (!failed[failed_map[i]].is_outofdate
/* if the block has a hash to check */
&& block_has_updated_hash(failed[failed_map[i]].block)
)
has_hash = 1;
}
@ -274,18 +283,26 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
error = 0;
/* if nothing failed, just recompute the parity */
if (failed_count == 0) {
raid_gen(diskmax, state->level, state->block_size, buffer);
return 0;
}
/* Here we have to try two different strategies to recover, because in case the 'sync' */
/* process is aborted, we don't know if the parity data is really updated just like after 'sync', */
/* or if it still represents the state before the 'sync'. */
/* As first, we assume that the parity is already computed for the current state */
/* Note that if the 'sync' ends normally, we don't have any DELETED, REP and CHG blocks */
/* and the two strategies are identical */
/* As first, we assume that the parity IS updated for the current state */
/* and that we are going to recover the state after the last 'sync'. */
/* This is the normal condition. */
/* In this case, parity contains correct info for BLK, CHG (new version) and NEW blocks, */
/* and not for DELETED ones. */
/* In this case, parity contains info from BLK, REP and CHG blocks, */
/* but not for DELETED. */
/* We need to put in the recovering process only the bad blocks, because all the */
/* others already contains the correct data read frin disk, and the parity is correctly computed for them. */
/* We are interested to recover BLK, CHG and NEW blocks if they are marked as bad, */
/* others already contains the correct data read from disk, and the parity is correctly computed for them. */
/* We are interested to recover BLK, REP and CHG blocks if they are marked as bad, */
/* but we are not interested in DELETED ones. */
n = 0;
@ -296,22 +313,14 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
assert(block_state != BLOCK_STATE_DELETED); /* we cannot have bad DELETED blocks */
if (block_state == BLOCK_STATE_BLK) { /* if we have the hash for it */
/* try to fetch the block using the hash */
if (state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* we have corrected it! */
} else {
/* otherwise try to recover it */
failed_map[n] = j;
++n;
/* we have something to try to recover */
something_to_recover = 1;
}
/* if we have the hash for it */
if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP)
/* try to fetch the block using the known hash */
&& state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0
) {
/* we already have corrected it! */
} else {
assert(block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_NEW);
/* otherwise it's CHG or NEW and we try to recover it */
/* otherwise try to recover it */
failed_map[n] = j;
++n;
@ -330,7 +339,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
if (ret == 0) {
/* reprocess the blocks for NEW and CHG ones, for which we don't have a hash to check */
/* reprocess the CHG blocks, for which we don't have a hash to check */
/* if they were BAD we have to use some euristics to ensure that we have recovered */
/* the state after the sync. If unsure, we assume the worst case */
@ -339,7 +348,22 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
if (failed[j].is_bad) {
unsigned block_state = block_state_get(failed[j].block);
if (block_state == BLOCK_STATE_NEW) {
/* BLK and REP blocks are always OK, because at this point */
/* we have already checked their hash */
if (block_state != BLOCK_STATE_CHG) {
assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP);
continue;
}
/* for CHG blocks we have to 'guess' if they are correct or not */
/* if the hash is invalid we cannot check the result */
/* this could happen if we have lost this information */
/* after an aborted sync */
if (hash_is_invalid(failed[j].block->hash)) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
} else if (hash_is_zero(failed[j].block->hash)) {
/* if the block is not filled with 0, we are sure to have */
/* restored it to the state after the 'sync' */
/* instead, if the block is filled with 0, it could be either that the */
@ -349,14 +373,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
/* it may contain garbage */
failed[j].is_outofdate = 1;
}
} else if (block_state == BLOCK_STATE_CHG) {
/* if the hash is a bogus value we cannot check the result */
/* this could happen if we have lost this information */
/* after an aborted sync */
if (memcmp(failed[j].block->hash, buffer_zero, HASH_SIZE) == 0) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
} else
} else {
/* if the hash is different than the previous one, we are sure to have */
/* restored it to the state after the 'sync' */
/* instead, if the hash matches, it could be either that the */
@ -375,14 +392,13 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
if (ret > 0)
error += ret;
/* Now assume that the parity computation was not updated at the current state, */
/* Now assume that the parity IS NOT updated at the current state, */
/* but still represent the state before the last 'sync' process. */
/* This may happen only if a 'sync' is aborted by a crash or manually. */
/* In this case, parity contains info for BLK, CHG (old version) and DELETED blocks, */
/* but not for CHG (new version) and NEW ones. */
/* In this case, parity contains info from BLK, REP (old version), CHG (old version) and DELETED blocks, */
/* but not for REP (new version) and CHG (new version). */
/* We are interested to recover BLK ones marked as bad, */
/* but we are not interested to recover NEW and CHG (new version) blocks, even if marked as bad, */
/* because we don't have parity for them and it's just impossible, */
/* but we are not interested to recover CHG (new version) and REP (new version) blocks, */
/* even if marked as bad, because we don't have parity for them and it's just impossible, */
/* and we are not interested to recover DELETED ones. */
n = 0;
something_to_recover = 0; /* keep track if there is at least one block to fix */
@ -391,16 +407,30 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
if (block_state == BLOCK_STATE_DELETED
|| block_state == BLOCK_STATE_CHG
|| block_state == BLOCK_STATE_REP
) {
/* If the block is CHG or DELETED, we don't have the original content of block, */
/* If the block is CHG, REP or DELETED, we don't have the original content of block, */
/* and we must try to recover it. */
/* This apply to CHG blocks even if they are not marked bad, */
/* This apply to CHG and REP blocks even if they are not marked bad, */
/* because the parity is computed with old content, and not with the new one. */
/* Note that this recovering is done just to make possible to recover any other BLK one, */
/* we are not really interested in DELETED and CHG (old version) ones. */
/* we are not really interested in DELETED, CHG (old version) and REP (old version). */
if (block_state == BLOCK_STATE_CHG
&& hash_is_zero(failed[j].block->hash)
) {
/* If the block was a ZERO block, restore it to the original 0 as before the 'sync' */
/* We do this to just allow recovering of other BLK ones */
memset(buffer[failed[j].index], 0, state->block_size);
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
/* try to fetch the old block using the old hash for CHG and DELETED blocks */
} else if ((block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_DELETED)
&& hash_is_real(failed[j].block->hash)
&& state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* try to fetch the old block using the old hash */
if (state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
} else {
@ -416,18 +446,6 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
/* note that if the block is not marked as is_bad, */
/* we are not going to write it in the disk */
failed[j].is_outofdate = 1;
} else if (block_state == BLOCK_STATE_NEW) {
/* If the block is NEW, restore it to the original 0 as before the 'sync' */
/* Also in this case, we do this to just allow recovering of other BLK ones */
memset(buffer[failed[j].index], 0, state->block_size);
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
/* mark that we have restored an old state */
/* note that if the block is not marked as is_bad, */
/* we are not going to write it in the disk */
failed[j].is_outofdate = 1;
} else if (failed[j].is_bad) {
/* If the block is bad we don't know its content, and we try to recover it */
/* At this point, we can have only BLK ones */
@ -436,7 +454,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
/* we have something we are interested to recover */
something_to_recover = 1;
/* we try to recover it */
failed_map[n] = j;
++n;
@ -447,7 +465,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
if (something_to_recover) {
ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
if (ret == 0) {
/* we alreay marked as outdated NEW and CHG blocks, we don't need to do it again */
/* we alreay marked as outdated CHG and REP blocks, we don't need to do it again */
return 0;
}
if (ret > 0)
@ -608,6 +626,8 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* mark the parity as invalid, and don't try to check/fix it */
/* because it will be recomputed at the next sync */
valid_parity = 0;
/* follow */
}
/* if the block is DELETED */
@ -714,7 +734,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
read_size = handle_read(&handle[j], block, buffer[j], state->block_size, stdlog);
if (read_size == -1) {
/* save the failed block for the check/fix */
failed[failed_count].is_bad = 1;
failed[failed_count].is_bad = 1; /* it's bad because we cannot read it */
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
@ -728,11 +748,13 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
countsize += read_size;
/* if the block is potentially not updated */
if (block_state == BLOCK_STATE_NEW || block_state == BLOCK_STATE_CHG) {
/* store it in the failed set, because potentially */
/* the parity may be still computed with the previous content */
failed[failed_count].is_bad = 0;
/* always insert CHG blocks, the repair functions needs all of them */
/* because the parity may be still referring at the old state */
/* and the repair must be aware of it */
if (block_state == BLOCK_STATE_CHG) {
/* we DO NOT mark them as bad to avoid to overwrite them with wrong data. */
/* if we don't have a hash, we always assume the first read of the block correct. */
failed[failed_count].is_bad = 0; /* we assume the CHG block correct */
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
@ -741,29 +763,44 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
continue;
}
/* if the block has the hash */
if (block_state == BLOCK_STATE_BLK) {
/* compute the hash of the block just read */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
assert(block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP);
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* save the failed block for the check/fix */
failed[failed_count].is_bad = 1;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
++failed_count;
/* compute the hash of the block just read */
if (rehash) {
memhash(state->prevhash, state->prevhashseed, hash, buffer[j], read_size);
} else {
memhash(state->hash, state->hashseed, hash, buffer[j], read_size);
}
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* save the failed block for the check/fix */
failed[failed_count].is_bad = 1; /* it's bad because the hash doesn't match */
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
++failed_count;
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
/* always insert REP blocks, the repair functions needs all of them */
/* because the parity may be still referring at the old state */
/* and the repair must be aware of it */
if (block_state == BLOCK_STATE_REP) {
failed[failed_count].is_bad = 0; /* it's not bad */
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
++failed_count;
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
}

132
elem.h
View file

@ -77,19 +77,15 @@ struct snapraid_filter {
#define BLOCK_STATE_BLK 1
/**
* The block is new and not yet hashed, and it's using a space previously empty.
* The block is new and not yet hashed.
* This happens when a new block overwrite a just removed block, or an empty space.
*
* The block hash field IS NOT set.
* The parity for this disk is not updated, but it contains 0.
*/
#define BLOCK_STATE_NEW 2
/**
* The block is new and not yet hashed, and it's using the space of a block not existing anymore.
* This happens when a new block overwrite a just removed block.
* The block hash field IS set but it may be also INVALID or ZERO,
* and it represents the hash of the OLD data.
*
* If the block was empty, the hash is set to the special ZERO value.
* If the block was lost, the hash is set to the special INVALID value.
*
* The block hash field IS set, and it represents the hash of the previous data,
* but only if it's different by all 0.
* The parity for this disk is not updated, but it contains the old data referenced by the hash.
*
* If the state is read from an incomplete sync, we don't really know if the hash is referring at the
@ -101,7 +97,16 @@ struct snapraid_filter {
* Check and fix are instead able to work with unsynched hashes.
* Scrub is not affected because it ignores CHG/DELETED blocks.
*/
#define BLOCK_STATE_CHG 3
#define BLOCK_STATE_CHG 2
/**
* The block is new and hashed.
* This happens when a new block overwrite a just removed block, or an empty space.
*
* The block hash field IS set, and it represents the hash of the NEW data.
* The parity for this disk is not updated, but it contains the old data referenced by the hash.
*/
#define BLOCK_STATE_REP 3
/**
* This block is a deleted one.
@ -462,42 +467,83 @@ static inline void block_state_set(struct snapraid_block* block, unsigned state)
}
/**
* Checks if the specified block is valid and has an updated hash.
* Checks if the specified block has an updated hash.
*
* Note that EMPTY / CHG / NEW / DELETED return 0.
* Note that EMPTY / CHG / DELETED return 0.
*/
static inline int block_has_updated_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK;
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP;
}
/**
* Checks if the specified block has any kind of hash.
* Even a not updated one.
* Checks if the specified block has an a past hash.
*
* Note that for DELETED / CHG with a NULL hash returns 0,
* because in these cases the hash is lost.
* Note that EMPTY / BLK / REP return 0.
*/
static inline int block_has_any_hash(const struct snapraid_block* block)
static inline int block_has_past_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED;
}
/**
* Checks if the specified hash is invalid.
*
* An invalid hash is represented with all bytes at 0x00.
*/
static inline int hash_is_invalid(const unsigned char* hash)
{
unsigned block_state = block_state_get(block);
unsigned i;
switch (block_state) {
case BLOCK_STATE_BLK :
return 1;
case BLOCK_STATE_CHG :
case BLOCK_STATE_DELETED :
/* the hash is valid only if different than 0 */
for(i=0;i<HASH_SIZE;++i)
if (block->hash[i] != 0)
return 1;
break;
}
for(i=0;i<HASH_SIZE;++i)
if (hash[i] != 0x00)
return 0;
return 0;
return 1;
}
/**
* Checks if the specified hash represent the zero block.
*
* A zero hash is represented with all bytes at 0xFF.
*/
static inline int hash_is_zero(const unsigned char* hash)
{
unsigned i;
for(i=0;i<HASH_SIZE;++i)
if (hash[i] != 0xFF)
return 0;
return 1;
}
/**
* Checks if the specified hash is a real hash.
*/
static inline int hash_is_real(const unsigned char* hash)
{
return !hash_is_zero(hash) && !hash_is_invalid(hash);
}
/**
* Set the hash to the special INVALID value.
*/
static inline void hash_invalid_set(unsigned char* hash)
{
memset(hash, 0x00, HASH_SIZE);
}
/**
* Set the hash to the special ZERO value.
*/
static inline void hash_zero_set(unsigned char* hash)
{
memset(hash, 0xFF, HASH_SIZE);
}
/**
@ -509,21 +555,8 @@ static inline int block_has_file(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG;
}
/**
* Checks if the specified block has the same presence state,
* meaning that it's now present when before it was also present,
* or it's now absent and before it was also absent.
*
* Note that NEW / DELETED return 0.
*/
static inline int block_has_same_presence(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_EMPTY || state == BLOCK_STATE_BLK || state == BLOCK_STATE_CHG;
return state == BLOCK_STATE_BLK
|| state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
}
/**
@ -535,7 +568,8 @@ static inline int block_has_invalid_parity(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_DELETED || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG;
return state == BLOCK_STATE_DELETED
|| state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP;
}
/**

55
scan.c
View file

@ -170,24 +170,27 @@ static void scan_file_deallocate(struct snapraid_state* state, struct snapraid_d
case BLOCK_STATE_BLK :
/* we keep the hash making it an "old" hash, because the parity is still containing data for it */
break;
case BLOCK_STATE_NEW :
/* NEW blocks don't have a hash, then we set it to the 'unspecified' 0 value */
memset(block->hash, 0, HASH_SIZE);
break;
case BLOCK_STATE_CHG :
/* if we have not already cleared the undeterminated hash */
if (!state->clear_undeterminate_hash) {
/* in these cases we don't know if the old state is still the one */
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* Note that this condition is possible only if: */
/* - new files added/modified */
/* - aborted sync, without saving the content file */
/* - files deleted after the aborted sync */
memset(block->hash, 0, HASH_SIZE);
/* may be or may be not have been updated with the data that it's now */
/* deleted. Then we reset the hash to a bogus value. */
/* For example: */
/* - One file is added */
/* - Sync aborted after updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - File is now deleted after the aborted sync */
/* - Sync again, deleting the blocks (exactly here) */
/* with the hash of CHG block not represeting the real parity state */
hash_invalid_set(block->hash);
}
break;
case BLOCK_STATE_REP :
/* we just don't know the old hash, and then we set it to invalid */
hash_invalid_set(block->hash);
break;
default:
fprintf(stderr, "Internal state inconsistency in scanning for block %u state %u\n", block->parity_pos, block_state);
exit(EXIT_FAILURE);
@ -205,7 +208,7 @@ static void scan_file_deallocate(struct snapraid_state* state, struct snapraid_d
}
/**
* Checks if a file is completely formed of NEW or CHG blocks,
* Checks if a file is completely formed of NEW/CHG/REP blocks,
* and if it has at least one block.
*/
static int file_is_full_invalid(struct snapraid_file* file)
@ -226,7 +229,7 @@ static int file_is_full_invalid(struct snapraid_file* file)
block_state = block_state_get(block);
if (block_state != BLOCK_STATE_NEW && block_state != BLOCK_STATE_CHG)
if (block_state != BLOCK_STATE_CHG && block_state != BLOCK_STATE_REP)
return 0;
}
@ -290,6 +293,7 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
block_max = tommy_arrayblk_size(&disk->blockarr);
for(i=0;i<file->blockmax;++i) {
struct snapraid_block* block;
unsigned block_state;
/* find a free block */
while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos)))
@ -307,12 +311,19 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
/* block to overwrite */
block = tommy_arrayblk_get(&disk->blockarr, block_pos);
/* state of the block */
block_state = block_state_get(block);
/* if the block is an empty one */
if (block == BLOCK_EMPTY) {
/* we just overwrite it with a NEW one */
block_state_set(&file->blockvec[i], BLOCK_STATE_NEW);
if (block_state == BLOCK_STATE_EMPTY) {
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
/* the block was empty and filled with zeros */
/* set the hash to the special ZERO value */
hash_zero_set(file->blockvec[i].hash);
} else {
/* otherwise it's a DELETED one */
assert(block_state == BLOCK_STATE_DELETED);
/* if we have not already cleared the undeterminated hash */
if (!state->clear_undeterminate_hash) {
@ -320,11 +331,13 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
/* stored inside the parity, because after an aborted sync, the parity */
/* may be or may be not have been updated with the new data */
/* Then we reset the hash to a bogus value */
/* Note that this condition is possible only if: */
/* - files are deleted */
/* - aborted sync, without saving the content file */
/* - files are readded after the aborted sync */
memset(block->hash, 0, HASH_SIZE);
/* For example: */
/* - One file is deleted */
/* - Sync aborted after, updating the parity to the new state, */
/* but without saving the content file representing this new state. */
/* - Another file is added again (exactly here) */
/* with the hash of DELETED block not represeting the real parity state */
hash_invalid_set(block->hash);
}
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);

11
scrub.c
View file

@ -182,6 +182,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
silent_error_on_this_block = 0;
/* if all the blocks at this address are synched */
/* if not, parity is not even checked */
block_is_unsynched = 0;
/* if we have to use the old hash */
@ -195,6 +196,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
int file_is_unsynched;
/* if the file on this disk is synched */
/* if not, silent errors are assumed as expected error */
file_is_unsynched = 0;
/* by default no rehash in case of "continue" */
@ -215,6 +217,15 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
continue;
}
/* if the block is unsynched, errors are expected */
if (block_has_invalid_parity(block)) {
/* report that the block and the file are not synched */
block_is_unsynched = 1;
file_is_unsynched = 1;
/* follow */
}
/* if the file is different than the current one, close it */
if (handle[j].file != 0 && handle[j].file != block_file_get(block)) {
/* keep a pointer at the file we are going to close for error reporting */

87
state.c
View file

@ -1088,8 +1088,10 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
sungetc(c, f);
}
if (strcmp(tag, "blk") == 0 || strcmp(tag, "new") == 0 || strcmp(tag, "chg") == 0) {
/* "blk"/""new"/"chg" command */
if (strcmp(tag, "blk") == 0 || strcmp(tag, "new") == 0
|| strcmp(tag, "chg") == 0 || strcmp(tag, "rep") == 0
) {
/* "blk"/""new"/"chg"/"rep" command */
block_off_t v_pos;
struct snapraid_block* block;
@ -1126,11 +1128,15 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
paritymax = v_pos + 1;
break;
case 'n' :
block_state_set(block, BLOCK_STATE_NEW);
/* deprecated NEW blocks are converted to CHG ones */
block_state_set(block, BLOCK_STATE_CHG);
break;
case 'c' :
block_state_set(block, BLOCK_STATE_CHG);
break;
case 'p' :
block_state_set(block, BLOCK_STATE_REP);
break;
}
block->parity_pos = v_pos;
@ -1139,7 +1145,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
if (v_pos + 1 > blockmax)
blockmax = v_pos + 1;
/* read the hash only for 'blk/chg', and not for 'new' */
/* read the hash only for 'blk/chg/rep', and not for 'new' */
if (tag[0] != 'n') {
c = sgetc(f);
if (c != ' ') {
@ -1147,17 +1153,23 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
exit(EXIT_FAILURE);
}
/* set the hash only if present */
ret = sgethex(f, block->hash, HASH_SIZE);
if (ret < 0) {
fprintf(stderr, "Invalid '%s' specification in '%s' at line %u\n", tag, path, line);
exit(EXIT_FAILURE);
}
} else {
/* set the ZERO hash for deprecated NEW blocks */
hash_zero_set(block->hash);
}
/* clear undeterminated hashes before a sync */
if (tag[0] != 'b' && state->clear_undeterminate_hash) {
memset(block->hash, 0, HASH_SIZE);
}
/* if the block contains a hash of past data */
/* and we are clearing such undeterminated hashes */
if (state->clear_undeterminate_hash
&& block_has_past_hash(block)
) {
/* set the hash value to INVALID */
hash_invalid_set(block->hash);
}
/* we must not overwrite existing blocks */
@ -1281,9 +1293,10 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE
exit(EXIT_FAILURE);
}
/* clear undeterminated hashes before a sync */
/* if we are clearing undeterminated hashes */
if (state->clear_undeterminate_hash) {
memset(deleted->block.hash, 0, HASH_SIZE);
/* set the hash value to INVALID */
hash_invalid_set(deleted->block.hash);
}
/* we must not overwrite existing blocks */
@ -1997,12 +2010,12 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
case BLOCK_STATE_BLK :
sputsl("blk ", f);
break;
case BLOCK_STATE_NEW :
sputsl("new ", f);
break;
case BLOCK_STATE_CHG :
sputsl("chg ", f);
break;
case BLOCK_STATE_REP :
sputsl("rep ", f);
break;
default:
fprintf(stderr, "Internal state inconsistency in saving for block %u state %u\n", block->parity_pos, block_state);
exit(EXIT_FAILURE);
@ -2010,10 +2023,8 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
sputu32(block->parity_pos, f);
if (block_state != BLOCK_STATE_NEW) {
sputc(' ', f);
sputhex(block->hash, HASH_SIZE, f);
}
sputc(' ', f);
sputhex(block->hash, HASH_SIZE, f);
sputeol(f);
if (serror(f)) {
@ -2383,11 +2394,15 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
paritymax = v_pos + 1;
break;
case 'n' :
block_state_set(block, BLOCK_STATE_NEW);
/* deprecated NEW blocks are converted to CHG ones */
block_state_set(block, BLOCK_STATE_CHG);
break;
case 'g' :
block_state_set(block, BLOCK_STATE_CHG);
break;
case 'p' :
block_state_set(block, BLOCK_STATE_REP);
break;
default:
decoding_error(path, f);
fprintf(stderr, "Invalid block type!\n");
@ -2396,18 +2411,25 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
block->parity_pos = v_pos;
/* read the hash only for 'blk/chg', and not for 'new' */
/* read the hash only for 'blk/chg/rep', and not for 'new' */
if (c != 'n') {
/* set the hash only if present */
ret = sread(f, block->hash, HASH_SIZE);
if (ret < 0) {
decoding_error(path, f);
exit(EXIT_FAILURE);
}
} else {
/* set the ZERO hash for deprecated NEW blocks */
hash_zero_set(block->hash);
}
/* clear undeterminated hashes before a sync */
if (c != 'b' && state->clear_undeterminate_hash)
memset(block->hash, 0, HASH_SIZE);
/* if the block contains a hash of past data */
/* and we are clearing such undeterminated hashes */
if (state->clear_undeterminate_hash
&& block_has_past_hash(block)
) {
/* set the hash value to INVALID */
hash_invalid_set(block->hash);
}
/* we must not overwrite existing blocks */
@ -2573,9 +2595,10 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
exit(EXIT_FAILURE);
}
/* clear undeterminated hashes before a sync */
/* if we are clearing undeterminated hashes */
if (state->clear_undeterminate_hash) {
memset(deleted->block.hash, 0, HASH_SIZE);
/* set the hash value to INVALID */
hash_invalid_set(deleted->block.hash);
}
/* we must not overwrite existing blocks */
@ -3092,12 +3115,12 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
case BLOCK_STATE_BLK :
sputc('b', f);
break;
case BLOCK_STATE_NEW :
sputc('n', f);
break;
case BLOCK_STATE_CHG :
sputc('g', f);
break;
case BLOCK_STATE_REP :
sputc('p', f);
break;
default:
fprintf(stderr, "Internal state inconsistency in saving for block %u state %u\n", v_pos, block_state);
exit(EXIT_FAILURE);
@ -3108,10 +3131,8 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
v_count = end - begin;
sputb32(v_count, f);
if (block_state != BLOCK_STATE_NEW) {
for(b=begin;b<end;++b) {
swrite(blockvec[b].hash, HASH_SIZE, f);
}
for(b=begin;b<end;++b) {
swrite(blockvec[b].hash, HASH_SIZE, f);
}
if (serror(f)) {

117
sync.c
View file

@ -209,6 +209,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
unsigned block_state;
/* by default no rehash in case of "continue" */
rehandle[j].block = 0;
@ -223,27 +224,31 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* get the block */
block = disk_block_get(handle[j].disk, i);
/* if the block is NEW or DELETED, we have to update the parity */
/* to include this block change */
if (!block_has_same_presence(block)) {
parity_needs_to_be_updated = 1;
/* get the state of the block */
block_state = block_state_get(block);
/* it's important to check this before any other check */
/* because for DELETED block, we skip at the next check */
/* now continue with processing */
}
/* if the block is NEW, DELETED or CHG, */
/* if the block has invalid parity, */
/* we have to take care of it in case of recover */
if (block_has_invalid_parity(block)) {
/* store it in the failed set, because */
/* the parity is still computed with the previous content */
/* the parity may be still computed with the previous content */
failed[failed_count].index = j;
failed[failed_count].size = state->block_size;
failed[failed_count].block = block;
++failed_count;
/* now continue with processing */
/* if the block has invalid parity, we have to update the parity */
/* to include this block change */
/* This also apply to CHG blocks, but we are going to handle */
/* later this case to do the updates only if really needed */
if (block_state != BLOCK_STATE_CHG)
parity_needs_to_be_updated = 1;
/* note that DELETE blocks are skipped in the next check */
/* and we have to store them in the failed blocks */
/* before skipping */
/* follow */
}
/* if the block has no file, meanining that it's EMPTY or DELETED, */
@ -361,34 +366,48 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (block_has_updated_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, handle[j].file->sub, block_file_pos(block));
fprintf(stderr, "Data error at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
fprintf(stderr, "WARNING! Unexpected data error in a data disk! The block is now marked as bad!\n");
fprintf(stderr, "Try with 'snapraid -e fix' to recover!\n");
/* if the file has invalid parity, it's a REP changed during the sync */
if (block_has_invalid_parity(block)) {
fprintf(stdlog, "error:%u:%s:%s: Unexpected change\n", i, handle[j].disk->name, handle[j].file->sub);
fprintf(stderr, "WARNING! You cannot modify files during a sync.\n");
fprintf(stderr, "Rerun the sync command when finished.\n");
/* save the failed block for the fix */
failed[failed_count].index = j;
failed[failed_count].size = read_size;
failed[failed_count].block = block;
++failed_count;
++error;
/* silent errors are very rare, and are not a signal that a disk */
/* is going to fail. So, we just continue marking the block as bad */
/* just like in scrub */
++silent_error;
silent_error_on_this_block = 1;
continue;
/* if the file is changed, it means that it was modified during sync */
/* this isn't a serious error, so we skip this block, and continue with others */
error_on_this_block = 1;
continue;
} else { /* otherwise it's a BLK with silent error */
fprintf(stdlog, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, handle[j].file->sub, block_file_pos(block));
fprintf(stderr, "Data error at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
fprintf(stderr, "WARNING! Unexpected data error in a data disk! The block is now marked as bad!\n");
fprintf(stderr, "Try with 'snapraid -e fix' to recover!\n");
/* save the failed block for the fix */
failed[failed_count].index = j;
failed[failed_count].size = read_size;
failed[failed_count].block = block;
++failed_count;
/* silent errors are very rare, and are not a signal that a disk */
/* is going to fail. So, we just continue marking the block as bad */
/* just like in scrub */
++silent_error;
silent_error_on_this_block = 1;
continue;
}
}
} else {
/* if until now the parity doesn't need to be update */
/* if until now the parity doesn't need to be updated */
if (!parity_needs_to_be_updated) {
/* for sure it's a CHG block, because EMPTY are processed before with "continue" */
/* and BLK have "block_has_updated_hash()" as 1, and all the others */
/* and BLK and REP have "block_has_updated_hash()" as 1, and all the others */
/* have "parity_needs_to_be_updated" already at 1 */
assert(block_state_get(block) == BLOCK_STATE_CHG);
/* if there is an hash */
if (block_has_any_hash(block)) {
if (hash_is_real(block->hash)) {
/* check if the hash is changed */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* the block is different, and we must update parity */
@ -412,6 +431,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* if we have only silent errors we can try to fix them on-the-fly */
if (!error_on_this_block && silent_error_on_this_block) {
unsigned failed_mac;
int something_to_recover = 0;
/* setup the blocks to recover */
failed_mac = 0;
@ -420,29 +440,35 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
unsigned char* block_copy = buffer[diskmax + state->level + failed[j].index];
unsigned block_state = block_state_get(failed[j].block);
/* make a copy of new content just read */
/* we try to recover only if at least one BLK is present */
if (block_state == BLOCK_STATE_BLK)
something_to_recover = 1;
/* save a copy of the content just read */
/* that it's going to be overwritten by the recovering function */
memcpy(block_copy, block_buffer, state->block_size);
if (block_state == BLOCK_STATE_NEW) {
/* restore the data to parity state */
if (block_state == BLOCK_STATE_CHG
&& hash_is_zero(failed[j].block->hash)
) {
/* if the block was filled with 0, restore this state */
/* and avoid to recover it */
memset(block_buffer, 0, state->block_size);
} else {
/* if we have too many failure */
if (failed_mac >= state->level) {
/* we cannot recover */
/* if we have too many failures, we cannot recover */
if (failed_mac >= state->level)
break;
}
/* otherwise it has to be recovered */
failed_map[failed_mac++] = failed[j].index;
}
}
/* if we have enough parity to fix */
if (j == failed_count) {
/* if we have something to recover and enough parity */
if (something_to_recover && j == failed_count) {
/* read the parity */
/* we are sure that parity exists because a silent error */
/* implies a BLK block with compute parity */
/* we are sure that parity exists because */
/* we have at least one BLK block */
for(l=0;l<state->level;++l) {
ret = parity_read(parity[l], i, buffer[diskmax+l], state->block_size, stdlog);
if (ret == -1) {
@ -485,10 +511,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
if (size < state->block_size)
memset(block_buffer + size, 0, state->block_size - size);
} else {
/* otherwise restore the new content */
/* because we are not interested in recovered */
/* state for CHG and DELETED blocks */
/* and we have to restore also NEW zeroed blocks */
/* otherwise restore the content */
/* because we are not interested in the old state */
/* that it's recovered for CHG, REP and DELETED blocks */
memcpy(block_buffer, block_copy, state->block_size);
}
}