rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improves recovering after an aborted sync.

Browse source
This commit is contained in:
Andrea Mazzoleni 2012-02-09 23:17:36 +01:00
parent f14ab1aa3b
commit 55fae37e7f
13 changed files with 758 additions and 137 deletions
Download patch file Download diff file Expand all files Collapse all files

4
HISTORY
View file

@ -3,6 +3,10 @@ SnapRAID HISTORY
1.9 2012/2
===========
* Implemented a more sophisticated recovering in case a harddisk failure
happens during a 'sync' command.
When using RAID6 it improves the chances of recovering data after an
aborted 'sync'.
* Fixed the count of new files.
1.8 2012/1

78
Makefile.am
View file

@ -139,13 +139,13 @@ check-local:
rm test/disk2/2*
rm test/disk3/3*
$(TESTENV) ./mktest$(EXEEXT) generate 3 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Move some files, sync and check
mv test/disk1/7* test/disk1/_a/
mv test/disk2/7* test/disk2/_a/
mv test/disk3/7* test/disk3/_a/
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete one disk, fix and check with raid5
rm -r test/disk3
@ -154,7 +154,7 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Sync after all the fixes with an empty drive to test --force-empty
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v --force-empty -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --force-empty -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete two disks, fix and check with raid6
rm -r test/disk1
@ -167,31 +167,85 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Sync after all the fixes with an empty drive to test --find-by-name
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v --find-by-name -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --find-by-name -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Delete some files and create some new, sync and check in multiple steps
rm test/disk1/4*
rm test/disk2/5*
rm test/disk3/6*
$(TESTENV) ./mktest$(EXEEXT) generate 4 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf -t 100 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 100 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf -t 700 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 700 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf -t 1000 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1000 sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Add some files, sync with abort, delete the new files, sync and check
echo --- Abort sync with additions and then delete the new additions
cp -a test/disk1/_a test/disk1/_c
cp -a test/disk2/_a test/disk2/_c
cp -a test/disk3/_a test/disk3/_c
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-kill-after-sync -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-kill-after-sync -c $(srcdir)/test-raid6.conf sync
rm -r test/disk1/_c
rm -r test/disk2/_c
rm -r test/disk3/_c
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Abort sync late with more additions than deletion and then delete after
cp -a test/disk1/_a test/disk1/_c
cp -a test/disk1/_b test/disk1/_d
mv test/disk1/_a test/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
rm -r test/disk1/_c
rm -r test/disk1/_d
mv test/_a test/disk1/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Abort sync late with more deletions than additions and then delete after
cp -a test/disk1/_a test/disk1/_c
mv test/disk1/_a test/_a
mv test/disk1/_b test/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
-rm -r test/disk1/_b
rm -r test/disk1/_c
mv test/_a test/disk1/_a
mv test/_b test/disk1/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Abort sync early with more additions than deletions and then delete after
cp -a test/disk1/_a test/disk1/_c
cp -a test/disk1/_b test/disk1/_d
mv test/disk1/_a test/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
rm -r test/disk1/_c
rm -r test/disk1/_d
mv test/_a test/disk1/_a
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Abort sync early with more deletions than additions and then delete after
cp -a test/disk1/_a test/disk1/_c
mv test/disk1/_a test/_a
mv test/disk1/_b test/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
-rm -r test/disk1/_a
-rm -r test/disk1/_b
rm -r test/disk1/_c
mv test/_a test/disk1/_a
mv test/_b test/disk1/_b
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
echo --- Delete some files, fix and check with raid5
rm test/disk1/8*
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log
@ -301,7 +355,7 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Sync after all the fixes
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo Success!

380
check.c
View file

@ -27,10 +27,15 @@
/****************************************************************************/
/* check */
/**
* A block that failed the hash check, or that was deleted.
*/
struct failed_struct {
unsigned index;
struct snapraid_block* block;
struct snapraid_handle* handle;
int is_bad; /**< If the block needs to be recovered and rewritten to the disk. */
int is_outofdate; /**< If that we have recovered may be not updated data, but an old version. */
unsigned index; /**< Index of the failed block. */
struct snapraid_block* block; /**< The failed block, or BLOCK_DELETED for a deleted block */
struct snapraid_handle* handle; /**< The file containing the failed block, or 0 for a deleted block */
};
/**
@ -66,10 +71,12 @@ static int blockcmp(struct snapraid_state* state, struct snapraid_block* block,
* Returns <0 if failure for missing strategy, >0 if data is wrong and we cannot rebuild correctly, 0 on success.
* If success, the parity and qarity are computed in the buffer variable.
*/
static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, struct failed_struct* failed, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
static int repair_step(struct snapraid_state* state, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
{
unsigned j;
int error = 0;
int has_hash;
int hash_checked;
/* no fix required */
if (failed_count == 0) {
@ -78,76 +85,267 @@ static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, st
return 0;
}
if (failed_count == 1 && buffer_parity != 0) {
/* check if there is at least a failed block that can be checked for correctness */
has_hash = 0;
for(j=0;j<failed_count;++j) {
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block)) /* if the block can be checked */
has_hash = 1;
}
/* RAID5 recovering from p */
if (failed_count == 1 && buffer_parity != 0 && has_hash) {
/* copy the redundancy to use */
memcpy(buffer[diskmax], buffer_parity, state->block_size);
/* recover */
raid5_recov_data(buffer, diskmax, state->block_size, failed[0].index);
raid5_recov_data(buffer, diskmax, state->block_size, failed[failed_map[0]].index);
hash_checked = 0; /* keep track if we check at leat one block */
/* check if the recovered blocks are OK */
for(j=0;j<1;++j) {
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block) /* if the block can be checked */
) {
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
}
}
if (j==1) {
/* if we checked something, and no block failed the check */
if (hash_checked && j==1) {
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
fprintf(stderr, "error:%u:parity: Data error\n", i);
fprintf(stderr, "error:%u:parity: Data error\n", pos);
++error;
}
if (failed_count == 1 && buffer_qarity != 0) {
/* RAID6 recovering from q, ignoring p */
if (failed_count == 1 && buffer_qarity != 0 && has_hash) {
/* copy the redundancy to use */
memcpy(buffer[diskmax+1], buffer_qarity, state->block_size);
raid6_recov_datap(buffer, diskmax, state->block_size, failed[0].index, buffer_zero);
raid6_recov_datap(buffer, diskmax, state->block_size, failed[failed_map[0]].index, buffer_zero);
hash_checked = 0; /* keep track if we check at leat one block */
/* check if the recovered blocks are OK */
for(j=0;j<1;++j) {
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block) /* if the block can be checked */
) {
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
}
}
if (j==1) {
/* if we checked something, and no block failed the check */
if (hash_checked && j==1) {
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
fprintf(stderr, "error:%u:qarity: Data error\n", i);
fprintf(stderr, "error:%u:qarity: Data error\n", pos);
++error;
}
if (failed_count == 2 && buffer_parity != 0 && buffer_qarity != 0) {
/* RAID6 recovering from q, using the restored p to check the result */
if (failed_count == 1 && buffer_parity != 0 && buffer_qarity != 0 && !has_hash) {
/* copy the redundancy to use */
memcpy(buffer[diskmax+1], buffer_qarity, state->block_size);
raid6_recov_datap(buffer, diskmax, state->block_size, failed[failed_map[0]].index, buffer_zero);
/* if the recovered parity block matches */
if (memcmp(buffer[diskmax], buffer_parity, state->block_size) == 0) {
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
fprintf(stderr, "error:%u:parity/qarity: Data error\n", pos);
++error;
}
/* RAID6 recovering from p+q */
if (failed_count == 2 && buffer_parity != 0 && buffer_qarity != 0 && has_hash) {
/* copy the redundancy to use */
memcpy(buffer[diskmax], buffer_parity, state->block_size);
memcpy(buffer[diskmax+1], buffer_qarity, state->block_size);
/* recover */
raid6_recov_2data(buffer, diskmax, state->block_size, failed[0].index, failed[1].index, buffer_zero);
raid6_recov_2data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, buffer_zero);
hash_checked = 0; /* keep track if we check at leat one block */
/* check if the recovered blocks are OK */
for(j=0;j<2;++j) {
if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) != 0)
break;
if (failed[failed_map[j]].block != BLOCK_DELETED /* if the block it not a deleted one */
&& block_has_hash(failed[failed_map[j]].block) /* if the block can be checked */
) {
hash_checked = 1;
if (blockcmp(state, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0)
break;
}
}
if (j==2) {
/* if we checked something, and no block failed the check */
if (hash_checked && j==2) {
/* recompute all the redundancy information */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
fprintf(stderr, "error:%u:parity/qarity: Data error\n", i);
fprintf(stderr, "error:%u:parity/qarity: Data error\n", pos);
++error;
}
/* no more stragety to fix */
/* return the number of failed attempts, or -1 if no strategy */
if (error)
return error;
else
return -1;
}
static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity, unsigned char* buffer_zero)
{
int ret;
int error;
unsigned j;
int n;
int has_bad;
unsigned map[2];
error = 0;
/* now assume that the parity is already computed for the current state */
/* we need to put in the recovering process only the bad blocks, because all the */
/* others already contains the correct data. */
/* this is the normal condition. */
n = 0;
has_bad = 0; /* keep track if there is at least one block to fix */
for(j=0;j<failed_count;++j) {
if (failed[j].is_bad) {
has_bad = 1;
/* we never set a DELETED block as bad */
assert(failed[j].block != BLOCK_DELETED);
if (n<2)
map[n] = j;
++n;
}
}
/* if nothing to fix */
if (!has_bad) {
/* recompute only parity and qarity */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
ret = repair_step(state, pos, diskmax, failed, map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret == 0) {
/* reprocess the blocks for NEW and CHG ones */
/* if they were BAD we have to use some euristics to check if we have recovered */
/* the state before or after the sync */
for(j=0;j<failed_count;++j) {
/* we take care only of BAD blocks we have to write back */
if (failed[j].is_bad) {
unsigned block_state = block_state_get(failed[j].block);
if (block_state == BLOCK_STATE_NEW) {
/* if the block is filled with 0, we have restored it to */
/* the state before the last partial 'sync' */
/* we ignore the case of a file really filled with 0 */
if (memcmp(buffer[failed[j].index], buffer_zero, state->block_size) == 0) {
/* it may contain garbage */
failed[j].is_outofdate = 1;
}
}
if (block_state == BLOCK_STATE_CHG) {
/* we cannot know if we recoverd the state before or after */
/* the partial 'sync' process. */
/* Just to be safe, we assume it is garbage */
failed[j].is_outofdate = 1;
}
}
}
return 0;
}
if (ret > 0)
error += ret;
/* now assume that the parity computation was not updated at the current state */
/* but still represent the state before the last 'sync' process. */
/* this may happen only if a 'sync' is aborted by a crash or manually */
n = 0;
has_bad = 0; /* keep track if there is at least one block to fix */
for(j=0;j<failed_count;++j) {
unsigned block_state;
if (failed[j].is_bad)
has_bad = 1;
if (failed[j].block == BLOCK_DELETED)
block_state = BLOCK_STATE_DELETED;
else
block_state = block_state_get(failed[j].block);
if (failed[j].is_bad /* if the block is bad we don't know its content */
|| block_state == BLOCK_STATE_DELETED /* we don't know the original content of deleted blocks */
|| block_state == BLOCK_STATE_CHG /* we don't know the original content of changed blocks */
) {
if (n<2)
map[n] = j;
++n;
}
/* now adjust the state as before the last 'sync' */
/* if the block is new, restore it to the original 0 as before the 'sync' */
if (block_state == BLOCK_STATE_NEW) {
memset(buffer[failed[j].index], 0, state->block_size);
/* note that from now the buffer is definitively lost */
/* we can do this only because it's the last retry of recovering */
/* mark that we have restored an old state */
failed[j].is_outofdate = 1;
}
}
/* if nothing to fix */
if (!has_bad) {
/* recompute only parity and qarity */
raid_gen(state->level, buffer, diskmax, state->block_size);
return 0;
}
ret = repair_step(state, pos, diskmax, failed, map, n, buffer, buffer_parity, buffer_qarity, buffer_zero);
if (ret == 0) {
/* reprocess the blocks for NEW and CHG ones */
for(j=0;j<failed_count;++j) {
/* we take care only of BAD blocks we have to write back */
if (failed[j].is_bad) {
unsigned block_state = block_state_get(failed[j].block);
if (block_state == BLOCK_STATE_NEW || block_state == BLOCK_STATE_CHG) {
/* for sure we have recovered the state before the sync */
failed[j].is_outofdate = 1;
}
}
}
return 0;
}
if (ret > 0)
error += ret;
/* return the number of failed attempts, or -1 if no strategy */
if (error)
return error;
else
@ -199,11 +397,11 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* for each disk */
one_tocheck = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block
&& block_flag_has(block, BLOCK_HAS_HASH) /* only if the block is hashed */
if (block_is_valid(block)
&& block_has_hash(block) /* only if the block is hashed */
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
@ -225,7 +423,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
for(i=blockstart;i<blockmax;++i) {
unsigned failed_count;
int one_tocheck;
int all_parity;
int check_parity;
unsigned char* buffer_parity;
unsigned char* buffer_qarity;
unsigned char* buffer_zero;
@ -233,11 +431,11 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* for each disk */
one_tocheck = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block
&& block_flag_has(block, BLOCK_HAS_HASH) /* only if the block is hashed */
if (block_is_valid(block)
&& block_has_hash(block) /* only if the block is hashed */
&& !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */
) {
one_tocheck = 1;
@ -249,7 +447,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
if (!one_tocheck)
continue;
all_parity = 1; /* if all hashed block have parity computed */
check_parity = 1; /* if we have to check the parity */
failed_count = 0; /* number of failed blocks */
/* for each disk, process the block */
@ -257,6 +455,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
int read_size;
unsigned char hash[HASH_SIZE];
struct snapraid_block* block;
unsigned block_state;
/* if the disk position is not used */
if (!handle[j].disk) {
@ -267,27 +466,37 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* if the disk block is not used */
block = disk_block_get(handle[j].disk, i);
if (!block) {
if (block == BLOCK_EMPTY) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* we try to check and fix only if the block is hashed */
/* if no hash is present also no parity is present, */
/* but we expect to had it excluded from the parity computation, */
/* so it's correct to assume it filled with 0 and DO NOT reset all_parity */
if (!block_flag_has(block, BLOCK_HAS_HASH)) {
/* if the block is deleted */
if (block == BLOCK_DELETED) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
/* store it in the failed set, because potentially */
/* the parity may be still computed with the previous content */
failed[failed_count].is_bad = 0;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = BLOCK_DELETED;
failed[failed_count].handle = 0;
++failed_count;
continue;
}
/* keep track if at least one hashed block has no parity */
/* if at least one block has no parity, doesn't make sense to check/fix the parity */
/* as errors are the normal condition */
if (!block_flag_has(block, BLOCK_HAS_PARITY))
all_parity = 0;
/* get the state of the block */
block_state = block_state_get(block);
/* if the parity is not valid */
if (block_state != BLOCK_STATE_BLK) {
/* mark the parity as invalid, and don't try to check/fix it */
/* because it will be recomputed at the next sync */
check_parity = 0;
}
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {
@ -329,6 +538,8 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
ret = handle_open(&handle[j], block_file_get(block));
if (ret == -1) {
/* save the failed block for the check/fix */
failed[failed_count].is_bad = 1;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
@ -356,6 +567,8 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
read_size = handle_read(&handle[j], block, buffer[j], state->block_size);
if (read_size == -1) {
/* save the failed block for the check/fix */
failed[failed_count].is_bad = 1;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
@ -366,23 +579,42 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
continue;
}
/* now compute the hash */
memhash(state->hash, hash, buffer[j], read_size);
countsize += read_size;
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* save the failed block for the check/fix */
/* if the block is potentially not updated */
if (block_state == BLOCK_STATE_NEW || block_state == BLOCK_STATE_CHG) {
/* store it in the failed set, because potentially */
/* the parity may be still computed with the previous content */
/* note that these blocks are never recovered, because we cannot check them */
failed[failed_count].is_bad = 0;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
++failed_count;
fprintf(stderr, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
countsize += read_size;
/* if the block has the hash */
if (block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_INV) {
/* now compute the hash */
memhash(state->hash, hash, buffer[j], read_size);
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* save the failed block for the check/fix */
failed[failed_count].is_bad = 1;
failed[failed_count].is_outofdate = 0;
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
++failed_count;
fprintf(stderr, "error:%u:%s:%s: Data error at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
++error;
continue;
}
}
}
/* buffers for parity read and not computed */
@ -430,21 +662,40 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
error += ret;
++unrecoverable_error;
/* print a list of all the errors in files */
for(j=0;j<failed_count;++j) {
if (failed[j].is_bad)
fprintf(stderr, "unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
}
if (fix) {
/* keep track of damaged files */
for(j=0;j<failed_count;++j) {
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
if (failed[j].is_bad)
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
}
}
} else {
/* check if the recover was only partial */
/* note that this could happen only when we have an incomplete 'sync' */
/* and that we have recovered is the state before the 'sync' */
int partial_recover_error = 0;
/* print a list of all the errors in files */
for(j=0;j<failed_count;++j) {
fprintf(stderr, "unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
if (failed[j].is_bad && failed[j].is_outofdate) {
++partial_recover_error;
fprintf(stderr, "unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
}
}
} else {
if (partial_recover_error != 0) {
error += partial_recover_error;
++unrecoverable_error;
}
/* check parity and q-parity only if all the blocks have it computed */
/* if you check/fix after a partial sync, it's OK to have parity errors on the blocks with invalid parity */
if (all_parity) {
if (check_parity) {
/* check the parity */
if (buffer_parity != 0 && memcmp(buffer_parity, buffer[diskmax], state->block_size) != 0) {
buffer_parity = 0;
@ -467,6 +718,10 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
if (fix) {
/* update the fixed files */
for(j=0;j<failed_count;++j) {
/* nothing to do if it doesn't need recovering */
if (!failed[j].is_bad)
continue;
/* do not fix if the file filtered out */
if (file_flag_has(block_file_get(failed[j].block), FILE_IS_EXCLUDED))
continue;
@ -487,6 +742,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
goto bail;
}
/* if we are not sure that the recovered content uptodate */
if (failed[j].is_outofdate) {
/* mark the file as damaged */
file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED);
continue;
}
/* mark the file as fixed */
file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED);
@ -497,7 +759,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* update parity and q-parity only if all the blocks have it computed */
/* if you check/fix after a partial sync, you do not want to fix parity */
/* for blocks that are going to have it computed in the sync completion */
if (all_parity) {
if (check_parity) {
/* update the parity */
if (buffer_parity == 0 && parity) {
ret = parity_write(parity, i, buffer[diskmax], state->block_size);
@ -536,13 +798,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
if (fix) {
/* for all the files of this block check if we need to fix the modification time */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
struct snapraid_file* collide_file;
uint64_t inode;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (!block) {
if (!block_is_valid(block)) {
/* if no block, no file and nothing to do */
continue;
}
@ -594,7 +856,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna
/* when recreating them the inodes may be reused in a different order, */
/* and at the next sync some files may have matching inode/size/time even if different name */
/* not allowing sync to detect that the file is changed and not renamed */
if (!collide_file /* if not in the databse, there is no collision */
if (!collide_file /* if not in the database, there is no collision */
|| strcmp(collide_file->sub, handle[j].file->sub) == 0 /* if the name is the same, it's the right collision */
|| collide_file->size != handle[j].file->size /* if the size is different, the collision is identified */
|| collide_file->mtime != handle[j].file->mtime /* if the mtime is different, the collision is identified */

4
dry.c
View file

@ -53,11 +53,11 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (!block)
if (!block_is_valid(block))
continue;
/* if the file is different than the current one, close it */

2
dup.c
View file

@ -52,7 +52,7 @@ struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_fil
for(i=0;i<file->blockmax;++i) {
memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE);
if (!block_flag_has(&file->blockvec[i], BLOCK_HAS_HASH)) {
if (!block_has_hash(&file->blockvec[i])) {
free(buf);
return 0;
}

89
elem.h
View file

@ -66,9 +66,48 @@ struct snapraid_filter {
struct snapraid_file;
#define BLOCK_HAS_HASH 1 /**< If the hash value is valid. */
#define BLOCK_HAS_PARITY 2 /**< If the parity block is valid. */
#define BLOCK_HAS_MASK 3 /**< Mask of all the flags. Used to mix them inside the file pointer. */
/**
* The block has both the hash and the parity computed.
* This is the normal state of a saved block.
*/
#define BLOCK_STATE_BLK 0
/**
* The block is new and not yet hashed, and it's using a space previously empty.
*/
#define BLOCK_STATE_NEW 1
/**
* The block has the hash computed, but the parity is invalid.
* This happens when the parity is invalidated by a change in another block.
*/
#define BLOCK_STATE_INV 2
/**
* The block is new and not yet hashed, and it's using the space of a block not existing anymore.
* This happens when a new block overwrite a just removed block.
*/
#define BLOCK_STATE_CHG 3
/**
* Mask used to store the previous states of the block.
* Used to mix them inside the file pointer.
*/
#define BLOCK_STATE_MASK 3
/**
* This block is a deleted one.
* Note that this state cannot be stored inside the block,
* but it's usually represented by the block ::BLOCK_DELETED.
*/
#define BLOCK_STATE_DELETED 4
/**
* This block is an empty one.
* Note that this state cannot be stored inside the block,
* but it's usually represented by the block ::BLOCK_EMPTY.
*/
#define BLOCK_STATE_EMPTY 5
/**
* Block of a file.
@ -79,6 +118,16 @@ struct snapraid_block {
unsigned char hash[HASH_SIZE]; /**< Hash of the block. */
};
/**
* Block pointer used to mark unused blocks.
*/
#define BLOCK_EMPTY 0
/**
* Block pointer used to mark deleted blocks.
*/
#define BLOCK_DELETED ((struct snapraid_block*)-1)
#define FILE_IS_PRESENT 1 /**< If it's seen as present. */
#define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */
#define FILE_IS_LARGER 4 /**< If a larger file was already detected. */
@ -204,7 +253,7 @@ int filter_content(tommy_list* contentlist, const char* path);
*/
static inline struct snapraid_file* block_file_get(struct snapraid_block* block)
{
return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)BLOCK_HAS_MASK);
return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK);
}
/**
@ -212,22 +261,33 @@ static inline struct snapraid_file* block_file_get(struct snapraid_block* block)
*/
static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file)
{
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_HAS_MASK) | (uintptr_t)file;
block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | (uintptr_t)file;
}
static inline int block_flag_has(struct snapraid_block* block, unsigned mask)
static inline unsigned block_state_get(struct snapraid_block* block)
{
return (block->file_mixed & mask) == mask;
return block->file_mixed & BLOCK_STATE_MASK;
}
static inline void block_flag_set(struct snapraid_block* block, unsigned mask)
static inline void block_state_set(struct snapraid_block* block, unsigned state)
{
block->file_mixed |= mask;
assert(state != BLOCK_STATE_EMPTY && state != BLOCK_STATE_DELETED);
block->file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK;
block->file_mixed |= state & BLOCK_STATE_MASK;
}
static inline void block_flag_clear(struct snapraid_block* block, unsigned mask)
static inline void block_clear_parity(struct snapraid_block* block)
{
block->file_mixed &= ~(uintptr_t)mask;
unsigned state = block_state_get(block);
if (state == BLOCK_STATE_BLK) {
block_state_set(block, BLOCK_STATE_INV);
}
}
static inline int block_has_hash(struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV;
}
/**
@ -240,6 +300,11 @@ block_off_t block_file_pos(struct snapraid_block* block);
*/
int block_is_last(struct snapraid_block* block);
static inline int block_is_valid(struct snapraid_block* block)
{
return block != BLOCK_EMPTY && block != BLOCK_DELETED;
}
/**
* Gets the size in bytes of the block.
* If it's the last block of a file it could be less than block_size.
@ -359,7 +424,7 @@ static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk,
if (pos < tommy_array_size(&disk->blockarr))
return tommy_array_get(&disk->blockarr, pos);
else
return 0;
return BLOCK_EMPTY;
}
/**

4
parity.c
View file

@ -37,8 +37,8 @@ block_off_t parity_resize(struct snapraid_state* state)
/* start from the declared size */
block_off_t block = tommy_array_size(&disk->blockarr);
/* decrease the block until an allocated block */
while (block > parity_block && tommy_array_get(&disk->blockarr, block - 1) == 0)
/* decrease the block until an allocated or deleted block */
while (block > parity_block && !block_is_valid(tommy_array_get(&disk->blockarr, block - 1)))
--block;
/* get the highest value */

21
scan.c
View file

@ -57,16 +57,17 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
if (disk->first_free_block > block_pos)
disk->first_free_block = block_pos;
/* clear the block */
tommy_array_set(&disk->blockarr, block_pos, 0);
/* set the block as deleted */
tommy_array_set(&disk->blockarr, block_pos, BLOCK_DELETED);
/* invalidate the block of all the other disks */
for(j=state->disklist;j!=0;j=j->next) {
struct snapraid_disk* oth_disk = j->data;
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
if (oth_block) {
if (block_is_valid(oth_block)) {
/* remove the parity info for this block */
block_flag_clear(oth_block, BLOCK_HAS_PARITY);
block_clear_parity(oth_block);
}
}
}
@ -97,7 +98,7 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
block_max = tommy_array_size(&disk->blockarr);
for(i=0;i<file->blockmax;++i) {
/* find a free block */
while (block_pos < block_max && tommy_array_get(&disk->blockarr, block_pos) != 0)
while (block_pos < block_max && block_is_valid(tommy_array_get(&disk->blockarr, block_pos)))
++block_pos;
/* if not found, allocate a new one */
@ -106,8 +107,16 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
tommy_array_grow(&disk->blockarr, block_max);
}
/* set it */
/* set the position */
file->blockvec[i].parity_pos = block_pos;
/* set the state depending if the used block was EMPTY or DELETED */
if (tommy_array_get(&disk->blockarr, block_pos) == BLOCK_EMPTY)
block_state_set(&file->blockvec[i], BLOCK_STATE_NEW);
else
block_state_set(&file->blockvec[i], BLOCK_STATE_CHG);
/* store in the disk map */
tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]);
}
if (file->blockmax) {

24
snapraid.1
View file

@ -851,10 +851,30 @@ BLOCK is the block position in the \(a"parity\(a" file.
.PP
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
.SS inv BLOCK [HASH]
.SS inv BLOCK HASH
Like \(a"blk\(a", but inform that the parity of this block is invalid.
.PP
The HASH may be missing if not yet computed.
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS new BLOCK
Like \(a"blk\(a", but for new allocated blocks for which the hash is not
yet computed.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS chg BLOCK
Like \(a"blk\(a", but for reallocated blocks for which the hash is not
yet computed.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS hole DISK
Defines the list of blocks that are deleted from a disk.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS off BLOCK
Defines the block deleted from a disk.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.

28
snapraid.d
View file

@ -609,10 +609,34 @@ Content
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
inv BLOCK [HASH]
inv BLOCK HASH
Like "blk", but inform that the parity of this block is invalid.
The HASH may be missing if not yet computed.
This field is used only when you interrupt manually the "sync"
command.
new BLOCK
Like "blk", but for new allocated blocks for which the hash is not
yet computed.
This field is used only when you interrupt manually the "sync"
command.
chg BLOCK
Like "blk", but for reallocated blocks for which the hash is not
yet computed.
This field is used only when you interrupt manually the "sync"
command.
hole DISK
Defines the list of blocks that are deleted from a disk.
This field is used only when you interrupt manually the "sync"
command.
off BLOCK
Defines the block deleted from a disk.
This field is used only when you interrupt manually the "sync"
command.

38
snapraid.txt
View file

@ -678,12 +678,44 @@ BLOCK is the block position in the "parity" file.
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
10.7 inv BLOCK [HASH]
---------------------
10.7 inv BLOCK HASH
-------------------
Like "blk", but inform that the parity of this block is invalid.
The HASH may be missing if not yet computed.
This field is used only when you interrupt manually the "sync"
command.
10.8 new BLOCK
--------------
Like "blk", but for new allocated blocks for which the hash is not
yet computed.
This field is used only when you interrupt manually the "sync"
command.
10.9 chg BLOCK
--------------
Like "blk", but for reallocated blocks for which the hash is not
yet computed.
This field is used only when you interrupt manually the "sync"
command.
10.10 hole DISK
---------------
Defines the list of blocks that are deleted from a disk.
This field is used only when you interrupt manually the "sync"
command.
10.11 off BLOCK
---------------
Defines the block deleted from a disk.
This field is used only when you interrupt manually the "sync"
command.

153
state.c
View file

@ -624,8 +624,8 @@ void state_read(struct snapraid_state* state)
sungetc(c, f);
}
if (strcmp(tag, "blk") == 0 || strcmp(tag, "inv") == 0) {
/* "blk"/"inv" command */
if (strcmp(tag, "blk") == 0 || strcmp(tag, "inv") == 0 || strcmp(tag, "new") == 0 || strcmp(tag, "chg") == 0) {
/* "blk"/"inv"/"new"/"chg" command */
block_off_t v_pos;
struct snapraid_block* block;
@ -653,22 +653,31 @@ void state_read(struct snapraid_state* state)
}
/* set the parity only if present */
if (tag[0] == 'b') {
block_flag_set(block, BLOCK_HAS_PARITY);
switch (tag[0]) {
case 'b' :
block_state_set(block, BLOCK_STATE_BLK);
hash = oathash8(hash, 'b');
} else {
break;
case 'i' :
block_state_set(block, BLOCK_STATE_INV);
hash = oathash8(hash, 'i');
break;
case 'n' :
block_state_set(block, BLOCK_STATE_NEW);
hash = oathash8(hash, 'n');
break;
case 'c' :
block_state_set(block, BLOCK_STATE_CHG);
hash = oathash8(hash, 'g');
break;
}
block->parity_pos = v_pos;
hash = oathash32(hash, v_pos);
/* check if we are at the end of the line */
c = sgeteol(f);
if (c == '\n') {
/* no hash present */
sungetc(c, f);
} else {
/* read the hash only for some state */
if (tag[0] == 'b' || tag[0] == 'i') {
c = sgeteol(f);
if (c != ' ') {
fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
@ -681,14 +690,6 @@ void state_read(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
hash = oathashm(hash, block->hash, HASH_SIZE);
block_flag_set(block, BLOCK_HAS_HASH);
}
/* parity implies hash */
if (block_flag_has(block, BLOCK_HAS_PARITY) && !block_flag_has(block, BLOCK_HAS_HASH)) {
fprintf(stderr, "Internal inconsistency in 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
/* insert the block in the block array */
@ -704,6 +705,28 @@ void state_read(struct snapraid_state* state)
/* stat */
++count_block;
} else if (strcmp(tag, "off") == 0) {
/* "off" command */
block_off_t v_pos;
hash = oathash8(hash, 'o');
if (!disk) {
fprintf(stderr, "Unexpected 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
ret = sgetu32(f, &v_pos);
if (ret < 0) {
fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathash32(hash, v_pos);
/* insert the block in the block array */
tommy_array_grow(&disk->blockarr, v_pos + 1);
tommy_array_set(&disk->blockarr, v_pos, BLOCK_DELETED);
} else if (strcmp(tag, "file") == 0) {
/* file */
char sub[PATH_MAX];
@ -813,6 +836,34 @@ void state_read(struct snapraid_state* state)
/* stat */
++count_file;
} else if (strcmp(tag, "hole") == 0) {
/* hole */
tommy_node* i;
hash = oathash8(hash, 'h');
if (file) {
fprintf(stderr, "Missing 'blk' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
ret = sgettok(f, buffer, sizeof(buffer));
if (ret < 0) {
fprintf(stderr, "Invalid 'hole' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
hash = oathashs(hash, buffer);
/* find the disk */
for(i=state->disklist;i!=0;i=i->next) {
disk = i->data;
if (strcmp(disk->name, buffer) == 0)
break;
}
if (!i) {
fprintf(stderr, "Disk named '%s' not found in '%s' at line %u\n", buffer, path, line);
exit(EXIT_FAILURE);
}
} else if (strcmp(tag, "symlink") == 0) {
/* symlink */
char sub[PATH_MAX];
@ -1123,8 +1174,8 @@ void state_write(struct snapraid_state* state)
/* for each file */
for(j=disk->filelist;j!=0;j=j->next) {
block_off_t k;
struct snapraid_file* file = j->data;
block_off_t k;
uint64_t size;
uint64_t mtime;
uint64_t inode;
@ -1155,22 +1206,35 @@ void state_write(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
/* for each block */
/* for each block in the file */
for(k=0;k<file->blockmax;++k) {
struct snapraid_block* block = &file->blockvec[k];
unsigned block_state;
if (block_flag_has(block, BLOCK_HAS_PARITY)) {
block_state = block_state_get(block);
switch (block_state) {
case BLOCK_STATE_BLK :
sputsl("blk ", f);
hash = oathash8(hash, 'b');
} else {
break;
case BLOCK_STATE_INV :
sputsl("inv ", f);
hash = oathash8(hash, 'i');
break;
case BLOCK_STATE_NEW :
sputsl("new ", f);
hash = oathash8(hash, 'n');
break;
case BLOCK_STATE_CHG :
sputsl("chg ", f);
hash = oathash8(hash, 'g');
break;
}
sputu32(block->parity_pos, f);
hash = oathash32(hash, block->parity_pos);
if (block_flag_has(block, BLOCK_HAS_HASH)) {
if (block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_INV) {
sputc(' ', f);
sputhex(block->hash, HASH_SIZE, f);
hash = oathashm(hash, block->hash, HASH_SIZE);
@ -1211,6 +1275,47 @@ void state_write(struct snapraid_state* state)
++count_link;
}
{
block_off_t k;
block_off_t blockmax;
int first_deleted;
first_deleted = 1;
blockmax = tommy_array_size(&disk->blockarr);
/* for each deleted block in the disk */
for(k=0;k<blockmax;++k) {
struct snapraid_block* block = tommy_array_get(&disk->blockarr, k);
if (block == BLOCK_DELETED) {
if (first_deleted) {
first_deleted = 0;
sputsl("hole ", f);
hash = oathash8(hash, 'h');
sputs(disk->name, f);
hash = oathashs(hash, disk->name);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
}
}
sputsl("off ", f);
hash = oathash8(hash, 'o');
sputu32(k, f);
hash = oathash32(hash, k);
sputeol(f);
if (serror(f)) {
fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno));
exit(EXIT_FAILURE);
}
}
}
}
}
sputsl("sign ", f);

70
sync.c
View file

@ -65,10 +65,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* for each disk */
one_invalid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block && !block_flag_has(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
if (block_is_valid(block)
&& block_state_get(block) != BLOCK_STATE_BLK
) {
one_invalid = 1;
break;
}
@ -92,18 +94,36 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* for each disk */
one_invalid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block && !block_flag_has(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY)) {
if (block_is_valid(block)
&& block_state_get(block) != BLOCK_STATE_BLK
) {
one_invalid = 1;
break;
}
}
/* if no invalid block skip */
if (!one_invalid)
if (!one_invalid) {
/* cleanup all the deleted blocks */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block == BLOCK_DELETED) {
/* set it to empty */
tommy_array_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
/* mark the state as needing write */
state->need_write = 1;
}
}
/* skip */
continue;
}
/* by default process the block, and skip it if something go wrong */
skip_this_block = 0;
@ -123,7 +143,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* if the disk block is not used */
block = disk_block_get(handle[j].disk, i);
if (!block) {
if (!block_is_valid(block)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
@ -196,10 +216,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
goto bail;
}
countsize += read_size;
/* now compute the hash */
memhash(state->hash, hash, buffer[j], read_size);
if (block_flag_has(block, BLOCK_HAS_HASH)) {
if (block_has_hash(block)) {
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block_file_get(block)->sub, block_file_pos(block));
@ -213,8 +235,6 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* this allow on error to do not save the failed computation */
memcpy(block->hash, hash, HASH_SIZE);
}
countsize += read_size;
}
/* if we have read all the data required, proceed with the parity */
@ -245,14 +265,23 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
/* for each disk, mark the blocks as processed */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = 0;
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (!block)
if (block == BLOCK_EMPTY) {
/* nothing to do */
continue;
}
if (block == BLOCK_DELETED) {
/* the parity is not updated without this block, so it's now empty */
tommy_array_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY);
continue;
}
/* now all the blocks have the hash and the parity computed */
block_flag_set(block, BLOCK_HAS_HASH | BLOCK_HAS_PARITY);
block_state_set(block, BLOCK_STATE_BLK);
}
}
@ -293,6 +322,7 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
{
block_off_t blockmax;
data_off_t size;
tommy_node* i;
int ret;
struct snapraid_parity parity;
struct snapraid_parity qarity;
@ -305,6 +335,22 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
blockmax = parity_resize(state);
size = blockmax * (data_off_t)state->block_size;
/* remove all the deleted blocks over the upper limit */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* disk = i->data;
block_off_t diskblockmax = tommy_array_size(&disk->blockarr);
block_off_t block;
for(block=blockmax;block<diskblockmax;++block) {
if (tommy_array_get(&disk->blockarr, block) == BLOCK_DELETED) {
tommy_array_set(&disk->blockarr, block, BLOCK_EMPTY);
/* mark the state as needing write */
state->need_write = 1;
}
}
}
if (blockstart > blockmax) {
fprintf(stderr, "Error in the starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax);
exit(EXIT_FAILURE);