rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Removes INV blocks. There are not unnecessary as we always keep deleted/changed ones to keep the information that the parity is invalid.

Browse source
This commit is contained in:
Andrea Mazzoleni 2013-02-24 19:43:53 +01:00
parent 5c4f6dc5c4
commit 1e3e88d51c
9 changed files with 70 additions and 179 deletions
Download patch file Download diff file Expand all files Collapse all files

29
TODO
View file

@ -3,6 +3,12 @@ SnapRAID TODO
This is the list of TODO items for SnapRAID.
- v2.2
* We can extend the fix command to read know deleted files from a specific directory,
even outside the array, compute the hash of all the blocks, and use it as a bag of data,
indexed by hash, to find missing data for DELETED and CHG blocks.
- v2.3
* If a content file is missing, sync should rewrite it, even if no change was done at the content.
@ -59,27 +65,8 @@ or completely garbage. But it makes sense to anyway write the "most likely" corr
to all the previous blocks, because we always proceeed in block order during a sync.
So, if for a block we can detect that we recovered using updated parity data,
also for all the previous blocks this is true.
At now this could happen only if we have NEW block resulting different than zero.
Another possibility is to have a CHG one, with an hash different than the old one,
(but at now we don't keep this old hash value).
Anyway, the problem is that it's difficult to use this information, without proceeding
backward in the recovering process.
But we could have an initial recovering phase trying to detect this point.
Or we can just continously saving it during a sync.
Anyway, if we keep track of old CHG hash, we get nearly the same effect.
The only difference is that we don't know if NEW files filled with 0
are really recovered or not, so they'll result with the .unrecoverable
extension.
* In the "interim" content file during sync, we must keep track of all the hash
of CHG blocks overwritten. These hash will allow to detect if a block is recovered to its
old or new state.
Note that we don't need to keep track of the hash of DEL blocks.
It will allow also to recover from deleted files. We can extend the fix command
to read know deleted files from a specific directory, even outside the array,
compute the hash of all the blocks, and use it as a bag of data, indexed by hash.
Likely we'll need to extend fix to allow an incremental process, store a .content
file of the hash already computed to avoid to recompute them.
Anyway, the case where this information could be useful should be present
only if changes are committed after an aborted sync.
* When fixing, before overwriting the present file, make a copy of it just in
case that the original file cannot be completely recovered.

34
check.c
View file

@ -460,7 +460,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
for(i=blockstart;i<blockmax;++i) {
unsigned failed_count;
int one_tocheck;
int check_parity;
int valid_parity;
/* for each disk */
one_tocheck = 0;
@ -480,10 +480,12 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
if (!one_tocheck)
continue;
/* If we have to check the parity data read from disk. */
/* If we have valid parity, and it makes sense to check its content. */
/* If we already know that the parity is invalid, we just read the file */
/* but we don't report errors */
/* Note that if check==0, we'll anyway skip the full parity check, */
/* because we also don't read it at all */
check_parity = 1;
valid_parity = 1;
/* keep track of the number of failed blocks */
failed_count = 0;
@ -513,17 +515,18 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* get the state of the block */
block_state = block_state_get(block);
/* if the parity is not valid */
if (block_has_invalid_parity(block)) {
/* mark the parity as invalid, and don't try to check/fix it */
/* because it will be recomputed at the next sync */
valid_parity = 0;
}
/* if the block is deleted */
if (block_state == BLOCK_STATE_DELETED) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
/* mark the parity as invalid, and don't try to check/fix it */
/* because it will be recomputed at the next sync */
/* note that for sure there is also another block invalidating */
/* the parity, but for completeness we invalidate it also here */
check_parity = 0;
/* store it in the failed set, because potentially */
/* the parity may be still computed with the previous content */
failed[failed_count].is_bad = 0;
@ -544,13 +547,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
continue;
}
/* if the parity is not valid */
if (block_state != BLOCK_STATE_BLK) {
/* mark the parity as invalid, and don't try to check/fix it */
/* because it will be recomputed at the next sync */
check_parity = 0;
}
/* if the file is different than the current one, close it */
if (handle[j].file != block_file_get(block)) {
ret = handle_close(&handle[j]);
@ -650,7 +646,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
}
/* if the block has the hash */
if (block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_INV) {
if (block_state == BLOCK_STATE_BLK) {
/* compute the hash of the block just read */
memhash(state->hash, hash, buffer[j], read_size);
@ -757,7 +753,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* now check parity and q-parity, but only if all the blocks have it computed */
/* if you check/fix after a partial sync, it's OK to have parity errors on the blocks with invalid parity */
/* and doesn't make sense to try to fix it */
if (check_parity) {
if (valid_parity) {
/* check the parity */
if (buffer_parity != 0 && memcmp(buffer_parity, buffer[diskmax], state->block_size) != 0) {
buffer_parity = 0;
@ -822,7 +818,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
/* update parity and q-parity only if all the blocks have it computed */
/* if you check/fix after a partial sync, you do not want to fix parity */
/* for blocks that are going to have it computed in the sync completion */
if (check_parity) {
if (valid_parity) {
/* update the parity */
if (buffer_parity == 0 && parity) {
ret = parity_write(parity, i, buffer[diskmax], state->block_size);

46
elem.h
View file

@ -86,15 +86,6 @@ struct snapraid_file;
*/
#define BLOCK_STATE_NEW 2
/**
* The block has the hash computed, but the parity is invalid.
* This happens when the parity is invalidated by a change in another disk.
*
* The block hash field IS set.
* The parity for this disk is updated, but there is at least another disk not updated parity.
*/
#define BLOCK_STATE_INV 3
/**
* The block is new and not yet hashed, and it's using the space of a block not existing anymore.
* This happens when a new block overwrite a just removed block.
@ -104,7 +95,7 @@ struct snapraid_file;
*
* If the hash is completely filled with 0, it means that its lost.
*/
#define BLOCK_STATE_CHG 4
#define BLOCK_STATE_CHG 3
/**
* This block is a deleted one.
@ -115,7 +106,7 @@ struct snapraid_file;
*
* If the hash is completely filled with 0, it means that its lost.
*/
#define BLOCK_STATE_DELETED 5
#define BLOCK_STATE_DELETED 4
/**
* Mask used to store the previous states of the block.
@ -381,37 +372,40 @@ static inline void block_state_set(struct snapraid_block* block, unsigned state)
block->file_mixed |= state & BLOCK_STATE_MASK;
}
static inline void block_clear_parity(struct snapraid_block* block)
{
unsigned state = block_state_get(block);
if (state == BLOCK_STATE_BLK) {
block_state_set(block, BLOCK_STATE_INV);
}
}
/**
* Check if the specified block has a valid and updated hash.
* Checks if the specified block has a valid and updated hash.
*
* Note that blocks with hash of old data, like CHG and DELETED ones, are not considered.
* Note that EMPTY return 0.
*/
static inline int block_has_hash(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV;
return state == BLOCK_STATE_BLK;
}
/**
* Check if the specified block has a valid file.
* Checks if the specified block is part of a file.
*
* It includes BLK/INV/NEW/CHG and excludes EMPTY/DELETED.
* Note that EMPTY return 0.
*/
static inline int block_has_file(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG;
return state == BLOCK_STATE_BLK || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG;
}
/**
* Checks if the block has an invalid parity than needs to be updated.
*
* Note that EMPTY return 0.
*/
static inline int block_has_invalid_parity(const struct snapraid_block* block)
{
unsigned state = block_state_get(block);
return state == BLOCK_STATE_DELETED || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG;
}
/**

21
scan.c
View file

@ -164,7 +164,6 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
struct snapraid_block* block = &file->blockvec[i];
block_off_t block_pos = block->parity_pos;
unsigned block_state;
tommy_node* j;
struct snapraid_deleted* deleted;
/* adjust the first free position */
@ -176,7 +175,6 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
block_state = block_state_get(block);
switch (block_state) {
case BLOCK_STATE_BLK :
case BLOCK_STATE_INV :
/* we keep the hash making it an "old" hash, because the parity is still containing data for it */
break;
case BLOCK_STATE_CHG :
@ -204,15 +202,6 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk*
/* set the deleted block in the block array */
tommy_array_set(&disk->blockarr, block_pos, &deleted->block);
/* invalidate the block of all the other disks */
for(j=state->disklist;j!=0;j=j->next) {
struct snapraid_disk* oth_disk = j->data;
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
/* remove the parity info for this block */
block_clear_parity(oth_block);
}
}
/* remove the file from the file containers */
@ -240,7 +229,6 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
block_pos = disk->first_free_block;
block_max = tommy_array_size(&disk->blockarr);
for(i=0;i<file->blockmax;++i) {
tommy_node* j;
struct snapraid_block* block;
/* find a free block */
@ -269,15 +257,6 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
memcpy(file->blockvec[i].hash, block->hash, HASH_SIZE);
}
/* invalidate the block of all the other disks */
for(j=state->disklist;j!=0;j=j->next) {
struct snapraid_disk* oth_disk = j->data;
struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos);
/* remove the parity info for this block */
block_clear_parity(oth_block);
}
/* store in the disk map, after invalidating all the other blocks */
tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]);
}

5
snapraid.1
View file

@ -1005,11 +1005,6 @@ BLOCK is the block position in the \(a"parity\(a" file.
.PP
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
.SS inv BLOCK HASH
Like \(a"blk\(a", but inform that the parity of this block is invalid.
.PP
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SS new BLOCK
Like \(a"blk\(a", but for new allocated blocks for which the hash is not
yet computed, and the stored parity doesn\(cqt take into account this

6
snapraid.d
View file

@ -727,12 +727,6 @@ Content
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
inv BLOCK HASH
Like "blk", but inform that the parity of this block is invalid.
This field is used only when you interrupt manually the "sync"
command.
new BLOCK
Like "blk", but for new allocated blocks for which the hash is not
yet computed, and the stored parity doesn't take into account this

26
snapraid.txt
View file

@ -810,15 +810,7 @@ BLOCK is the block position in the "parity" file.
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
10.7 inv BLOCK HASH
-------------------
Like "blk", but inform that the parity of this block is invalid.
This field is used only when you interrupt manually the "sync"
command.
10.8 new BLOCK
10.7 new BLOCK
--------------
Like "blk", but for new allocated blocks for which the hash is not
@ -828,7 +820,7 @@ new block.
This field is used only when you interrupt manually the "sync"
command.
10.9 chg BLOCK
10.8 chg BLOCK
--------------
Like "blk", but for reallocated blocks for which the hash is not
@ -838,15 +830,15 @@ of the block.
This field is used only when you interrupt manually the "sync"
command.
10.10 hole DISK
---------------
10.9 hole DISK
--------------
Defines the list of blocks that are deleted from a disk.
This field is used only when you interrupt manually the "sync"
command.
10.11 off BLOCK
10.10 off BLOCK
---------------
Defines a block deleted from a disk, part of the last defined hole,
@ -855,22 +847,22 @@ for which the parity is computed using the previous value.
This field is used only when you interrupt manually the "sync"
command.
10.12 symlink DISK PATH
10.11 symlink DISK PATH
-----------------------
Defines a symbolic link in the specified DISK.
10.13 hardlink DISK PATH
10.12 hardlink DISK PATH
------------------------
Defines a hard link in the specified DISK.
10.14 to PATH
10.13 to PATH
-------------
Defines where the previous symlink or hardlink links to.
10.15 dir DISK PATH
10.14 dir DISK PATH
-------------------
Defines an empty directory in the specified DISK.

51
state.c
View file

@ -678,7 +678,6 @@ static void state_map(struct snapraid_state* state)
static void state_content_check(struct snapraid_state* state, const char* path)
{
tommy_node* i;
block_off_t j;
/* checks that any map has different name and position */
for(i=state->maplist;i!=0;i=i->next) {
@ -696,48 +695,6 @@ static void state_content_check(struct snapraid_state* state, const char* path)
}
}
}
/* check the parity validity */
for(j=0;1;++j) { /* for all the blocks */
int at_least_one = 0;
int parity_is_valid = 0;
int parity_is_invalid = 0;
/* for all the disks */
for(i=state->disklist;i!=0;i=i->next) {
struct snapraid_disk* disk = i->data;
/* if the block is present */
block_off_t size = tommy_array_size(&disk->blockarr);
if (j < size) {
struct snapraid_block* block = tommy_array_get(&disk->blockarr, j);
/* mark if at least one block is addressed */
at_least_one = 1;
if (block == BLOCK_EMPTY) {
/* not relevant */
} else {
unsigned block_state = block_state_get(block);
if (block_state == BLOCK_STATE_BLK) {
parity_is_valid = 1;
} else {
parity_is_invalid = 1;
}
}
}
}
/* we cannot have different blocks reporting both valid and invalid parity */
if (parity_is_valid && parity_is_invalid) {
fprintf(stderr, "Internal inconsistency in parity validity at block %u\n", j);
exit(EXIT_FAILURE);
}
/* stop if reached the end of all the arrays */
if (!at_least_one)
break;
}
}
void state_read(struct snapraid_state* state)
@ -873,10 +830,6 @@ void state_read(struct snapraid_state* state)
block_state_set(block, BLOCK_STATE_BLK);
hash = oathash8(hash, 'b');
break;
case 'i' :
block_state_set(block, BLOCK_STATE_INV);
hash = oathash8(hash, 'i');
break;
case 'n' :
block_state_set(block, BLOCK_STATE_NEW);
hash = oathash8(hash, 'n');
@ -1653,10 +1606,6 @@ void state_write(struct snapraid_state* state)
sputsl("blk ", f);
hash = oathash8(hash, 'b');
break;
case BLOCK_STATE_INV :
sputsl("inv ", f);
hash = oathash8(hash, 'i');
break;
case BLOCK_STATE_NEW :
sputsl("new ", f);
hash = oathash8(hash, 'n');

31
sync.c
View file

@ -64,24 +64,28 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
countmax = 0;
for(i=blockstart;i<blockmax;++i) {
int one_invalid;
int one_valid;
/* for each disk */
one_invalid = 0;
one_valid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
if (block_has_file(block)
&& block_state_get(block) != BLOCK_STATE_BLK
) {
if (block_has_file(block)) {
one_valid = 1;
}
if (block_has_invalid_parity(block)) {
one_invalid = 1;
break;
}
}
/* if no invalid block skip */
if (!one_invalid)
/* if none valid or none invalid, we don't need to update */
if (!one_invalid || !one_valid) {
continue;
}
++countmax;
}
@ -98,27 +102,28 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
state_progress_begin(state, blockstart, blockmax, countmax);
for(i=blockstart;i<blockmax;++i) {
int one_invalid;
int one_valid;
int skip_this_block;
int ret;
/* for each disk */
one_invalid = 0;
one_valid = 0;
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = BLOCK_EMPTY;
if (handle[j].disk)
block = disk_block_get(handle[j].disk, i);
/* if the block is used, and it needs some update */
if (block_has_file(block)
&& block_state_get(block) != BLOCK_STATE_BLK
) {
if (block_has_file(block)) {
one_valid = 1;
}
if (block_has_invalid_parity(block)) {
one_invalid = 1;
break;
}
}
/* if no invalid block skip */
if (!one_invalid) {
/* if none valid or none invalid, we don't need to update */
if (!one_invalid || !one_valid) {
/* cleanup all the deleted blocks at this position */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block = BLOCK_EMPTY;