diff --git a/TODO b/TODO index 60fb5ab8..65a56fec 100644 --- a/TODO +++ b/TODO @@ -3,6 +3,12 @@ SnapRAID TODO This is the list of TODO items for SnapRAID. +- v2.2 + +* We can extend the fix command to read know deleted files from a specific directory, +even outside the array, compute the hash of all the blocks, and use it as a bag of data, +indexed by hash, to find missing data for DELETED and CHG blocks. + - v2.3 * If a content file is missing, sync should rewrite it, even if no change was done at the content. @@ -59,27 +65,8 @@ or completely garbage. But it makes sense to anyway write the "most likely" corr to all the previous blocks, because we always proceeed in block order during a sync. So, if for a block we can detect that we recovered using updated parity data, also for all the previous blocks this is true. -At now this could happen only if we have NEW block resulting different than zero. -Another possibility is to have a CHG one, with an hash different than the old one, -(but at now we don't keep this old hash value). -Anyway, the problem is that it's difficult to use this information, without proceeding -backward in the recovering process. -But we could have an initial recovering phase trying to detect this point. -Or we can just continously saving it during a sync. -Anyway, if we keep track of old CHG hash, we get nearly the same effect. -The only difference is that we don't know if NEW files filled with 0 -are really recovered or not, so they'll result with the .unrecoverable -extension. - -* In the "interim" content file during sync, we must keep track of all the hash -of CHG blocks overwritten. These hash will allow to detect if a block is recovered to its -old or new state. -Note that we don't need to keep track of the hash of DEL blocks. -It will allow also to recover from deleted files. We can extend the fix command -to read know deleted files from a specific directory, even outside the array, -compute the hash of all the blocks, and use it as a bag of data, indexed by hash. -Likely we'll need to extend fix to allow an incremental process, store a .content -file of the hash already computed to avoid to recompute them. +Anyway, the case where this information could be useful should be present +only if changes are committed after an aborted sync. * When fixing, before overwriting the present file, make a copy of it just in case that the original file cannot be completely recovered. diff --git a/check.c b/check.c index c81329cc..687c6538 100644 --- a/check.c +++ b/check.c @@ -460,7 +460,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, for(i=blockstart;iblock_size); - /* mark the parity as invalid, and don't try to check/fix it */ - /* because it will be recomputed at the next sync */ - /* note that for sure there is also another block invalidating */ - /* the parity, but for completeness we invalidate it also here */ - check_parity = 0; - /* store it in the failed set, because potentially */ /* the parity may be still computed with the previous content */ failed[failed_count].is_bad = 0; @@ -544,13 +547,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, continue; } - /* if the parity is not valid */ - if (block_state != BLOCK_STATE_BLK) { - /* mark the parity as invalid, and don't try to check/fix it */ - /* because it will be recomputed at the next sync */ - check_parity = 0; - } - /* if the file is different than the current one, close it */ if (handle[j].file != block_file_get(block)) { ret = handle_close(&handle[j]); @@ -650,7 +646,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, } /* if the block has the hash */ - if (block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_INV) { + if (block_state == BLOCK_STATE_BLK) { /* compute the hash of the block just read */ memhash(state->hash, hash, buffer[j], read_size); @@ -757,7 +753,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* now check parity and q-parity, but only if all the blocks have it computed */ /* if you check/fix after a partial sync, it's OK to have parity errors on the blocks with invalid parity */ /* and doesn't make sense to try to fix it */ - if (check_parity) { + if (valid_parity) { /* check the parity */ if (buffer_parity != 0 && memcmp(buffer_parity, buffer[diskmax], state->block_size) != 0) { buffer_parity = 0; @@ -822,7 +818,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* update parity and q-parity only if all the blocks have it computed */ /* if you check/fix after a partial sync, you do not want to fix parity */ /* for blocks that are going to have it computed in the sync completion */ - if (check_parity) { + if (valid_parity) { /* update the parity */ if (buffer_parity == 0 && parity) { ret = parity_write(parity, i, buffer[diskmax], state->block_size); diff --git a/elem.h b/elem.h index 2a13ffbb..d2f12e0c 100644 --- a/elem.h +++ b/elem.h @@ -86,15 +86,6 @@ struct snapraid_file; */ #define BLOCK_STATE_NEW 2 -/** - * The block has the hash computed, but the parity is invalid. - * This happens when the parity is invalidated by a change in another disk. - * - * The block hash field IS set. - * The parity for this disk is updated, but there is at least another disk not updated parity. - */ -#define BLOCK_STATE_INV 3 - /** * The block is new and not yet hashed, and it's using the space of a block not existing anymore. * This happens when a new block overwrite a just removed block. @@ -104,7 +95,7 @@ struct snapraid_file; * * If the hash is completely filled with 0, it means that its lost. */ -#define BLOCK_STATE_CHG 4 +#define BLOCK_STATE_CHG 3 /** * This block is a deleted one. @@ -115,7 +106,7 @@ struct snapraid_file; * * If the hash is completely filled with 0, it means that its lost. */ -#define BLOCK_STATE_DELETED 5 +#define BLOCK_STATE_DELETED 4 /** * Mask used to store the previous states of the block. @@ -381,37 +372,40 @@ static inline void block_state_set(struct snapraid_block* block, unsigned state) block->file_mixed |= state & BLOCK_STATE_MASK; } -static inline void block_clear_parity(struct snapraid_block* block) -{ - unsigned state = block_state_get(block); - - if (state == BLOCK_STATE_BLK) { - block_state_set(block, BLOCK_STATE_INV); - } -} - /** - * Check if the specified block has a valid and updated hash. + * Checks if the specified block has a valid and updated hash. * - * Note that blocks with hash of old data, like CHG and DELETED ones, are not considered. + * Note that EMPTY return 0. */ static inline int block_has_hash(const struct snapraid_block* block) { unsigned state = block_state_get(block); - return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV; + return state == BLOCK_STATE_BLK; } /** - * Check if the specified block has a valid file. + * Checks if the specified block is part of a file. * - * It includes BLK/INV/NEW/CHG and excludes EMPTY/DELETED. + * Note that EMPTY return 0. */ static inline int block_has_file(const struct snapraid_block* block) { unsigned state = block_state_get(block); - return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG; + return state == BLOCK_STATE_BLK || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG; +} + +/** + * Checks if the block has an invalid parity than needs to be updated. + * + * Note that EMPTY return 0. + */ +static inline int block_has_invalid_parity(const struct snapraid_block* block) +{ + unsigned state = block_state_get(block); + + return state == BLOCK_STATE_DELETED || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG; } /** diff --git a/scan.c b/scan.c index 3de027f8..ceb4289c 100644 --- a/scan.c +++ b/scan.c @@ -164,7 +164,6 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* struct snapraid_block* block = &file->blockvec[i]; block_off_t block_pos = block->parity_pos; unsigned block_state; - tommy_node* j; struct snapraid_deleted* deleted; /* adjust the first free position */ @@ -176,7 +175,6 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* block_state = block_state_get(block); switch (block_state) { case BLOCK_STATE_BLK : - case BLOCK_STATE_INV : /* we keep the hash making it an "old" hash, because the parity is still containing data for it */ break; case BLOCK_STATE_CHG : @@ -204,15 +202,6 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* /* set the deleted block in the block array */ tommy_array_set(&disk->blockarr, block_pos, &deleted->block); - - /* invalidate the block of all the other disks */ - for(j=state->disklist;j!=0;j=j->next) { - struct snapraid_disk* oth_disk = j->data; - struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos); - - /* remove the parity info for this block */ - block_clear_parity(oth_block); - } } /* remove the file from the file containers */ @@ -240,7 +229,6 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* block_pos = disk->first_free_block; block_max = tommy_array_size(&disk->blockarr); for(i=0;iblockmax;++i) { - tommy_node* j; struct snapraid_block* block; /* find a free block */ @@ -269,15 +257,6 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* memcpy(file->blockvec[i].hash, block->hash, HASH_SIZE); } - /* invalidate the block of all the other disks */ - for(j=state->disklist;j!=0;j=j->next) { - struct snapraid_disk* oth_disk = j->data; - struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos); - - /* remove the parity info for this block */ - block_clear_parity(oth_block); - } - /* store in the disk map, after invalidating all the other blocks */ tommy_array_set(&disk->blockarr, block_pos, &file->blockvec[i]); } diff --git a/snapraid.1 b/snapraid.1 index f47688ab..afe4b240 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -1005,11 +1005,6 @@ BLOCK is the block position in the \(a"parity\(a" file. .PP HASH is the hash of the block. In the last block of the file, the HASH is the hash of only the used part of the block. -.SS inv BLOCK HASH -Like \(a"blk\(a", but inform that the parity of this block is invalid. -.PP -This field is used only when you interrupt manually the \(a"sync\(a" -command. .SS new BLOCK Like \(a"blk\(a", but for new allocated blocks for which the hash is not yet computed, and the stored parity doesn\(cqt take into account this diff --git a/snapraid.d b/snapraid.d index ff82ad05..4b2233f3 100644 --- a/snapraid.d +++ b/snapraid.d @@ -727,12 +727,6 @@ Content HASH is the hash of the block. In the last block of the file, the HASH is the hash of only the used part of the block. - inv BLOCK HASH - Like "blk", but inform that the parity of this block is invalid. - - This field is used only when you interrupt manually the "sync" - command. - new BLOCK Like "blk", but for new allocated blocks for which the hash is not yet computed, and the stored parity doesn't take into account this diff --git a/snapraid.txt b/snapraid.txt index 370bea59..19cf2ca2 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -810,15 +810,7 @@ BLOCK is the block position in the "parity" file. HASH is the hash of the block. In the last block of the file, the HASH is the hash of only the used part of the block. -10.7 inv BLOCK HASH -------------------- - -Like "blk", but inform that the parity of this block is invalid. - -This field is used only when you interrupt manually the "sync" -command. - -10.8 new BLOCK +10.7 new BLOCK -------------- Like "blk", but for new allocated blocks for which the hash is not @@ -828,7 +820,7 @@ new block. This field is used only when you interrupt manually the "sync" command. -10.9 chg BLOCK +10.8 chg BLOCK -------------- Like "blk", but for reallocated blocks for which the hash is not @@ -838,15 +830,15 @@ of the block. This field is used only when you interrupt manually the "sync" command. -10.10 hole DISK ---------------- +10.9 hole DISK +-------------- Defines the list of blocks that are deleted from a disk. This field is used only when you interrupt manually the "sync" command. -10.11 off BLOCK +10.10 off BLOCK --------------- Defines a block deleted from a disk, part of the last defined hole, @@ -855,22 +847,22 @@ for which the parity is computed using the previous value. This field is used only when you interrupt manually the "sync" command. -10.12 symlink DISK PATH +10.11 symlink DISK PATH ----------------------- Defines a symbolic link in the specified DISK. -10.13 hardlink DISK PATH +10.12 hardlink DISK PATH ------------------------ Defines a hard link in the specified DISK. -10.14 to PATH +10.13 to PATH ------------- Defines where the previous symlink or hardlink links to. -10.15 dir DISK PATH +10.14 dir DISK PATH ------------------- Defines an empty directory in the specified DISK. diff --git a/state.c b/state.c index d998a209..ab6274e6 100644 --- a/state.c +++ b/state.c @@ -678,7 +678,6 @@ static void state_map(struct snapraid_state* state) static void state_content_check(struct snapraid_state* state, const char* path) { tommy_node* i; - block_off_t j; /* checks that any map has different name and position */ for(i=state->maplist;i!=0;i=i->next) { @@ -696,48 +695,6 @@ static void state_content_check(struct snapraid_state* state, const char* path) } } } - - /* check the parity validity */ - for(j=0;1;++j) { /* for all the blocks */ - int at_least_one = 0; - int parity_is_valid = 0; - int parity_is_invalid = 0; - - /* for all the disks */ - for(i=state->disklist;i!=0;i=i->next) { - struct snapraid_disk* disk = i->data; - - /* if the block is present */ - block_off_t size = tommy_array_size(&disk->blockarr); - if (j < size) { - struct snapraid_block* block = tommy_array_get(&disk->blockarr, j); - - /* mark if at least one block is addressed */ - at_least_one = 1; - - if (block == BLOCK_EMPTY) { - /* not relevant */ - } else { - unsigned block_state = block_state_get(block); - if (block_state == BLOCK_STATE_BLK) { - parity_is_valid = 1; - } else { - parity_is_invalid = 1; - } - } - } - } - - /* we cannot have different blocks reporting both valid and invalid parity */ - if (parity_is_valid && parity_is_invalid) { - fprintf(stderr, "Internal inconsistency in parity validity at block %u\n", j); - exit(EXIT_FAILURE); - } - - /* stop if reached the end of all the arrays */ - if (!at_least_one) - break; - } } void state_read(struct snapraid_state* state) @@ -873,10 +830,6 @@ void state_read(struct snapraid_state* state) block_state_set(block, BLOCK_STATE_BLK); hash = oathash8(hash, 'b'); break; - case 'i' : - block_state_set(block, BLOCK_STATE_INV); - hash = oathash8(hash, 'i'); - break; case 'n' : block_state_set(block, BLOCK_STATE_NEW); hash = oathash8(hash, 'n'); @@ -1653,10 +1606,6 @@ void state_write(struct snapraid_state* state) sputsl("blk ", f); hash = oathash8(hash, 'b'); break; - case BLOCK_STATE_INV : - sputsl("inv ", f); - hash = oathash8(hash, 'i'); - break; case BLOCK_STATE_NEW : sputsl("new ", f); hash = oathash8(hash, 'n'); diff --git a/sync.c b/sync.c index 174d8a9b..3ad00051 100644 --- a/sync.c +++ b/sync.c @@ -64,24 +64,28 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari countmax = 0; for(i=blockstart;i