diff --git a/Makefile.am b/Makefile.am index fe4b8690..d4f07f82 100644 --- a/Makefile.am +++ b/Makefile.am @@ -209,7 +209,7 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check - echo --- Abort sync with additions and then delete the new additions + echo --- Abort sync with additions and then delete the new additions and sync again cp -pR test/disk1/_a test/disk1/_c cp -pR test/disk2/_a test/disk2/_c cp -pR test/disk3/_a test/disk3/_c @@ -219,53 +219,38 @@ endif rm -r test/disk3/_c $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check - echo --- Abort sync late with more additions than deletion and then delete after - cp -pR test/disk1/_a test/disk1/_c - cp -pR test/disk1/_b test/disk1/_d - mv test/disk1/_a test/_a - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-kill-after-sync sync - rm -r test/disk2/_a - rm -r test/disk2/_b - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log - -rm -r test/disk1/_a + echo --- Abort sync late with additions and then delete some unchanged stuff and fix with RAID5 + cp -pR test/disk1/_b test/disk1/_c + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf --test-kill-after-sync sync + rm -r test/disk2 + mkdir test/disk2 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log rm -r test/disk1/_c - rm -r test/disk1/_d - mv test/_a test/disk1/_a $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync - echo --- Abort sync late with more deletions than additions and then delete after + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check + echo --- Abort sync late with more deletions than additions and then delete some unchanged stuff and fix with RAID5 cp -pR test/disk1/_a test/disk1/_c mv test/disk1/_a test/_a mv test/disk1/_b test/_b $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-kill-after-sync sync - rm -r test/disk2/_a - rm -r test/disk2/_b - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log - -rm -r test/disk1/_a - -rm -r test/disk1/_b - rm -r test/disk1/_c - mv test/_a test/disk1/_a - mv test/_b test/disk1/_b - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync - echo --- Abort sync early with more additions than deletions and then delete after - cp -pR test/disk1/_a test/disk1/_c - cp -pR test/disk1/_b test/disk1/_d - mv test/disk1/_a test/_a - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync - rm -r test/disk2/_a - rm -r test/disk2/_b - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log - -rm -r test/disk1/_a - rm -r test/disk1/_c - rm -r test/disk1/_d - mv test/_a test/disk1/_a - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync - echo --- Abort sync early with more deletions than additions and then delete after - cp -pR test/disk1/_a test/disk1/_c - mv test/disk1/_a test/_a - mv test/disk1/_b test/_b - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync - rm -r test/disk2/_a - rm -r test/disk2/_b + rm -r test/disk2 + mkdir test/disk2 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log + -rm -r test/disk1/_a + -rm -r test/disk1/_b + rm -r test/disk1/_c + mv test/_a test/disk1/_a + mv test/_b test/disk1/_b + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check + echo --- Abort sync early with more deletions than additions and then delete some unchanged stuff and fix with RAID6 + cp -pR test/disk1/_a test/disk1/_c + mv test/disk1/_a test/_a + mv test/disk1/_b test/_b + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf -t 1 --test-kill-after-sync sync + rm -r test/disk2 + mkdir test/disk2 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-unrecoverable -c $(srcdir)/test-raid5.conf fix 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf fix 2>> test.log -rm -r test/disk1/_a -rm -r test/disk1/_b @@ -273,6 +258,7 @@ endif mv test/_a test/disk1/_a mv test/_b test/disk1/_b $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check echo --- Delete some files, fix with -m and check with raid5 rm test/disk1/8* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log diff --git a/check.c b/check.c index 88331e44..c81329cc 100644 --- a/check.c +++ b/check.c @@ -57,6 +57,7 @@ struct failed_struct { /** * Checks if a block hash matches the specified buffer. + * Return ==0 if equal */ static int blockcmp(struct snapraid_state* state, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero) { @@ -106,8 +107,7 @@ static int repair_step(struct snapraid_state* state, unsigned pos, unsigned disk /* if there isn't, we have to sacrifice a parity block to check that the result is correct */ has_hash = 0; for(j=0;jblock_size) == 0) { @@ -298,10 +292,22 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, } if (block_state == BLOCK_STATE_CHG) { - /* we cannot know if we recoverd the state before or after */ - /* the partial 'sync' process. */ - /* just to be safe, we assume it is garbage */ - failed[j].is_outofdate = 1; + /* if the hash is a bogus value we cannot check the result */ + /* this could happen if we have lost this information */ + /* after an aborted sync */ + if (memcmp(failed[j].block->hash, buffer_zero, HASH_SIZE) == 0) { + /* it may contain garbage */ + failed[j].is_outofdate = 1; + } else + /* if the hash is different the previous one, we are sure to have */ + /* restored it to the state after the 'sync' */ + /* instead, if the hash matches, it could be either that the */ + /* block after the sync has this hash, or that */ + /* we restored the block before the 'sync'. */ + if (blockcmp(state, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) { + /* it may contain garbage */ + failed[j].is_outofdate = 1; + } } } } @@ -322,10 +328,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, if (failed[j].is_bad) has_bad = 1; - if (failed[j].block == BLOCK_DELETED) - block_state = BLOCK_STATE_DELETED; - else - block_state = block_state_get(failed[j].block); + block_state = block_state_get(failed[j].block); if (failed[j].is_bad /* if the block is bad we don't know its content */ || block_state == BLOCK_STATE_DELETED /* we don't know the original content of deleted blocks */ @@ -435,8 +438,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, struct snapraid_block* block = BLOCK_EMPTY; if (handle[j].disk) block = disk_block_get(handle[j].disk, i); - if (block_is_valid(block) - && block_has_hash(block) /* only if the block is hashed */ + if (block_has_hash(block) /* only if the block is hashed */ && !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */ ) { one_tocheck = 1; @@ -466,8 +468,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, struct snapraid_block* block = BLOCK_EMPTY; if (handle[j].disk) block = disk_block_get(handle[j].disk, i); - if (block_is_valid(block) - && block_has_hash(block) /* only if the block is hashed */ + if (block_has_hash(block) /* only if the block is hashed */ && !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */ ) { one_tocheck = 1; @@ -509,11 +510,14 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, continue; } + /* get the state of the block */ + block_state = block_state_get(block); + /* if the block is deleted */ - if (block == BLOCK_DELETED) { + if (block_state == BLOCK_STATE_DELETED) { /* use an empty block */ memset(buffer[j], 0, state->block_size); - + /* mark the parity as invalid, and don't try to check/fix it */ /* because it will be recomputed at the next sync */ /* note that for sure there is also another block invalidating */ @@ -525,7 +529,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, failed[failed_count].is_bad = 0; failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; - failed[failed_count].block = BLOCK_DELETED; + failed[failed_count].block = block; failed[failed_count].handle = 0; ++failed_count; continue; @@ -540,9 +544,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, continue; } - /* get the state of the block */ - block_state = block_state_get(block); - /* if the parity is not valid */ if (block_state != BLOCK_STATE_BLK) { /* mark the parity as invalid, and don't try to check/fix it */ @@ -868,8 +869,9 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, if (handle[j].disk) block = disk_block_get(handle[j].disk, i); - if (!block_is_valid(block)) { - /* if no block, no file and nothing to do */ + + if (!block_has_file(block)) { + /* if no file, nothing to do */ continue; } @@ -1094,12 +1096,12 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, fprintf(stderr, "Error stating hardlink-to '%s'. %s.\n", pathto, strerror(errno)); fprintf(stderr, "hardlinkerror:%s:%s:%s: Hardlink to stat error\n", disk->name, link->sub, link->linkto); ++error; - } else if (!S_ISREG(st.st_mode)) { + } else if (!S_ISREG(stto.st_mode)) { failed = 1; fprintf(stderr, "hardlinkerror:%s:%s:%s: Hardlink-to error for not regular file\n", disk->name, link->sub, link->linkto); ++error; - } else if (st.st_ino != stto.st_ino) { + } else if (!failed && st.st_ino != stto.st_ino) { failed = 1; fprintf(stderr, "Mismatch hardlink '%s' and '%s'. Different inode.\n", path, pathto); diff --git a/dry.c b/dry.c index 49465166..9b9c1faf 100644 --- a/dry.c +++ b/dry.c @@ -57,8 +57,11 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit if (handle[j].disk) block = disk_block_get(handle[j].disk, i); - if (!block_is_valid(block)) + + if (!block_has_file(block)) { + /* if no file, nothing to do */ continue; + } /* if the file is different than the current one, close it */ if (handle[j].file != block_file_get(block)) { diff --git a/elem.c b/elem.c index fe68d609..291ea4ce 100644 --- a/elem.c +++ b/elem.c @@ -295,6 +295,36 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size) return block_size; } +struct snapraid_deleted* deleted_alloc(void) +{ + struct snapraid_deleted* deleted; + + deleted = malloc_nofail(sizeof(struct snapraid_deleted)); + + /* set the state as deleted */ + deleted->block.file_mixed = BLOCK_STATE_DELETED; + + return deleted; +} + +struct snapraid_deleted* deleted_dup(struct snapraid_block* block) +{ + struct snapraid_deleted* deleted; + + deleted = deleted_alloc(); + + /* copy data from deleted block */ + deleted->block.parity_pos = block->parity_pos; + memcpy(deleted->block.hash, block->hash, HASH_SIZE); + + return deleted; +} + +void deleted_free(struct snapraid_deleted* deleted) +{ + free(deleted); +} + struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode) { struct snapraid_file* file; @@ -415,6 +445,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev disk->device = dev; disk->first_free_block = 0; tommy_list_init(&disk->filelist); + tommy_list_init(&disk->deletedlist); tommy_hashdyn_init(&disk->inodeset); tommy_hashdyn_init(&disk->pathset); tommy_list_init(&disk->linklist); @@ -429,6 +460,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev void disk_free(struct snapraid_disk* disk) { tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free); + tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)deleted_free); tommy_hashdyn_done(&disk->inodeset); tommy_hashdyn_done(&disk->pathset); tommy_list_foreach(&disk->linklist, (tommy_foreach_func*)link_free); diff --git a/elem.h b/elem.h index c3e21398..2a13ffbb 100644 --- a/elem.h +++ b/elem.h @@ -72,45 +72,63 @@ struct snapraid_file; * This is the normal state of a saved block. * Note that if exists at least one BLOCK_STATE_BLK in a disk, all the other * blocks at the same address in other disks can be only BLOCK_EMPTY or BLOCK_STATE_BLK. + * + * The block hash field IS set. + * The parity for this disk is updated. */ -#define BLOCK_STATE_BLK 0 +#define BLOCK_STATE_BLK 1 /** * The block is new and not yet hashed, and it's using a space previously empty. + * + * The block hash field IS NOT set. + * The parity for this disk is not updated, but it contains 0. */ -#define BLOCK_STATE_NEW 1 +#define BLOCK_STATE_NEW 2 /** * The block has the hash computed, but the parity is invalid. - * This happens when the parity is invalidated by a change in another block. + * This happens when the parity is invalidated by a change in another disk. + * + * The block hash field IS set. + * The parity for this disk is updated, but there is at least another disk not updated parity. */ -#define BLOCK_STATE_INV 2 +#define BLOCK_STATE_INV 3 /** * The block is new and not yet hashed, and it's using the space of a block not existing anymore. * This happens when a new block overwrite a just removed block. + * + * The block hash field IS set, and it represents the hash of the previous data. + * The parity for this disk is not updated, but it contains the old data referenced by the hash. + * + * If the hash is completely filled with 0, it means that its lost. */ -#define BLOCK_STATE_CHG 3 +#define BLOCK_STATE_CHG 4 + +/** + * This block is a deleted one. + * This happens when a file is deleted. + * + * The block hash field IS set, and it represents the hash of the previous data. + * The parity for this disk is not updated, but it contains the old data referenced by the hash. + * + * If the hash is completely filled with 0, it means that its lost. + */ +#define BLOCK_STATE_DELETED 5 /** * Mask used to store the previous states of the block. * Used to mix them inside the file pointer. */ -#define BLOCK_STATE_MASK 3 - -/** - * This block is a deleted one. - * Note that this state cannot be stored inside the block, - * but it's usually represented by the block ::BLOCK_DELETED. - */ -#define BLOCK_STATE_DELETED 4 +#define BLOCK_STATE_MASK 7 /** * This block is an empty one. * Note that this state cannot be stored inside the block, - * but it's usually represented by the block ::BLOCK_EMPTY. + * and it's represented by the block ::BLOCK_EMPTY. */ -#define BLOCK_STATE_EMPTY 5 +#define BLOCK_STATE_EMPTY 8 /** * Block of a file. @@ -126,11 +144,6 @@ struct snapraid_block { */ #define BLOCK_EMPTY 0 -/** - * Block pointer used to mark deleted blocks. - */ -#define BLOCK_DELETED ((struct snapraid_block*)-1) - #define FILE_IS_PRESENT 1 /**< If it's seen as present. */ #define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */ #define FILE_IS_LARGER 4 /**< If a larger file was already detected. Just to avoid to report it more times. */ @@ -188,6 +201,21 @@ struct snapraid_dir { tommy_hashdyn_node nodeset; }; +/** + * Deleted entry. + */ +struct snapraid_deleted { + /** + * Deleted block. + * This block is always in state BLOCK_STATE_DELETED, + * and it's used to keep the old hash during a sync. + */ + struct snapraid_block block; + + /* nodes for data structures */ + tommy_node node; +}; + /** * Disk. */ @@ -196,14 +224,33 @@ struct snapraid_disk { char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */ uint64_t device; /**< Device identifier. */ block_off_t first_free_block; /**< First free searching block. */ - tommy_list filelist; /**< List of all the files. */ + + /**< + * Block array of the disk. + * + * Each element points to a snapraid_block structure, or it's BLOCK_EMPTY. + */ + tommy_array blockarr; + + /** + * List of all the snapraid_file for the disk. + */ + tommy_list filelist; + + /** + * List of all the snapraid_deleted blocks for the disk. + * + * These files are kept allocated, because the blocks are still referenced in + * the ::blockarr. + */ + tommy_list deletedlist; + tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */ tommy_hashdyn pathset; /**< Hashtable by path of all the files. */ tommy_list linklist; /**< List of all the links. */ tommy_hashdyn linkset; /**< Hashtable by name of all the links. */ tommy_list dirlist; /**< List of all the dirs. */ tommy_hashdyn dirset; /**< Hashtable by name of all the dirs. */ - tommy_array blockarr; /**< Block array of the disk. */ /* nodes for data structures */ tommy_node node; @@ -297,17 +344,39 @@ static inline struct snapraid_file* block_file_get(struct snapraid_block* block) */ static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file) { - block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | (uintptr_t)file; + uintptr_t ptr = (uintptr_t)file; + + /* ensure that the pointer doesn't use the flag space */ + if ((ptr & (uintptr_t)BLOCK_STATE_MASK) != 0) { + fprintf(stderr, "Internal error for pointer not aligned\n"); + exit(EXIT_FAILURE); + } + + block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | ptr; } +/** + * Get the state of the block. + * + * For this function, it's allowed to pass a NULL block + * pointer than results in the BLOCK_STATE_EMPTY state. + */ static inline unsigned block_state_get(const struct snapraid_block* block) { + if (block == BLOCK_EMPTY) + return BLOCK_STATE_EMPTY; + return block->file_mixed & BLOCK_STATE_MASK; } static inline void block_state_set(struct snapraid_block* block, unsigned state) { - assert(state != BLOCK_STATE_EMPTY && state != BLOCK_STATE_DELETED); + /* ensure that the state can be stored inside the file pointer */ + if ((state & BLOCK_STATE_MASK) != state) { + fprintf(stderr, "Internal error when setting the block state %u\n", state); + exit(EXIT_FAILURE); + } + block->file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK; block->file_mixed |= state & BLOCK_STATE_MASK; } @@ -315,17 +384,36 @@ static inline void block_state_set(struct snapraid_block* block, unsigned state) static inline void block_clear_parity(struct snapraid_block* block) { unsigned state = block_state_get(block); + if (state == BLOCK_STATE_BLK) { block_state_set(block, BLOCK_STATE_INV); } } +/** + * Check if the specified block has a valid and updated hash. + * + * Note that blocks with hash of old data, like CHG and DELETED ones, are not considered. + */ static inline int block_has_hash(const struct snapraid_block* block) { unsigned state = block_state_get(block); + return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV; } +/** + * Check if the specified block has a valid file. + * + * It includes BLK/INV/NEW/CHG and excludes EMPTY/DELETED. + */ +static inline int block_has_file(const struct snapraid_block* block) +{ + unsigned state = block_state_get(block); + + return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV || state == BLOCK_STATE_NEW || state == BLOCK_STATE_CHG; +} + /** * Gets the relative position of a block inside the file. */ @@ -336,20 +424,27 @@ block_off_t block_file_pos(struct snapraid_block* block); */ int block_is_last(struct snapraid_block* block); -/** - * Checks if the block is a real allocated block. - */ -static inline int block_is_valid(struct snapraid_block* block) -{ - return block != BLOCK_EMPTY && block != BLOCK_DELETED; -} - /** * Gets the size in bytes of the block. * If it's the last block of a file it could be less than block_size. */ unsigned block_file_size(struct snapraid_block* block, unsigned block_size); +/** + * Allocates a deleted block. + */ +struct snapraid_deleted* deleted_alloc(void); + +/** + * Allocates a deleted block from a real one. + */ +struct snapraid_deleted* deleted_dup(struct snapraid_block* block); + +/** + * Frees a deleted block. + */ +void deleted_free(struct snapraid_deleted* deleted); + static inline int file_flag_has(const struct snapraid_file* file, unsigned mask) { return (file->flag & mask) == mask; diff --git a/maketest.sh b/maketest.sh index bed42353..2ce3c392 100755 --- a/maketest.sh +++ b/maketest.sh @@ -11,7 +11,7 @@ if ! make lcov_reset check lcov_capture lcov_html; then exit 1 fi -rm -r cov_release +rm -rf cov_release mv cov cov_release if ! make distclean; then diff --git a/parity.c b/parity.c index b570c2f2..7a0bbec0 100644 --- a/parity.c +++ b/parity.c @@ -37,10 +37,10 @@ block_off_t parity_size(struct snapraid_state* state) /* start from the declared size */ block_off_t block = tommy_array_size(&disk->blockarr); - /* decrease the block until an allocated one */ + /* decrease the block until an used one */ /* we don't stop at deleted blocks, because we want to have them cleared */ /* if they are at the end of the parity */ - while (block > parity_block && !block_is_valid(tommy_array_get(&disk->blockarr, block - 1))) + while (block > parity_block && !block_has_file(tommy_array_get(&disk->blockarr, block - 1))) --block; /* get the highest value */ diff --git a/scan.c b/scan.c index c7aab6e8..3de027f8 100644 --- a/scan.c +++ b/scan.c @@ -161,25 +161,57 @@ static void scan_file_remove(struct snapraid_state* state, struct snapraid_disk* /* free all the blocks of the file */ for(i=0;iblockmax;++i) { - block_off_t block_pos = file->blockvec[i].parity_pos; + struct snapraid_block* block = &file->blockvec[i]; + block_off_t block_pos = block->parity_pos; + unsigned block_state; tommy_node* j; + struct snapraid_deleted* deleted; /* adjust the first free position */ if (disk->first_free_block > block_pos) disk->first_free_block = block_pos; - /* set the block as deleted */ - tommy_array_set(&disk->blockarr, block_pos, BLOCK_DELETED); + /* in case we scan after an aborted sync, */ + /* we could get also intermediate states like inv/chg/new */ + block_state = block_state_get(block); + switch (block_state) { + case BLOCK_STATE_BLK : + case BLOCK_STATE_INV : + /* we keep the hash making it an "old" hash, because the parity is still containing data for it */ + break; + case BLOCK_STATE_CHG : + case BLOCK_STATE_NEW : + /* in these cases we don't know if the old state is still the one */ + /* stored inside the parity, because after an aborted sync, the parity */ + /* may be or may be not have been updated with the new data */ + /* Them we reset the hash to a bogus value */ + /* Note that this condition is possible only if: */ + /* - new files added/modified */ + /* - aborted sync, without saving the content file */ + /* - files deleted after the aborted sync */ + memset(block->hash, 0, HASH_SIZE); + break; + default: + fprintf(stderr, "Internal state inconsistency in scanning for block %u state %u\n", block->parity_pos, block_state); + exit(EXIT_FAILURE); + } + + /* allocated a new deleted block from the block we are going to delete */ + deleted = deleted_dup(block); + + /* insert it in the list of deleted blocks */ + tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); + + /* set the deleted block in the block array */ + tommy_array_set(&disk->blockarr, block_pos, &deleted->block); /* invalidate the block of all the other disks */ for(j=state->disklist;j!=0;j=j->next) { struct snapraid_disk* oth_disk = j->data; struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos); - if (block_is_valid(oth_block)) { - /* remove the parity info for this block */ - block_clear_parity(oth_block); - } + /* remove the parity info for this block */ + block_clear_parity(oth_block); } } @@ -209,9 +241,10 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* block_max = tommy_array_size(&disk->blockarr); for(i=0;iblockmax;++i) { tommy_node* j; + struct snapraid_block* block; /* find a free block */ - while (block_pos < block_max && block_is_valid(tommy_array_get(&disk->blockarr, block_pos))) + while (block_pos < block_max && block_has_file(tommy_array_get(&disk->blockarr, block_pos))) ++block_pos; /* if not found, allocate a new one */ @@ -223,21 +256,26 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk* /* set the position */ file->blockvec[i].parity_pos = block_pos; - /* set the state depending if the used block was EMPTY or DELETED */ - if (tommy_array_get(&disk->blockarr, block_pos) == BLOCK_EMPTY) + /* block to overwrite */ + block = tommy_array_get(&disk->blockarr, block_pos); + + /* if the block is an empty one */ + if (block == BLOCK_EMPTY) { + /* we just overwrite it with a NEW one */ block_state_set(&file->blockvec[i], BLOCK_STATE_NEW); - else + } else { + /* otherwise it's a DELETED one, that we convert in CHG keeping the hash */ block_state_set(&file->blockvec[i], BLOCK_STATE_CHG); + memcpy(file->blockvec[i].hash, block->hash, HASH_SIZE); + } /* invalidate the block of all the other disks */ for(j=state->disklist;j!=0;j=j->next) { struct snapraid_disk* oth_disk = j->data; struct snapraid_block* oth_block = disk_block_get(oth_disk, block_pos); - if (block_is_valid(oth_block)) { - /* remove the parity info for this block */ - block_clear_parity(oth_block); - } + /* remove the parity info for this block */ + block_clear_parity(oth_block); } /* store in the disk map, after invalidating all the other blocks */ diff --git a/state.c b/state.c index 034b3d21..d998a209 100644 --- a/state.c +++ b/state.c @@ -717,11 +717,9 @@ static void state_content_check(struct snapraid_state* state, const char* path) if (block == BLOCK_EMPTY) { /* not relevant */ - } else if (block == BLOCK_DELETED) { - parity_is_invalid = 1; } else { - unsigned state = block_state_get(block); - if (state == BLOCK_STATE_BLK) { + unsigned block_state = block_state_get(block); + if (block_state == BLOCK_STATE_BLK) { parity_is_valid = 1; } else { parity_is_invalid = 1; @@ -892,8 +890,8 @@ void state_read(struct snapraid_state* state) block->parity_pos = v_pos; hash = oathash32(hash, v_pos); - /* read the hash only for some state */ - if (tag[0] == 'b' || tag[0] == 'i') { + /* read the hash only for 'blk/inv/chg', and not for 'new' */ + if (tag[0] != 'n') { c = sgeteol(f); if (c != ' ') { fprintf(stderr, "Invalid 'blk' specification in '%s' at line %u\n", path, line); @@ -909,9 +907,15 @@ void state_read(struct snapraid_state* state) hash = oathashm(hash, block->hash, HASH_SIZE); } + /* we must not overwrite existing blocks */ + if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + fprintf(stderr, "Internal inconsistency for 'blk' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + /* insert the block in the block array */ - tommy_array_grow(&disk->blockarr, block->parity_pos + 1); - tommy_array_set(&disk->blockarr, block->parity_pos, block); + tommy_array_grow(&disk->blockarr, v_pos + 1); + tommy_array_set(&disk->blockarr, v_pos, block); /* check for termination of the block list */ ++blockidx; @@ -925,6 +929,7 @@ void state_read(struct snapraid_state* state) } else if (strcmp(tag, "off") == 0) { /* "off" command */ block_off_t v_pos; + struct snapraid_deleted* deleted; hash = oathash8(hash, 'o'); @@ -933,24 +938,45 @@ void state_read(struct snapraid_state* state) exit(EXIT_FAILURE); } + /* allocate a new deleted block */ + deleted = deleted_alloc(); + + /* insert it in the list of deleted blocks */ + tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); + ret = sgetu32(f, &v_pos); if (ret < 0) { fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } + deleted->block.parity_pos = v_pos; hash = oathash32(hash, v_pos); - /* insert the block in the block array */ - tommy_array_grow(&disk->blockarr, v_pos + 1); + /* read the hash */ + c = sgeteol(f); + if (c != ' ') { + fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } - /* deleted block must be empty blocks */ - if (tommy_array_get(&disk->blockarr, v_pos) != 0) { + /* set the hash only if present */ + ret = sgethex(f, deleted->block.hash, HASH_SIZE); + if (ret < 0) { + fprintf(stderr, "Invalid 'off' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + hash = oathashm(hash, deleted->block.hash, HASH_SIZE); + + /* we must not overwrite existing blocks */ + if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { fprintf(stderr, "Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); } - tommy_array_set(&disk->blockarr, v_pos, BLOCK_DELETED); + /* insert the block in the block array */ + tommy_array_grow(&disk->blockarr, v_pos + 1); + tommy_array_set(&disk->blockarr, v_pos, &deleted->block); } else if (strcmp(tag, "file") == 0) { /* file */ char sub[PATH_MAX]; @@ -1639,12 +1665,15 @@ void state_write(struct snapraid_state* state) sputsl("chg ", f); hash = oathash8(hash, 'g'); break; + default: + fprintf(stderr, "Internal state inconsistency in saving for block %u state %u\n", block->parity_pos, block_state); + exit(EXIT_FAILURE); } sputu32(block->parity_pos, f); hash = oathash32(hash, block->parity_pos); - if (block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_INV) { + if (block_state != BLOCK_STATE_NEW) { sputc(' ', f); sputhex(block->hash, HASH_SIZE, f); hash = oathashm(hash, block->hash, HASH_SIZE); @@ -1712,6 +1741,7 @@ void state_write(struct snapraid_state* state) ++count_dir; } + /* save all the delete blocks */ { block_off_t k; block_off_t blockmax; @@ -1723,7 +1753,7 @@ void state_write(struct snapraid_state* state) /* for each deleted block in the disk */ for(k=0;kblockarr, k); - if (block == BLOCK_DELETED) { + if (block_state_get(block) == BLOCK_STATE_DELETED) { if (first_deleted) { first_deleted = 0; @@ -1744,6 +1774,10 @@ void state_write(struct snapraid_state* state) sputu32(k, f); hash = oathash32(hash, k); + sputc(' ', f); + sputhex(block->hash, HASH_SIZE, f); + hash = oathashm(hash, block->hash, HASH_SIZE); + sputeol(f); if (serror(f)) { fprintf(stderr, "Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); @@ -1965,9 +1999,9 @@ int state_progress(struct snapraid_state* state, block_off_t blockpos, block_off if (delta > 5 && countpos > 0) { unsigned m, h; - data_off_t todo = countmax - countpos; + data_off_t to_do = countmax - countpos; - m = todo * delta / (60 * countpos); + m = to_do * delta / (60 * countpos); h = m / 60; m = m % 60; diff --git a/sync.c b/sync.c index 3ec5a317..174d8a9b 100644 --- a/sync.c +++ b/sync.c @@ -71,7 +71,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari struct snapraid_block* block = BLOCK_EMPTY; if (handle[j].disk) block = disk_block_get(handle[j].disk, i); - if (block_is_valid(block) + if (block_has_file(block) && block_state_get(block) != BLOCK_STATE_BLK ) { one_invalid = 1; @@ -107,7 +107,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari struct snapraid_block* block = BLOCK_EMPTY; if (handle[j].disk) block = disk_block_get(handle[j].disk, i); - if (block_is_valid(block) + + /* if the block is used, and it needs some update */ + if (block_has_file(block) && block_state_get(block) != BLOCK_STATE_BLK ) { one_invalid = 1; @@ -117,12 +119,14 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* if no invalid block skip */ if (!one_invalid) { - /* cleanup all the deleted blocks */ + /* cleanup all the deleted blocks at this position */ for(j=0;jblockarr, i, BLOCK_EMPTY); @@ -155,9 +159,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari continue; } - /* if the disk block is not used */ + /* if the block is not used */ block = disk_block_get(handle[j].disk, i); - if (!block_is_valid(block)) { + if (!block_has_file(block)) { /* use an empty block */ memset(buffer[j], 0, state->block_size); continue; @@ -290,8 +294,9 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari continue; } - if (block == BLOCK_DELETED) { - /* the parity is not updated without this block, so it's now empty */ + /* if it's a deleted block */ + if (block_state_get(block) == BLOCK_STATE_DELETED) { + /* the parity is now updated without this block, so it's now empty */ tommy_array_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY); continue; } @@ -370,14 +375,17 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t loaded_size = state->loaded_blockmax * (data_off_t)state->block_size; /* remove all the deleted blocks over the new parity size */ + /* we know for sure that after the parity size, there is no used block */ + /* and then it's safe to lose any deleted block info */ for(i=state->disklist;i!=0;i=i->next) { struct snapraid_disk* disk = i->data; block_off_t diskblockmax = tommy_array_size(&disk->blockarr); - block_off_t block; + block_off_t block_pos; - for(block=blockmax;blockblockarr, block) == BLOCK_DELETED) { - tommy_array_set(&disk->blockarr, block, BLOCK_EMPTY); + for(block_pos=blockmax;block_posblockarr, block_pos); + if (block_state_get(block) == BLOCK_STATE_DELETED) { + tommy_array_set(&disk->blockarr, block_pos, BLOCK_EMPTY); /* mark the state as needing write */ state->need_write = 1;