From 0407c945db43d66ce069a2a77b8a1b8d879743ea Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2015 18:19:48 +0200 Subject: [PATCH] Change the interface for parity allocation This will allow a more compact representation of the parity allocation --- TODO | 12 ++ cmdline/check.c | 6 +- cmdline/dry.c | 2 +- cmdline/dup.c | 6 +- cmdline/elem.c | 106 ++++++++++- cmdline/elem.h | 62 +++++-- cmdline/import.c | 6 +- cmdline/import.h | 2 +- cmdline/parity.c | 13 +- cmdline/scan.c | 428 +++++++++++++++++++++++---------------------- cmdline/scrub.c | 2 +- cmdline/snapraid.c | 22 ++- cmdline/state.c | 44 ++--- cmdline/status.c | 12 +- cmdline/sync.c | 12 +- 15 files changed, 449 insertions(+), 286 deletions(-) diff --git a/TODO b/TODO index 6961e525..07fdef35 100644 --- a/TODO +++ b/TODO @@ -56,6 +56,18 @@ Very fast to load from disk as we insert only at the end. + We can also get rid of deleted blocks, as the hash will be stored anyway. + We can put the state into the hash, dropping 3 bits of the hash of the 128 ones. +- A problem is that we are going to move hash from file to parity. +How to keep the hash of the file when the file is not in the parity ? +We'll have to allocate hash also in the file, and move data. +We can keep the hash ALWAYS on the file! +But how to handle DELETED blocks ? At now we handle them one-by-one, +We have to change to handle them like files, to allow to have DELETED files, inside the tree. +Here the problem is how to handle when DELETED blocks are overwritten. +Note that we need the UNDELETE state to allow to recover in FIX. +The HASH is also used to search for DELETED blocks. + +TODO CONVERT DELETED BLOCKS TO DELETE FILES + * Restore ownership and permissions, at least in Unix. * Restore directory timestamps. diff --git a/cmdline/check.c b/cmdline/check.c index 1aa74ea3..c99bf893 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -619,7 +619,7 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s continue; } - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (!block_has_file(block)) { /* if no file, nothing to do */ continue; @@ -819,7 +819,7 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct if (!handle[j].disk) continue; - block = disk_block_get(handle[j].disk, i); + block = fs_par2block_get(handle[j].disk, i); /* try to recover all files, even the ones without hash */ /* because in some cases we can recover also them */ @@ -943,7 +943,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } /* if the disk block is not used */ - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (block == BLOCK_EMPTY) { /* use an empty block */ memset(buffer[j], 0, state->block_size); diff --git a/cmdline/dry.c b/cmdline/dry.c index 8c2b0815..aaa32349 100644 --- a/cmdline/dry.c +++ b/cmdline/dry.c @@ -67,7 +67,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit continue; } - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (!block_has_file(block)) { /* if no file, nothing to do */ diff --git a/cmdline/dup.c b/cmdline/dup.c index c7de7497..cb148f64 100644 --- a/cmdline/dup.c +++ b/cmdline/dup.c @@ -50,9 +50,11 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { - memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE); + struct snapraid_block* block = fs_file2block_get(disk, file, i); - if (!block_has_updated_hash(&file->blockvec[i])) { + memcpy(buf + i * HASH_SIZE, block->hash, HASH_SIZE); + + if (!block_has_updated_hash(block)) { free(buf); free(hash); return 0; diff --git a/cmdline/elem.c b/cmdline/elem.c index b79076a6..a101233c 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -309,7 +309,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub) return 1; } -int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file) +int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file) { unsigned i; @@ -318,12 +318,8 @@ int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapr /* check each block of the file */ for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block; - snapraid_info info; - - block = &file->blockvec[i]; - - info = info_get(infoarr, block->parity_pos); + block_off_t parity_pos = fs_file2par_get(disk, file, i); + snapraid_info info = info_get(infoarr, parity_pos); /* if the file has a bad block, don't exclude it */ if (info_get_bad(info)) @@ -444,7 +440,6 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ file->flag = 0; file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); - /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { file->blockvec[i].parity_pos = POS_INVALID; file->blockvec[i].file_mixed = 0; @@ -460,10 +455,40 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ return file; } +struct snapraid_file* file_dup(struct snapraid_file* copy) +{ + struct snapraid_file* file; + block_off_t i; + + file = malloc_nofail(sizeof(struct snapraid_file)); + file->sub = strdup_nofail(copy->sub); + file->size = copy->size; + file->blockmax = copy->blockmax; + file->mtime_sec = copy->mtime_sec; + file->mtime_nsec = copy->mtime_nsec; + file->inode = copy->inode; + file->physical = copy->physical; + file->flag = copy->flag; + file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); + + for (i = 0; i < file->blockmax; ++i) { + file->blockvec[i].file_mixed = copy->blockvec[i].file_mixed; + file->blockvec[i].parity_pos = copy->blockvec[i].parity_pos; + memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE); + + /* set the file to overwrite the copied one */ + block_file_set(&file->blockvec[i], file); + } + + return file; +} + void file_free(struct snapraid_file* file) { free(file->sub); + file->sub = 0; free(file->blockvec); + file->blockvec = 0; free(file); } @@ -739,6 +764,71 @@ void disk_free(struct snapraid_disk* disk) free(disk); } +block_off_t disk_block_size(struct snapraid_disk* disk) +{ + return tommy_arrayblk_size(&disk->blockarr); +} + +void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos) +{ + (void)disk; + + assert(file_pos < file->blockmax); + + file->blockvec[file_pos].parity_pos = parity_pos; +} + +struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) +{ + struct snapraid_block* block = fs_par2block_get(disk, parity_pos); + + if (block == BLOCK_EMPTY) + return 0; + + if (file_pos) + *file_pos = block_file_pos(block); + + return block_file_get(block); +} + +block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) +{ + (void)disk; + + assert(file_pos < file->blockmax); + + return file->blockvec[file_pos].parity_pos; +} + +struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) +{ + (void)disk; + + assert(file_pos < file->blockmax); + + return &file->blockvec[file_pos]; +} + +void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos) +{ + tommy_arrayblk_grow(&disk->blockarr, pos + 1); + tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY); +} + +void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block) +{ + tommy_arrayblk_grow(&disk->blockarr, pos + 1); + tommy_arrayblk_set(&disk->blockarr, pos, block); +} + +struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos) +{ + if (parity_pos < tommy_arrayblk_size(&disk->blockarr)) + return tommy_arrayblk_get(&disk->blockarr, parity_pos); + else + return BLOCK_EMPTY; +} + int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) { block_off_t i; diff --git a/cmdline/elem.h b/cmdline/elem.h index 4329c42f..55864715 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -465,7 +465,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub); * Filter a file if bad. * Return !=0 if the file is correct and it should be excluded. */ -int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file); +int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file); /** * Filters a dir using a list of filters. @@ -503,7 +503,7 @@ static inline void block_file_set(struct snapraid_block* block, struct snapraid_ /* LCOV_EXCL_STOP */ } - block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | ptr; + block->file_mixed = (block->file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr; } /** @@ -700,12 +700,17 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask) } /** - * Allocates a file. + * Allocate a file. */ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical); /** - * Deallocates a file. + * Duplicate a file. + */ +struct snapraid_file* file_dup(struct snapraid_file* copy); + +/** + * Deallocate a file. */ void file_free(struct snapraid_file* file); @@ -884,16 +889,47 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev void disk_free(struct snapraid_disk* disk); /** - * Gets a specific block of a disk. - * Returns 0 if the block is over the end of the disk or not used. + * Get the size of the disk in blocks. */ -static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, block_off_t pos) -{ - if (pos < tommy_arrayblk_size(&disk->blockarr)) - return tommy_arrayblk_get(&disk->blockarr, pos); - else - return BLOCK_EMPTY; -} +block_off_t disk_block_size(struct snapraid_disk* disk); + +/** + * Set a filesystem mapping between file and parity positions. + */ +void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos); + +/** + * Get the file position from the parity position. + * Return 0 if no file is using it. + */ +struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos); + +/** + * Get the parity position from the file position. + */ +block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos); + +/** + * Get the block from the file position. + */ +struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos); + +/** + * Clear a specific block. + */ +void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos); + +/** + * Set the block from parity position. + * In case of holes, everything is initialized to 0. + */ +void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block); + +/** + * Get the block from the parity position. + * Returns BLOCK_EMPTY==0 if the block is over the end of the disk or not used. + */ +struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos); /** * Check if a disk is totally empty and can be discarded from the content file. diff --git a/cmdline/import.c b/cmdline/import.c index 3f435bd3..86ee8be3 100644 --- a/cmdline/import.c +++ b/cmdline/import.c @@ -66,7 +66,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t file->path = strdup_nofail(path); file->size = size; file->blockmax = (size + block_size - 1) / block_size; - file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block)); + file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block)); buffer = malloc_nofail(block_size); @@ -98,7 +98,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t offset = 0; for (i = 0; i < file->blockmax; ++i) { - struct snapraid_import_block* block = &file->blockvec[i]; + struct snapraid_import_block* block = &file->blockimp[i]; unsigned read_size = block_size; if (read_size > size) read_size = size; @@ -145,7 +145,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t void import_file_free(struct snapraid_import_file* file) { free(file->path); - free(file->blockvec); + free(file->blockimp); free(file); } diff --git a/cmdline/import.h b/cmdline/import.h index d60af8cd..f397875e 100644 --- a/cmdline/import.h +++ b/cmdline/import.h @@ -46,7 +46,7 @@ struct snapraid_import_block { */ struct snapraid_import_file { data_off_t size; /**< Size of the file. */ - struct snapraid_import_block* blockvec; /**< All the blocks of the file. */ + struct snapraid_import_block* blockimp; /**< All the blocks of the file. */ block_off_t blockmax; /**< Number of blocks. */ char* path; /**< Full path of the file. */ diff --git a/cmdline/parity.c b/cmdline/parity.c index 902be1af..b7617669 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -37,12 +37,12 @@ block_off_t parity_allocated_size(struct snapraid_state* state) struct snapraid_disk* disk = i->data; /* start from the declared size */ - block_off_t block = tommy_arrayblk_size(&disk->blockarr); + block_off_t block = disk_block_size(disk); /* decrease the block until an allocated one, but part of a file */ /* we don't stop at deleted blocks, because we want to have them cleared */ /* if they are at the end of the parity */ - while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1))) + while (block > parity_block && !block_has_file(fs_par2block_get(disk, block - 1))) --block; /* get the highest value */ @@ -64,10 +64,10 @@ block_off_t parity_used_size(struct snapraid_state* state) struct snapraid_disk* disk = i->data; /* start from the declared size */ - block_off_t block = tommy_arrayblk_size(&disk->blockarr); + block_off_t block = disk_block_size(disk); /* decrease the block until an used one */ - while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1))) + while (block > parity_block && !block_has_file_and_valid_parity(fs_par2block_get(disk, block - 1))) --block; /* get the highest value */ @@ -95,7 +95,7 @@ int parity_is_invalid(struct snapraid_state* state) one_valid = 0; for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) { struct snapraid_disk* disk = node_disk->data; - struct snapraid_block* block = disk_block_get(disk, i); + struct snapraid_block* block = fs_par2block_get(disk, i); if (block_has_file(block)) one_valid = 1; @@ -134,7 +134,8 @@ void parity_overflow(struct snapraid_state* state, data_off_t size) struct snapraid_file* file = j->data; if (file->blockmax > 0) { - if (file->blockvec[file->blockmax - 1].parity_pos >= blockalloc) { + block_off_t parity_pos = fs_file2par_get(disk, file, file->blockmax - 1); + if (parity_pos >= blockalloc) { log_tag("outofparity:%s:%s\n", disk->name, esc(file->sub)); if (first) { first = 0; diff --git a/cmdline/scan.c b/cmdline/scan.c index ab7d2751..0c72fda6 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -155,22 +155,116 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, } /** - * Removes the specified file from the parity. + * Insert the specified file in the parity. + */ +static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file) +{ + struct snapraid_state* state = scan->state; + struct snapraid_disk* disk = scan->disk; + block_off_t i; + block_off_t parity_pos; + + /* state changed */ + state->need_write = 1; + + /* allocate the blocks of the file */ + parity_pos = disk->first_free_block; + for (i = 0; i < file->blockmax; ++i) { + snapraid_info info; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + + /* increment the position until the first really free block */ + while (block_has_file(fs_par2block_get(disk, parity_pos))) + ++parity_pos; + + /* set the position */ + block->parity_pos = parity_pos; + + /* get block specific info */ + info = info_get(&state->infoarr, parity_pos); + + /* if the file block already has an updated hash without rehash */ + if (block_has_updated_hash(block) && !info_get_rehash(info)) { + /* the only possible case is for REP blocks */ + assert(block_state_get(block) == BLOCK_STATE_REP); + + /* convert to a REP block */ + block_state_set(block, BLOCK_STATE_REP); + + /* and keep the hash as it's */ + } else { + struct snapraid_block* over_block; + unsigned block_state; + + /* convert to a CHG block */ + block_state_set(block, BLOCK_STATE_CHG); + + /* block to overwrite */ + over_block = fs_par2block_get(disk, parity_pos); + + /* state of the block we are going to overwrite */ + block_state = block_state_get(over_block); + + /* if the block is an empty one */ + if (block_state == BLOCK_STATE_EMPTY) { + /* the block was empty and filled with zeros */ + /* set the hash to the special ZERO value */ + hash_zero_set(block->hash); + } else { + /* otherwise it's a DELETED one */ + assert(block_state == BLOCK_STATE_DELETED); + + /* copy the past hash of the block */ + memcpy(block->hash, over_block->hash, HASH_SIZE); + + /* if we have not already cleared the past hash */ + if (!state->clear_past_hash) { + /* in this case we don't know if the old state is still the one */ + /* stored inside the parity, because after an aborted sync, the parity */ + /* may be or may be not have been updated with the new data */ + /* Then we reset the hash to a bogus value */ + /* For example: */ + /* - One file is deleted */ + /* - Sync aborted after, updating the parity to the new state, */ + /* but without saving the content file representing this new state. */ + /* - Another file is added again (exactly here) */ + /* with the hash of DELETED block not represeting the real parity state */ + hash_invalid_set(block->hash); + } + } + } + + /* store in the disk map, after invalidating all the other blocks */ + fs_par2block_set(disk, parity_pos, block); + + /* set the new free position */ + disk->first_free_block = parity_pos + 1; + } + + /* insert in the list of contained files */ + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); +} + +/** + * Remove the specified file from the parity. + * The file is then inserted in the deleted set, and it should no be deallocated. */ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file) { struct snapraid_state* state = scan->state; struct snapraid_disk* disk = scan->disk; - block_off_t i; + /* remove from the list of contained files */ + tommy_list_remove_existing(&disk->filelist, &file->nodelist); + /* state changed */ state->need_write = 1; /* free all the blocks of the file */ for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = &file->blockvec[i]; - block_off_t block_pos = block->parity_pos; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + block_off_t parity_pos = fs_file2par_get(disk, file, i); unsigned block_state; struct snapraid_deleted* deleted; @@ -178,8 +272,8 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil /* note that doing all the deletions before alllocations, */ /* first_free_block is always 0 and the "if" is never triggered */ /* but we keep this code anyway for completeness. */ - if (disk->first_free_block > block_pos) - disk->first_free_block = block_pos; + if (disk->first_free_block > parity_pos) + disk->first_free_block = parity_pos; /* allocated a new deleted block from the block we are going to delete */ deleted = deleted_dup(block); @@ -223,75 +317,13 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); /* set the deleted block in the block array */ - tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block); - } -} - -/** - * Checks if a file is completely formed of blocks with invalid parity, - * and no rehash is tagged, and if it has at least one block. - */ -static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file) -{ - block_off_t i; - - /* with no block, it never has an invalid parity */ - if (file->blockmax == 0) - return 0; - - /* check all blocks */ - for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = &file->blockvec[i]; - snapraid_info info; - - /* exclude blocks with parity */ - if (!block_has_invalid_parity(block)) - return 0; - - /* get block specific info */ - info = info_get(&state->infoarr, block->parity_pos); - - /* if rehash fails */ - if (info_get_rehash(info)) - return 0; + fs_par2block_set(disk, parity_pos, &deleted->block); } - return 1; + file_free(file); } -/** - * Checks if a file is completely formed of blocks with an updated hash, - * and no rehash is tagged, and if it has at least one block. - */ -static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file) -{ - block_off_t i; - - /* with no block, it never has a hash */ - if (file->blockmax == 0) - return 0; - - /* check all blocks */ - for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = &file->blockvec[i]; - snapraid_info info; - - /* exclude blocks without hash */ - if (!block_has_updated_hash(block)) - return 0; - - /* get block specific info */ - info = info_get(&state->infoarr, block->parity_pos); - - /* exclude blocks needing a rehash */ - if (info_get_rehash(info)) - return 0; - } - - return 1; -} - -static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file) +static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file) { struct snapraid_state* state = scan->state; struct snapraid_disk* disk = scan->disk; @@ -312,9 +344,76 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid } } + /* insert in the delayed list */ tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file); } +/** + * Checks if a file is completely formed of blocks with invalid parity, + * and no rehash is tagged, and if it has at least one block. + */ +static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + block_off_t i; + + /* with no block, it never has an invalid parity */ + if (file->blockmax == 0) + return 0; + + /* check all blocks */ + for (i = 0; i < file->blockmax; ++i) { + snapraid_info info; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + block_off_t parity_pos = fs_file2par_get(disk, file, i); + + /* exclude blocks with parity */ + if (!block_has_invalid_parity(block)) + return 0; + + /* get block specific info */ + info = info_get(&state->infoarr, parity_pos); + + /* if rehash fails */ + if (info_get_rehash(info)) + return 0; + } + + return 1; +} + +/** + * Checks if a file is completely formed of blocks with an updated hash, + * and no rehash is tagged, and if it has at least one block. + */ +static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + block_off_t i; + + /* with no block, it never has a hash */ + if (file->blockmax == 0) + return 0; + + /* check all blocks */ + for (i = 0; i < file->blockmax; ++i) { + snapraid_info info; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + block_off_t parity_pos = fs_file2par_get(disk, file, i); + + /* exclude blocks without hash */ + if (!block_has_updated_hash(block)) + return 0; + + /* get block specific info */ + info = info_get(&state->infoarr, parity_pos); + + /* exclude blocks needing a rehash */ + if (info_get_rehash(info)) + return 0; + } + + return 1; +} + /** * Refresh the file info. * @@ -399,7 +498,43 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc } /** - * Keeps the file as it's (or with only a name/inode modification). + * Insert the file in the data set. + */ +static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file) +{ + struct snapraid_disk* disk = scan->disk; + + /* insert the file in the containers */ + if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); + + /* delayed allocationg of the parity */ + scan_file_delayed_allocate(scan, file); +} + +/** + * Remove the file from the data set. + * + * File is then deleted. + */ +static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file) +{ + struct snapraid_disk* disk = scan->disk; + + /* remove the file from the containers */ + if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) + tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); + tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset); + tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset); + + /* deallocate the file from the parity */ + scan_file_deallocate(scan, file); +} + +/** + * Keep the file as it's (or with only a name/inode modification). * * If the file is kept, nothing has to be done. * @@ -414,139 +549,17 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* fil struct snapraid_disk* disk = scan->disk; /* if the file is full invalid, schedule a reinsert at later stage */ - if (file_is_full_invalid_parity_and_stable(scan->state, file)) { - /* deallocate the file from the parity */ - scan_file_deallocate(scan, file); + if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) { + struct snapraid_file* copy = file_dup(file); - /* remove the file from the list */ - tommy_list_remove_existing(&disk->filelist, &file->nodelist); + /* remove the file */ + scan_file_remove(scan, file); - /* insert the file in the delayed block allocation */ - scan_file_delayed_insert(scan, file); + /* reinsert the copy in the delayed list */ + scan_file_insert(scan, copy); } } -/** - * Removes the specified file from the data set. - */ -static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file) -{ - struct snapraid_disk* disk = scan->disk; - - /* deallocate the file from the parity */ - scan_file_deallocate(scan, file); - - /* remove the file from the file containers */ - if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) { - tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); - } - tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset); - tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset); - tommy_list_remove_existing(&disk->filelist, &file->nodelist); - - /* deallocate */ - file_free(file); -} - -/** - * Inserts the specified file in the data set. - */ -static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file) -{ - struct snapraid_state* state = scan->state; - struct snapraid_disk* disk = scan->disk; - block_off_t i; - block_off_t block_max; - block_off_t block_pos; - - /* state changed */ - state->need_write = 1; - - /* allocate the blocks of the file */ - block_pos = disk->first_free_block; - block_max = tommy_arrayblk_size(&disk->blockarr); - for (i = 0; i < file->blockmax; ++i) { - snapraid_info info; - struct snapraid_block* block = &file->blockvec[i]; - - /* increment the position until the first really free block */ - while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos))) - ++block_pos; - - /* if not found, allocate a new one */ - if (block_pos == block_max) { - ++block_max; - tommy_arrayblk_grow(&disk->blockarr, block_max); - } - - /* set the position */ - block->parity_pos = block_pos; - - /* get block specific info */ - info = info_get(&state->infoarr, block_pos); - - /* if the file block already has an updated hash without rehash */ - if (block_has_updated_hash(block) && !info_get_rehash(info)) { - /* the only possible case is for REP blocks */ - assert(block_state_get(block) == BLOCK_STATE_REP); - - /* convert to a REP block */ - block_state_set(&file->blockvec[i], BLOCK_STATE_REP); - - /* and keep the hash as it's */ - } else { - struct snapraid_block* over_block; - unsigned block_state; - - /* convert to a CHG block */ - block_state_set(block, BLOCK_STATE_CHG); - - /* block to overwrite */ - over_block = tommy_arrayblk_get(&disk->blockarr, block_pos); - - /* state of the block we are going to overwrite */ - block_state = block_state_get(over_block); - - /* if the block is an empty one */ - if (block_state == BLOCK_STATE_EMPTY) { - /* the block was empty and filled with zeros */ - /* set the hash to the special ZERO value */ - hash_zero_set(block->hash); - } else { - /* otherwise it's a DELETED one */ - assert(block_state == BLOCK_STATE_DELETED); - - /* copy the past hash of the block */ - memcpy(block->hash, over_block->hash, HASH_SIZE); - - /* if we have not already cleared the past hash */ - if (!state->clear_past_hash) { - /* in this case we don't know if the old state is still the one */ - /* stored inside the parity, because after an aborted sync, the parity */ - /* may be or may be not have been updated with the new data */ - /* Then we reset the hash to a bogus value */ - /* For example: */ - /* - One file is deleted */ - /* - Sync aborted after, updating the parity to the new state, */ - /* but without saving the content file representing this new state. */ - /* - Another file is added again (exactly here) */ - /* with the hash of DELETED block not represeting the real parity state */ - hash_invalid_set(block->hash); - } - } - } - - /* store in the disk map, after invalidating all the other blocks */ - tommy_arrayblk_set(&disk->blockarr, block_pos, block); - - /* set the new free position */ - disk->first_free_block = block_pos + 1; - } - - /* note that the file is already added in the file hashtables */ - tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); -} - /** * Processes a file. */ @@ -869,7 +882,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash); /* if found, and it's a fully hashed file */ - if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) { + if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) { /* assume that the file is a copy, and reuse the hash */ file_copy(other_file, file); @@ -909,13 +922,8 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, } } - /* insert the file in the file hashtables, to allow to find duplicate hardlinks */ - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); - tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); - - /* insert the file in the delayed block allocation */ - scan_file_delayed_insert(scan, file); + /* insert the file in the delayed list */ + scan_file_insert(scan, file); } /** @@ -1544,8 +1552,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) /* next node */ node = node->next; - /* insert it */ - scan_file_insert(scan, file); + /* insert in the parity */ + scan_file_allocate(scan, file); } /* mark the disk without reliable physical offset if it has duplicates */ diff --git a/cmdline/scrub.c b/cmdline/scrub.c index dbc597f9..89cae70d 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -200,7 +200,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } /* if the block is not used */ - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (!block_has_file(block)) { /* use an empty block */ memset(buffer[j], 0, state->block_size); diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 2e111ef4..a20f7cb1 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -1001,9 +1001,29 @@ int main(int argc, char* argv[]) /* in the next state read ensures to clear all the past hashes in case */ /* we are reading from an incomplete sync */ /* The undeterminated hash are only for CHG/DELETED blocks for which we don't */ - /* know if the previous interrupted sync was able to update or not the parity */ + /* know if the previous interrupted sync was able to update or not the parity. */ /* The sync process instead needs to trust this information because it's used */ /* to avoid to recompute the parity if all the input are equals as before. */ + + /* In these cases we don't know if the old state is still the one */ + /* stored inside the parity, because after an aborted sync, the parity */ + /* may be or may be not have been updated with the data that may be now */ + /* deleted. Then we reset the hash to a bogus value. */ + + /* An example for CHG blocks is: */ + /* - One file is added creating a CHG block with ZERO state */ + /* - Sync aborted after updating the parity to the new state, */ + /* but without saving the content file representing this new BLK state. */ + /* - File is now deleted after the aborted sync */ + /* - Sync again, deleting the blocks overt the CHG ones */ + /* with the hash of CHG blocks not represeting the real parity state */ + + /* An example for DELETED blocks is: */ + /* - One file is deleted creating DELETED blocks */ + /* - Sync aborted after, updating the parity to the new state, */ + /* but without saving the content file representing this new EMPTY state. */ + /* - Another file is added again over the DELETE ones */ + /* with the hash of DELETED blocks not represeting the real parity state */ state.clear_past_hash = 1; state_read(&state); diff --git a/cmdline/state.c b/cmdline/state.c index bfafbc90..9c04198c 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1389,7 +1389,7 @@ static int position_is_required(struct snapraid_state* state, block_off_t pos) /* check for each disk */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); /* if we have at least one file, the position is needed */ if (block_has_file(block)) @@ -1418,7 +1418,7 @@ static int info_is_required(struct snapraid_state* state, block_off_t pos) /* check for each disk */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); /* if we have at least one synced file, the info is required */ if (block_state_get(block) == BLOCK_STATE_BLK) @@ -1435,12 +1435,12 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos /* check for each disk if block is really used */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); /* if the block is deleted */ if (block_state_get(block) == BLOCK_STATE_DELETED) { /* set it to empty */ - tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY); + fs_par2block_clear(disk, pos); } } } @@ -1450,7 +1450,7 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos */ static int is_block_deleted(struct snapraid_disk* disk, block_off_t pos) { - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); return block_state_get(block) == BLOCK_STATE_DELETED; } @@ -1616,7 +1616,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for '%s' specification in '%s' at line %u\n", tag, path, line); exit(EXIT_FAILURE); @@ -1624,8 +1624,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* insert the block in the block array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + 1); - tommy_arrayblk_set(&disk->blockarr, v_pos, block); + fs_par2block_set(disk, v_pos, block); /* check for termination of the block list */ ++blockidx; @@ -1765,7 +1764,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); @@ -1776,8 +1775,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); /* insert the block in the block array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + 1); - tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block); + fs_par2block_set(disk, v_pos, &deleted->block); } else if (strcmp(tag, "file") == 0) { /* file */ char sub[PATH_MAX]; @@ -2756,7 +2754,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) unsigned block_state = block_state_get(block); /* consistency check */ - if (block != disk_block_get(disk, block->parity_pos)) { + if (block != fs_par2block_get(disk, block->parity_pos)) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub); exit(EXIT_FAILURE); @@ -2858,7 +2856,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) } for (b = 0; b < blockmax; ++b) { if (is_block_deleted(disk, b)) { - struct snapraid_block* block = disk_block_get(disk, b); + struct snapraid_block* block = fs_par2block_get(disk, b); /* consistency check */ if (block->parity_pos != b) { @@ -3190,9 +3188,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_START */ } - /* grow the array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count); - /* fill the blocks in the run */ while (v_count) { struct snapraid_block* block = &file->blockvec[v_idx]; @@ -3277,7 +3272,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ decoding_error(path, f); log_fatal("Internal inconsistency in block existence!\n"); @@ -3286,7 +3281,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* insert the block in the block array */ - tommy_arrayblk_set(&disk->blockarr, v_pos, block); + fs_par2block_set(disk, v_pos, block); /* go to the next block */ ++v_idx; @@ -3437,9 +3432,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST switch (c) { case 'o' : - /* grow the array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count); - /* if it's a run of deleted blocks */ while (v_count) { struct snapraid_deleted* deleted; @@ -3466,7 +3458,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos); if (state->opt.skip_content_check) { @@ -3482,7 +3474,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); /* insert the block in the block array */ - tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block); + fs_par2block_set(disk, v_pos, &deleted->block); } /* go to next block */ @@ -4176,7 +4168,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) struct snapraid_block* block = blockvec + b; /* consistency check */ - if (block != disk_block_get(disk, block->parity_pos)) { + if (block != fs_par2block_get(disk, block->parity_pos)) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub); exit(EXIT_FAILURE); @@ -4275,7 +4267,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* write all the hash */ while (begin < end) { - struct snapraid_block* block = disk_block_get(disk, begin); + struct snapraid_block* block = fs_par2block_get(disk, begin); /* consistency check */ if (block->parity_pos != begin) { @@ -4677,7 +4669,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0 || filter_path(filterlist_file, 0, disk->name, file->sub) != 0 || filter_existence(filter_missing, disk->dir, file->sub) != 0 - || filter_correctness(filter_error, &state->infoarr, file) != 0 + || filter_correctness(filter_error, &state->infoarr, disk, file) != 0 ) { file_flag_set(file, FILE_IS_EXCLUDED); } diff --git a/cmdline/status.c b/cmdline/status.c index dc3c2c6e..d807c0cd 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -136,12 +136,13 @@ int state_status(struct snapraid_state* state) /* check fragmentation */ if (file->blockmax != 0) { block_off_t prev_pos; + block_off_t last_pos; int fragmented; fragmented = 0; - prev_pos = file->blockvec[0].parity_pos; + prev_pos = fs_file2par_get(disk, file, 0); for (j = 1; j < file->blockmax; ++j) { - block_off_t parity_pos = file->blockvec[j].parity_pos; + block_off_t parity_pos = fs_file2par_get(disk, file, j); if (prev_pos + 1 != parity_pos) { fragmented = 1; ++extra_fragment; @@ -151,8 +152,9 @@ int state_status(struct snapraid_state* state) } /* keep track of latest block used */ - if (file->blockvec[file->blockmax - 1].parity_pos > disk_block_latest_used) { - disk_block_latest_used = file->blockvec[file->blockmax - 1].parity_pos; + last_pos = fs_file2par_get(disk, file, file->blockmax - 1); + if (last_pos > disk_block_latest_used) { + disk_block_latest_used = last_pos; } if (fragmented) { @@ -279,7 +281,7 @@ int state_status(struct snapraid_state* state) one_valid = 0; for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) { struct snapraid_disk* disk = node_disk->data; - struct snapraid_block* block = disk_block_get(disk, i); + struct snapraid_block* block = fs_par2block_get(disk, i); if (block_has_file(block)) one_valid = 1; diff --git a/cmdline/sync.c b/cmdline/sync.c index 8da6946b..936a42ef 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -65,7 +65,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta struct snapraid_block* block; unsigned block_state; - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); /* get the state of the block */ block_state = block_state_get(block); @@ -101,7 +101,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta unsigned char hash[HASH_SIZE]; unsigned block_state; - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); /* get the state of the block */ block_state = block_state_get(block); @@ -424,7 +424,7 @@ static int block_is_enabled(block_off_t i, struct snapraid_handle* handle, unsig if (!handle[j].disk) continue; - block = disk_block_get(handle[j].disk, i); + block = fs_par2block_get(handle[j].disk, i); if (block_has_file(block)) one_valid = 1; @@ -584,7 +584,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } /* get the block */ - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); /* get the state of the block */ block_state = block_state_get(block); @@ -1029,7 +1029,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari for (j = 0; j < diskmax; ++j) { struct snapraid_block* block = BLOCK_EMPTY; if (handle[j].disk) - block = disk_block_get(handle[j].disk, i); + block = fs_par2block_get(handle[j].disk, i); if (block == BLOCK_EMPTY) { /* nothing to do */ @@ -1039,7 +1039,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* if it's a deleted block */ if (block_state_get(block) == BLOCK_STATE_DELETED) { /* the parity is now updated without this block, so it's now empty */ - tommy_arrayblk_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY); + fs_par2block_clear(handle[j].disk, i); continue; }