From 2801634aa4f939337c59456369e88879297ac9c9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 29 Sep 2015 22:51:17 +0200 Subject: [PATCH 01/12] Some more notes about the planned changes --- TODO | 46 +++++++++++++++++++++++++++++++++++++++------- 1 file changed, 39 insertions(+), 7 deletions(-) diff --git a/TODO b/TODO index 375c7edb..6961e525 100644 --- a/TODO +++ b/TODO @@ -17,17 +17,16 @@ The content file is instead missing in all platforms. - Minor * Check how to reduce memory occupation using smaller memory block. -We can theoreticallty reduce the memory occupation from 28/36 to -17 bytes. 16 for the hash and 1 for the type. -We can store the parity_position<->file,pos relation using some form of binary tree, +We can reduce the memory occupation from 28 (32bit)/36 (64bit) to 17 bytes. +16 for the hash and 1 for the type. + ++ We can store the parity_position<->file,pos relation using some form of binary tree, and then drop completely blockarr, and the extra data in memory_block. -A simpler simplification is to convert, blockarr to a parity_position->file,pos -relation, removing the file pointer inside the block. struct node { uint32 parity_pos /* parity position */ - file* file; /* pointer at the file */ - uint32 offset; /* position in the file */ + file* file_ptr; /* pointer at the file */ + uint32 file_pos; /* position in the file */ uint32 count; /* number of sequential blocks */ void* left_by_pp; /* tree by parity_pos */ void* right_by_pp; @@ -35,6 +34,28 @@ struct node { void* right_by_fp; }; +4*8*number_of_fragments = 32*30000 = 960 KB +but we have to add the extra allocation for each small block, so it could be +(4*8+32)**number_of_fragments = 64*30000 = 1920 KB = 1.9 MB +Low memory occupation. +We can likely make a fast insertion at the end when loading, putting nodes in a list, +and later insert them in the tree. + ++ Or instead of a tree we can use two vectors with binary search. + +parity_pos -> file,pos +[parity_pos,file_ptr,file_pos] 4+4+4 + +file,pos -> parity_pos +[file_ptr,file_pos,parity_pos] 4+4+4 + +(3*4*2)*number_of_fragments=24*30000 = 720 KB = 0.7 MB +Less memory occupation, but slow to change in the middle, like removing and adding files. +Very fast to load from disk as we insert only at the end. + ++ We can also get rid of deleted blocks, as the hash will be stored anyway. ++ We can put the state into the hash, dropping 3 bits of the hash of the 128 ones. + * Restore ownership and permissions, at least in Unix. * Restore directory timestamps. @@ -96,6 +117,17 @@ rest of the block. The problem is that we don't have any hash to ensure that the block is partially recovered, or completely garbage. But it makes sense to anyway write the "most likely" correct one. +* How to handle corrupt copied/moved files ? At now pre-hash checks them, +but if a corruption is found, it stops the sync process. +This is not optimal, as that blocks sync until a manual intervention. +This is the reason why the check of REP (copied) blocks is not enabled by default, +but only inside pre-hash. +A possible solution could be: +If a copied block is corrupt, we can recover the original one from parity (if moved), +or read it directly from disk (if copied), and we can use this one to build the parity, +working around the corruption, that can be later fixed with "fix". +This requires to process first copied blocks, with a possible complex dependency chain. + - Pooling * Add a new "commit" command to move changes from the pool to the array. From 0407c945db43d66ce069a2a77b8a1b8d879743ea Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2015 18:19:48 +0200 Subject: [PATCH 02/12] Change the interface for parity allocation This will allow a more compact representation of the parity allocation --- TODO | 12 ++ cmdline/check.c | 6 +- cmdline/dry.c | 2 +- cmdline/dup.c | 6 +- cmdline/elem.c | 106 ++++++++++- cmdline/elem.h | 62 +++++-- cmdline/import.c | 6 +- cmdline/import.h | 2 +- cmdline/parity.c | 13 +- cmdline/scan.c | 428 +++++++++++++++++++++++---------------------- cmdline/scrub.c | 2 +- cmdline/snapraid.c | 22 ++- cmdline/state.c | 44 ++--- cmdline/status.c | 12 +- cmdline/sync.c | 12 +- 15 files changed, 449 insertions(+), 286 deletions(-) diff --git a/TODO b/TODO index 6961e525..07fdef35 100644 --- a/TODO +++ b/TODO @@ -56,6 +56,18 @@ Very fast to load from disk as we insert only at the end. + We can also get rid of deleted blocks, as the hash will be stored anyway. + We can put the state into the hash, dropping 3 bits of the hash of the 128 ones. +- A problem is that we are going to move hash from file to parity. +How to keep the hash of the file when the file is not in the parity ? +We'll have to allocate hash also in the file, and move data. +We can keep the hash ALWAYS on the file! +But how to handle DELETED blocks ? At now we handle them one-by-one, +We have to change to handle them like files, to allow to have DELETED files, inside the tree. +Here the problem is how to handle when DELETED blocks are overwritten. +Note that we need the UNDELETE state to allow to recover in FIX. +The HASH is also used to search for DELETED blocks. + +TODO CONVERT DELETED BLOCKS TO DELETE FILES + * Restore ownership and permissions, at least in Unix. * Restore directory timestamps. diff --git a/cmdline/check.c b/cmdline/check.c index 1aa74ea3..c99bf893 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -619,7 +619,7 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s continue; } - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (!block_has_file(block)) { /* if no file, nothing to do */ continue; @@ -819,7 +819,7 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct if (!handle[j].disk) continue; - block = disk_block_get(handle[j].disk, i); + block = fs_par2block_get(handle[j].disk, i); /* try to recover all files, even the ones without hash */ /* because in some cases we can recover also them */ @@ -943,7 +943,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } /* if the disk block is not used */ - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (block == BLOCK_EMPTY) { /* use an empty block */ memset(buffer[j], 0, state->block_size); diff --git a/cmdline/dry.c b/cmdline/dry.c index 8c2b0815..aaa32349 100644 --- a/cmdline/dry.c +++ b/cmdline/dry.c @@ -67,7 +67,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit continue; } - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (!block_has_file(block)) { /* if no file, nothing to do */ diff --git a/cmdline/dup.c b/cmdline/dup.c index c7de7497..cb148f64 100644 --- a/cmdline/dup.c +++ b/cmdline/dup.c @@ -50,9 +50,11 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { - memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE); + struct snapraid_block* block = fs_file2block_get(disk, file, i); - if (!block_has_updated_hash(&file->blockvec[i])) { + memcpy(buf + i * HASH_SIZE, block->hash, HASH_SIZE); + + if (!block_has_updated_hash(block)) { free(buf); free(hash); return 0; diff --git a/cmdline/elem.c b/cmdline/elem.c index b79076a6..a101233c 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -309,7 +309,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub) return 1; } -int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file) +int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file) { unsigned i; @@ -318,12 +318,8 @@ int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapr /* check each block of the file */ for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block; - snapraid_info info; - - block = &file->blockvec[i]; - - info = info_get(infoarr, block->parity_pos); + block_off_t parity_pos = fs_file2par_get(disk, file, i); + snapraid_info info = info_get(infoarr, parity_pos); /* if the file has a bad block, don't exclude it */ if (info_get_bad(info)) @@ -444,7 +440,6 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ file->flag = 0; file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); - /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { file->blockvec[i].parity_pos = POS_INVALID; file->blockvec[i].file_mixed = 0; @@ -460,10 +455,40 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ return file; } +struct snapraid_file* file_dup(struct snapraid_file* copy) +{ + struct snapraid_file* file; + block_off_t i; + + file = malloc_nofail(sizeof(struct snapraid_file)); + file->sub = strdup_nofail(copy->sub); + file->size = copy->size; + file->blockmax = copy->blockmax; + file->mtime_sec = copy->mtime_sec; + file->mtime_nsec = copy->mtime_nsec; + file->inode = copy->inode; + file->physical = copy->physical; + file->flag = copy->flag; + file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); + + for (i = 0; i < file->blockmax; ++i) { + file->blockvec[i].file_mixed = copy->blockvec[i].file_mixed; + file->blockvec[i].parity_pos = copy->blockvec[i].parity_pos; + memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE); + + /* set the file to overwrite the copied one */ + block_file_set(&file->blockvec[i], file); + } + + return file; +} + void file_free(struct snapraid_file* file) { free(file->sub); + file->sub = 0; free(file->blockvec); + file->blockvec = 0; free(file); } @@ -739,6 +764,71 @@ void disk_free(struct snapraid_disk* disk) free(disk); } +block_off_t disk_block_size(struct snapraid_disk* disk) +{ + return tommy_arrayblk_size(&disk->blockarr); +} + +void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos) +{ + (void)disk; + + assert(file_pos < file->blockmax); + + file->blockvec[file_pos].parity_pos = parity_pos; +} + +struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) +{ + struct snapraid_block* block = fs_par2block_get(disk, parity_pos); + + if (block == BLOCK_EMPTY) + return 0; + + if (file_pos) + *file_pos = block_file_pos(block); + + return block_file_get(block); +} + +block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) +{ + (void)disk; + + assert(file_pos < file->blockmax); + + return file->blockvec[file_pos].parity_pos; +} + +struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) +{ + (void)disk; + + assert(file_pos < file->blockmax); + + return &file->blockvec[file_pos]; +} + +void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos) +{ + tommy_arrayblk_grow(&disk->blockarr, pos + 1); + tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY); +} + +void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block) +{ + tommy_arrayblk_grow(&disk->blockarr, pos + 1); + tommy_arrayblk_set(&disk->blockarr, pos, block); +} + +struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos) +{ + if (parity_pos < tommy_arrayblk_size(&disk->blockarr)) + return tommy_arrayblk_get(&disk->blockarr, parity_pos); + else + return BLOCK_EMPTY; +} + int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) { block_off_t i; diff --git a/cmdline/elem.h b/cmdline/elem.h index 4329c42f..55864715 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -465,7 +465,7 @@ int filter_existence(int filter_missing, const char* dir, const char* sub); * Filter a file if bad. * Return !=0 if the file is correct and it should be excluded. */ -int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_file* file); +int filter_correctness(int filter_error, tommy_arrayblkof* infoarr, struct snapraid_disk* disk, struct snapraid_file* file); /** * Filters a dir using a list of filters. @@ -503,7 +503,7 @@ static inline void block_file_set(struct snapraid_block* block, struct snapraid_ /* LCOV_EXCL_STOP */ } - block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | ptr; + block->file_mixed = (block->file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr; } /** @@ -700,12 +700,17 @@ static inline void file_flag_clear(struct snapraid_file* file, unsigned mask) } /** - * Allocates a file. + * Allocate a file. */ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical); /** - * Deallocates a file. + * Duplicate a file. + */ +struct snapraid_file* file_dup(struct snapraid_file* copy); + +/** + * Deallocate a file. */ void file_free(struct snapraid_file* file); @@ -884,16 +889,47 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev void disk_free(struct snapraid_disk* disk); /** - * Gets a specific block of a disk. - * Returns 0 if the block is over the end of the disk or not used. + * Get the size of the disk in blocks. */ -static inline struct snapraid_block* disk_block_get(struct snapraid_disk* disk, block_off_t pos) -{ - if (pos < tommy_arrayblk_size(&disk->blockarr)) - return tommy_arrayblk_get(&disk->blockarr, pos); - else - return BLOCK_EMPTY; -} +block_off_t disk_block_size(struct snapraid_disk* disk); + +/** + * Set a filesystem mapping between file and parity positions. + */ +void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos); + +/** + * Get the file position from the parity position. + * Return 0 if no file is using it. + */ +struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos); + +/** + * Get the parity position from the file position. + */ +block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos); + +/** + * Get the block from the file position. + */ +struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos); + +/** + * Clear a specific block. + */ +void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos); + +/** + * Set the block from parity position. + * In case of holes, everything is initialized to 0. + */ +void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block); + +/** + * Get the block from the parity position. + * Returns BLOCK_EMPTY==0 if the block is over the end of the disk or not used. + */ +struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos); /** * Check if a disk is totally empty and can be discarded from the content file. diff --git a/cmdline/import.c b/cmdline/import.c index 3f435bd3..86ee8be3 100644 --- a/cmdline/import.c +++ b/cmdline/import.c @@ -66,7 +66,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t file->path = strdup_nofail(path); file->size = size; file->blockmax = (size + block_size - 1) / block_size; - file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block)); + file->blockimp = malloc_nofail(file->blockmax * sizeof(struct snapraid_import_block)); buffer = malloc_nofail(block_size); @@ -98,7 +98,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t offset = 0; for (i = 0; i < file->blockmax; ++i) { - struct snapraid_import_block* block = &file->blockvec[i]; + struct snapraid_import_block* block = &file->blockimp[i]; unsigned read_size = block_size; if (read_size > size) read_size = size; @@ -145,7 +145,7 @@ static void import_file(struct snapraid_state* state, const char* path, uint64_t void import_file_free(struct snapraid_import_file* file) { free(file->path); - free(file->blockvec); + free(file->blockimp); free(file); } diff --git a/cmdline/import.h b/cmdline/import.h index d60af8cd..f397875e 100644 --- a/cmdline/import.h +++ b/cmdline/import.h @@ -46,7 +46,7 @@ struct snapraid_import_block { */ struct snapraid_import_file { data_off_t size; /**< Size of the file. */ - struct snapraid_import_block* blockvec; /**< All the blocks of the file. */ + struct snapraid_import_block* blockimp; /**< All the blocks of the file. */ block_off_t blockmax; /**< Number of blocks. */ char* path; /**< Full path of the file. */ diff --git a/cmdline/parity.c b/cmdline/parity.c index 902be1af..b7617669 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -37,12 +37,12 @@ block_off_t parity_allocated_size(struct snapraid_state* state) struct snapraid_disk* disk = i->data; /* start from the declared size */ - block_off_t block = tommy_arrayblk_size(&disk->blockarr); + block_off_t block = disk_block_size(disk); /* decrease the block until an allocated one, but part of a file */ /* we don't stop at deleted blocks, because we want to have them cleared */ /* if they are at the end of the parity */ - while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1))) + while (block > parity_block && !block_has_file(fs_par2block_get(disk, block - 1))) --block; /* get the highest value */ @@ -64,10 +64,10 @@ block_off_t parity_used_size(struct snapraid_state* state) struct snapraid_disk* disk = i->data; /* start from the declared size */ - block_off_t block = tommy_arrayblk_size(&disk->blockarr); + block_off_t block = disk_block_size(disk); /* decrease the block until an used one */ - while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1))) + while (block > parity_block && !block_has_file_and_valid_parity(fs_par2block_get(disk, block - 1))) --block; /* get the highest value */ @@ -95,7 +95,7 @@ int parity_is_invalid(struct snapraid_state* state) one_valid = 0; for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) { struct snapraid_disk* disk = node_disk->data; - struct snapraid_block* block = disk_block_get(disk, i); + struct snapraid_block* block = fs_par2block_get(disk, i); if (block_has_file(block)) one_valid = 1; @@ -134,7 +134,8 @@ void parity_overflow(struct snapraid_state* state, data_off_t size) struct snapraid_file* file = j->data; if (file->blockmax > 0) { - if (file->blockvec[file->blockmax - 1].parity_pos >= blockalloc) { + block_off_t parity_pos = fs_file2par_get(disk, file, file->blockmax - 1); + if (parity_pos >= blockalloc) { log_tag("outofparity:%s:%s\n", disk->name, esc(file->sub)); if (first) { first = 0; diff --git a/cmdline/scan.c b/cmdline/scan.c index ab7d2751..0c72fda6 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -155,22 +155,116 @@ static void scan_link(struct snapraid_scan* scan, int is_diff, const char* sub, } /** - * Removes the specified file from the parity. + * Insert the specified file in the parity. + */ +static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* file) +{ + struct snapraid_state* state = scan->state; + struct snapraid_disk* disk = scan->disk; + block_off_t i; + block_off_t parity_pos; + + /* state changed */ + state->need_write = 1; + + /* allocate the blocks of the file */ + parity_pos = disk->first_free_block; + for (i = 0; i < file->blockmax; ++i) { + snapraid_info info; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + + /* increment the position until the first really free block */ + while (block_has_file(fs_par2block_get(disk, parity_pos))) + ++parity_pos; + + /* set the position */ + block->parity_pos = parity_pos; + + /* get block specific info */ + info = info_get(&state->infoarr, parity_pos); + + /* if the file block already has an updated hash without rehash */ + if (block_has_updated_hash(block) && !info_get_rehash(info)) { + /* the only possible case is for REP blocks */ + assert(block_state_get(block) == BLOCK_STATE_REP); + + /* convert to a REP block */ + block_state_set(block, BLOCK_STATE_REP); + + /* and keep the hash as it's */ + } else { + struct snapraid_block* over_block; + unsigned block_state; + + /* convert to a CHG block */ + block_state_set(block, BLOCK_STATE_CHG); + + /* block to overwrite */ + over_block = fs_par2block_get(disk, parity_pos); + + /* state of the block we are going to overwrite */ + block_state = block_state_get(over_block); + + /* if the block is an empty one */ + if (block_state == BLOCK_STATE_EMPTY) { + /* the block was empty and filled with zeros */ + /* set the hash to the special ZERO value */ + hash_zero_set(block->hash); + } else { + /* otherwise it's a DELETED one */ + assert(block_state == BLOCK_STATE_DELETED); + + /* copy the past hash of the block */ + memcpy(block->hash, over_block->hash, HASH_SIZE); + + /* if we have not already cleared the past hash */ + if (!state->clear_past_hash) { + /* in this case we don't know if the old state is still the one */ + /* stored inside the parity, because after an aborted sync, the parity */ + /* may be or may be not have been updated with the new data */ + /* Then we reset the hash to a bogus value */ + /* For example: */ + /* - One file is deleted */ + /* - Sync aborted after, updating the parity to the new state, */ + /* but without saving the content file representing this new state. */ + /* - Another file is added again (exactly here) */ + /* with the hash of DELETED block not represeting the real parity state */ + hash_invalid_set(block->hash); + } + } + } + + /* store in the disk map, after invalidating all the other blocks */ + fs_par2block_set(disk, parity_pos, block); + + /* set the new free position */ + disk->first_free_block = parity_pos + 1; + } + + /* insert in the list of contained files */ + tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); +} + +/** + * Remove the specified file from the parity. + * The file is then inserted in the deleted set, and it should no be deallocated. */ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_file* file) { struct snapraid_state* state = scan->state; struct snapraid_disk* disk = scan->disk; - block_off_t i; + /* remove from the list of contained files */ + tommy_list_remove_existing(&disk->filelist, &file->nodelist); + /* state changed */ state->need_write = 1; /* free all the blocks of the file */ for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = &file->blockvec[i]; - block_off_t block_pos = block->parity_pos; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + block_off_t parity_pos = fs_file2par_get(disk, file, i); unsigned block_state; struct snapraid_deleted* deleted; @@ -178,8 +272,8 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil /* note that doing all the deletions before alllocations, */ /* first_free_block is always 0 and the "if" is never triggered */ /* but we keep this code anyway for completeness. */ - if (disk->first_free_block > block_pos) - disk->first_free_block = block_pos; + if (disk->first_free_block > parity_pos) + disk->first_free_block = parity_pos; /* allocated a new deleted block from the block we are going to delete */ deleted = deleted_dup(block); @@ -223,75 +317,13 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); /* set the deleted block in the block array */ - tommy_arrayblk_set(&disk->blockarr, block_pos, &deleted->block); - } -} - -/** - * Checks if a file is completely formed of blocks with invalid parity, - * and no rehash is tagged, and if it has at least one block. - */ -static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_file* file) -{ - block_off_t i; - - /* with no block, it never has an invalid parity */ - if (file->blockmax == 0) - return 0; - - /* check all blocks */ - for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = &file->blockvec[i]; - snapraid_info info; - - /* exclude blocks with parity */ - if (!block_has_invalid_parity(block)) - return 0; - - /* get block specific info */ - info = info_get(&state->infoarr, block->parity_pos); - - /* if rehash fails */ - if (info_get_rehash(info)) - return 0; + fs_par2block_set(disk, parity_pos, &deleted->block); } - return 1; + file_free(file); } -/** - * Checks if a file is completely formed of blocks with an updated hash, - * and no rehash is tagged, and if it has at least one block. - */ -static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_file* file) -{ - block_off_t i; - - /* with no block, it never has a hash */ - if (file->blockmax == 0) - return 0; - - /* check all blocks */ - for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = &file->blockvec[i]; - snapraid_info info; - - /* exclude blocks without hash */ - if (!block_has_updated_hash(block)) - return 0; - - /* get block specific info */ - info = info_get(&state->infoarr, block->parity_pos); - - /* exclude blocks needing a rehash */ - if (info_get_rehash(info)) - return 0; - } - - return 1; -} - -static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid_file* file) +static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file) { struct snapraid_state* state = scan->state; struct snapraid_disk* disk = scan->disk; @@ -312,9 +344,76 @@ static void scan_file_delayed_insert(struct snapraid_scan* scan, struct snapraid } } + /* insert in the delayed list */ tommy_list_insert_tail(&scan->file_insert_list, &file->nodelist, file); } +/** + * Checks if a file is completely formed of blocks with invalid parity, + * and no rehash is tagged, and if it has at least one block. + */ +static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + block_off_t i; + + /* with no block, it never has an invalid parity */ + if (file->blockmax == 0) + return 0; + + /* check all blocks */ + for (i = 0; i < file->blockmax; ++i) { + snapraid_info info; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + block_off_t parity_pos = fs_file2par_get(disk, file, i); + + /* exclude blocks with parity */ + if (!block_has_invalid_parity(block)) + return 0; + + /* get block specific info */ + info = info_get(&state->infoarr, parity_pos); + + /* if rehash fails */ + if (info_get_rehash(info)) + return 0; + } + + return 1; +} + +/** + * Checks if a file is completely formed of blocks with an updated hash, + * and no rehash is tagged, and if it has at least one block. + */ +static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct snapraid_disk* disk, struct snapraid_file* file) +{ + block_off_t i; + + /* with no block, it never has a hash */ + if (file->blockmax == 0) + return 0; + + /* check all blocks */ + for (i = 0; i < file->blockmax; ++i) { + snapraid_info info; + struct snapraid_block* block = fs_file2block_get(disk, file, i); + block_off_t parity_pos = fs_file2par_get(disk, file, i); + + /* exclude blocks without hash */ + if (!block_has_updated_hash(block)) + return 0; + + /* get block specific info */ + info = info_get(&state->infoarr, parity_pos); + + /* exclude blocks needing a rehash */ + if (info_get_rehash(info)) + return 0; + } + + return 1; +} + /** * Refresh the file info. * @@ -399,7 +498,43 @@ static void scan_file_refresh(struct snapraid_scan* scan, const char* sub, struc } /** - * Keeps the file as it's (or with only a name/inode modification). + * Insert the file in the data set. + */ +static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file) +{ + struct snapraid_disk* disk = scan->disk; + + /* insert the file in the containers */ + if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) + tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); + tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); + tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); + + /* delayed allocationg of the parity */ + scan_file_delayed_allocate(scan, file); +} + +/** + * Remove the file from the data set. + * + * File is then deleted. + */ +static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file) +{ + struct snapraid_disk* disk = scan->disk; + + /* remove the file from the containers */ + if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) + tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); + tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset); + tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset); + + /* deallocate the file from the parity */ + scan_file_deallocate(scan, file); +} + +/** + * Keep the file as it's (or with only a name/inode modification). * * If the file is kept, nothing has to be done. * @@ -414,139 +549,17 @@ static void scan_file_keep(struct snapraid_scan* scan, struct snapraid_file* fil struct snapraid_disk* disk = scan->disk; /* if the file is full invalid, schedule a reinsert at later stage */ - if (file_is_full_invalid_parity_and_stable(scan->state, file)) { - /* deallocate the file from the parity */ - scan_file_deallocate(scan, file); + if (file_is_full_invalid_parity_and_stable(scan->state, disk, file)) { + struct snapraid_file* copy = file_dup(file); - /* remove the file from the list */ - tommy_list_remove_existing(&disk->filelist, &file->nodelist); + /* remove the file */ + scan_file_remove(scan, file); - /* insert the file in the delayed block allocation */ - scan_file_delayed_insert(scan, file); + /* reinsert the copy in the delayed list */ + scan_file_insert(scan, copy); } } -/** - * Removes the specified file from the data set. - */ -static void scan_file_remove(struct snapraid_scan* scan, struct snapraid_file* file) -{ - struct snapraid_disk* disk = scan->disk; - - /* deallocate the file from the parity */ - scan_file_deallocate(scan, file); - - /* remove the file from the file containers */ - if (!file_flag_has(file, FILE_IS_WITHOUT_INODE)) { - tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset); - } - tommy_hashdyn_remove_existing(&disk->pathset, &file->pathset); - tommy_hashdyn_remove_existing(&disk->stampset, &file->stampset); - tommy_list_remove_existing(&disk->filelist, &file->nodelist); - - /* deallocate */ - file_free(file); -} - -/** - * Inserts the specified file in the data set. - */ -static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* file) -{ - struct snapraid_state* state = scan->state; - struct snapraid_disk* disk = scan->disk; - block_off_t i; - block_off_t block_max; - block_off_t block_pos; - - /* state changed */ - state->need_write = 1; - - /* allocate the blocks of the file */ - block_pos = disk->first_free_block; - block_max = tommy_arrayblk_size(&disk->blockarr); - for (i = 0; i < file->blockmax; ++i) { - snapraid_info info; - struct snapraid_block* block = &file->blockvec[i]; - - /* increment the position until the first really free block */ - while (block_pos < block_max && block_has_file(tommy_arrayblk_get(&disk->blockarr, block_pos))) - ++block_pos; - - /* if not found, allocate a new one */ - if (block_pos == block_max) { - ++block_max; - tommy_arrayblk_grow(&disk->blockarr, block_max); - } - - /* set the position */ - block->parity_pos = block_pos; - - /* get block specific info */ - info = info_get(&state->infoarr, block_pos); - - /* if the file block already has an updated hash without rehash */ - if (block_has_updated_hash(block) && !info_get_rehash(info)) { - /* the only possible case is for REP blocks */ - assert(block_state_get(block) == BLOCK_STATE_REP); - - /* convert to a REP block */ - block_state_set(&file->blockvec[i], BLOCK_STATE_REP); - - /* and keep the hash as it's */ - } else { - struct snapraid_block* over_block; - unsigned block_state; - - /* convert to a CHG block */ - block_state_set(block, BLOCK_STATE_CHG); - - /* block to overwrite */ - over_block = tommy_arrayblk_get(&disk->blockarr, block_pos); - - /* state of the block we are going to overwrite */ - block_state = block_state_get(over_block); - - /* if the block is an empty one */ - if (block_state == BLOCK_STATE_EMPTY) { - /* the block was empty and filled with zeros */ - /* set the hash to the special ZERO value */ - hash_zero_set(block->hash); - } else { - /* otherwise it's a DELETED one */ - assert(block_state == BLOCK_STATE_DELETED); - - /* copy the past hash of the block */ - memcpy(block->hash, over_block->hash, HASH_SIZE); - - /* if we have not already cleared the past hash */ - if (!state->clear_past_hash) { - /* in this case we don't know if the old state is still the one */ - /* stored inside the parity, because after an aborted sync, the parity */ - /* may be or may be not have been updated with the new data */ - /* Then we reset the hash to a bogus value */ - /* For example: */ - /* - One file is deleted */ - /* - Sync aborted after, updating the parity to the new state, */ - /* but without saving the content file representing this new state. */ - /* - Another file is added again (exactly here) */ - /* with the hash of DELETED block not represeting the real parity state */ - hash_invalid_set(block->hash); - } - } - } - - /* store in the disk map, after invalidating all the other blocks */ - tommy_arrayblk_set(&disk->blockarr, block_pos, block); - - /* set the new free position */ - disk->first_free_block = block_pos + 1; - } - - /* note that the file is already added in the file hashtables */ - tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); -} - /** * Processes a file. */ @@ -869,7 +882,7 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash); /* if found, and it's a fully hashed file */ - if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) { + if (other_file && file_is_full_hashed_and_stable(scan->state, other_disk, other_file)) { /* assume that the file is a copy, and reuse the hash */ file_copy(other_file, file); @@ -909,13 +922,8 @@ static void scan_file(struct snapraid_scan* scan, int is_diff, const char* sub, } } - /* insert the file in the file hashtables, to allow to find duplicate hardlinks */ - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); - tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); - - /* insert the file in the delayed block allocation */ - scan_file_delayed_insert(scan, file); + /* insert the file in the delayed list */ + scan_file_insert(scan, file); } /** @@ -1544,8 +1552,8 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) /* next node */ node = node->next; - /* insert it */ - scan_file_insert(scan, file); + /* insert in the parity */ + scan_file_allocate(scan, file); } /* mark the disk without reliable physical offset if it has duplicates */ diff --git a/cmdline/scrub.c b/cmdline/scrub.c index dbc597f9..89cae70d 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -200,7 +200,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } /* if the block is not used */ - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); if (!block_has_file(block)) { /* use an empty block */ memset(buffer[j], 0, state->block_size); diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 2e111ef4..a20f7cb1 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -1001,9 +1001,29 @@ int main(int argc, char* argv[]) /* in the next state read ensures to clear all the past hashes in case */ /* we are reading from an incomplete sync */ /* The undeterminated hash are only for CHG/DELETED blocks for which we don't */ - /* know if the previous interrupted sync was able to update or not the parity */ + /* know if the previous interrupted sync was able to update or not the parity. */ /* The sync process instead needs to trust this information because it's used */ /* to avoid to recompute the parity if all the input are equals as before. */ + + /* In these cases we don't know if the old state is still the one */ + /* stored inside the parity, because after an aborted sync, the parity */ + /* may be or may be not have been updated with the data that may be now */ + /* deleted. Then we reset the hash to a bogus value. */ + + /* An example for CHG blocks is: */ + /* - One file is added creating a CHG block with ZERO state */ + /* - Sync aborted after updating the parity to the new state, */ + /* but without saving the content file representing this new BLK state. */ + /* - File is now deleted after the aborted sync */ + /* - Sync again, deleting the blocks overt the CHG ones */ + /* with the hash of CHG blocks not represeting the real parity state */ + + /* An example for DELETED blocks is: */ + /* - One file is deleted creating DELETED blocks */ + /* - Sync aborted after, updating the parity to the new state, */ + /* but without saving the content file representing this new EMPTY state. */ + /* - Another file is added again over the DELETE ones */ + /* with the hash of DELETED blocks not represeting the real parity state */ state.clear_past_hash = 1; state_read(&state); diff --git a/cmdline/state.c b/cmdline/state.c index bfafbc90..9c04198c 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1389,7 +1389,7 @@ static int position_is_required(struct snapraid_state* state, block_off_t pos) /* check for each disk */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); /* if we have at least one file, the position is needed */ if (block_has_file(block)) @@ -1418,7 +1418,7 @@ static int info_is_required(struct snapraid_state* state, block_off_t pos) /* check for each disk */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); /* if we have at least one synced file, the info is required */ if (block_state_get(block) == BLOCK_STATE_BLK) @@ -1435,12 +1435,12 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos /* check for each disk if block is really used */ for (i = state->disklist; i != 0; i = i->next) { struct snapraid_disk* disk = i->data; - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); /* if the block is deleted */ if (block_state_get(block) == BLOCK_STATE_DELETED) { /* set it to empty */ - tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY); + fs_par2block_clear(disk, pos); } } } @@ -1450,7 +1450,7 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos */ static int is_block_deleted(struct snapraid_disk* disk, block_off_t pos) { - struct snapraid_block* block = disk_block_get(disk, pos); + struct snapraid_block* block = fs_par2block_get(disk, pos); return block_state_get(block) == BLOCK_STATE_DELETED; } @@ -1616,7 +1616,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for '%s' specification in '%s' at line %u\n", tag, path, line); exit(EXIT_FAILURE); @@ -1624,8 +1624,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* insert the block in the block array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + 1); - tommy_arrayblk_set(&disk->blockarr, v_pos, block); + fs_par2block_set(disk, v_pos, block); /* check for termination of the block list */ ++blockidx; @@ -1765,7 +1764,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); @@ -1776,8 +1775,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); /* insert the block in the block array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + 1); - tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block); + fs_par2block_set(disk, v_pos, &deleted->block); } else if (strcmp(tag, "file") == 0) { /* file */ char sub[PATH_MAX]; @@ -2756,7 +2754,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) unsigned block_state = block_state_get(block); /* consistency check */ - if (block != disk_block_get(disk, block->parity_pos)) { + if (block != fs_par2block_get(disk, block->parity_pos)) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub); exit(EXIT_FAILURE); @@ -2858,7 +2856,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) } for (b = 0; b < blockmax; ++b) { if (is_block_deleted(disk, b)) { - struct snapraid_block* block = disk_block_get(disk, b); + struct snapraid_block* block = fs_par2block_get(disk, b); /* consistency check */ if (block->parity_pos != b) { @@ -3190,9 +3188,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_START */ } - /* grow the array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count); - /* fill the blocks in the run */ while (v_count) { struct snapraid_block* block = &file->blockvec[v_idx]; @@ -3277,7 +3272,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ decoding_error(path, f); log_fatal("Internal inconsistency in block existence!\n"); @@ -3286,7 +3281,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* insert the block in the block array */ - tommy_arrayblk_set(&disk->blockarr, v_pos, block); + fs_par2block_set(disk, v_pos, block); /* go to the next block */ ++v_idx; @@ -3437,9 +3432,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST switch (c) { case 'o' : - /* grow the array */ - tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count); - /* if it's a run of deleted blocks */ while (v_count) { struct snapraid_deleted* deleted; @@ -3466,7 +3458,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* we must not overwrite existing blocks */ - if (disk_block_get(disk, v_pos) != BLOCK_EMPTY) { + if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos); if (state->opt.skip_content_check) { @@ -3482,7 +3474,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); /* insert the block in the block array */ - tommy_arrayblk_set(&disk->blockarr, v_pos, &deleted->block); + fs_par2block_set(disk, v_pos, &deleted->block); } /* go to next block */ @@ -4176,7 +4168,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) struct snapraid_block* block = blockvec + b; /* consistency check */ - if (block != disk_block_get(disk, block->parity_pos)) { + if (block != fs_par2block_get(disk, block->parity_pos)) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub); exit(EXIT_FAILURE); @@ -4275,7 +4267,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* write all the hash */ while (begin < end) { - struct snapraid_block* block = disk_block_get(disk, begin); + struct snapraid_block* block = fs_par2block_get(disk, begin); /* consistency check */ if (block->parity_pos != begin) { @@ -4677,7 +4669,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom if (filter_path(filterlist_disk, 0, disk->name, file->sub) != 0 || filter_path(filterlist_file, 0, disk->name, file->sub) != 0 || filter_existence(filter_missing, disk->dir, file->sub) != 0 - || filter_correctness(filter_error, &state->infoarr, file) != 0 + || filter_correctness(filter_error, &state->infoarr, disk, file) != 0 ) { file_flag_set(file, FILE_IS_EXCLUDED); } diff --git a/cmdline/status.c b/cmdline/status.c index dc3c2c6e..d807c0cd 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -136,12 +136,13 @@ int state_status(struct snapraid_state* state) /* check fragmentation */ if (file->blockmax != 0) { block_off_t prev_pos; + block_off_t last_pos; int fragmented; fragmented = 0; - prev_pos = file->blockvec[0].parity_pos; + prev_pos = fs_file2par_get(disk, file, 0); for (j = 1; j < file->blockmax; ++j) { - block_off_t parity_pos = file->blockvec[j].parity_pos; + block_off_t parity_pos = fs_file2par_get(disk, file, j); if (prev_pos + 1 != parity_pos) { fragmented = 1; ++extra_fragment; @@ -151,8 +152,9 @@ int state_status(struct snapraid_state* state) } /* keep track of latest block used */ - if (file->blockvec[file->blockmax - 1].parity_pos > disk_block_latest_used) { - disk_block_latest_used = file->blockvec[file->blockmax - 1].parity_pos; + last_pos = fs_file2par_get(disk, file, file->blockmax - 1); + if (last_pos > disk_block_latest_used) { + disk_block_latest_used = last_pos; } if (fragmented) { @@ -279,7 +281,7 @@ int state_status(struct snapraid_state* state) one_valid = 0; for (node_disk = state->disklist; node_disk != 0; node_disk = node_disk->next) { struct snapraid_disk* disk = node_disk->data; - struct snapraid_block* block = disk_block_get(disk, i); + struct snapraid_block* block = fs_par2block_get(disk, i); if (block_has_file(block)) one_valid = 1; diff --git a/cmdline/sync.c b/cmdline/sync.c index 8da6946b..936a42ef 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -65,7 +65,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta struct snapraid_block* block; unsigned block_state; - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); /* get the state of the block */ block_state = block_state_get(block); @@ -101,7 +101,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta unsigned char hash[HASH_SIZE]; unsigned block_state; - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); /* get the state of the block */ block_state = block_state_get(block); @@ -424,7 +424,7 @@ static int block_is_enabled(block_off_t i, struct snapraid_handle* handle, unsig if (!handle[j].disk) continue; - block = disk_block_get(handle[j].disk, i); + block = fs_par2block_get(handle[j].disk, i); if (block_has_file(block)) one_valid = 1; @@ -584,7 +584,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } /* get the block */ - block = disk_block_get(disk, i); + block = fs_par2block_get(disk, i); /* get the state of the block */ block_state = block_state_get(block); @@ -1029,7 +1029,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari for (j = 0; j < diskmax; ++j) { struct snapraid_block* block = BLOCK_EMPTY; if (handle[j].disk) - block = disk_block_get(handle[j].disk, i); + block = fs_par2block_get(handle[j].disk, i); if (block == BLOCK_EMPTY) { /* nothing to do */ @@ -1039,7 +1039,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* if it's a deleted block */ if (block_state_get(block) == BLOCK_STATE_DELETED) { /* the parity is now updated without this block, so it's now empty */ - tommy_arrayblk_set(&handle[j].disk->blockarr, i, BLOCK_EMPTY); + fs_par2block_clear(handle[j].disk, i); continue; } From 23983d6e9c71d97d00a3ba71ad2854b4ee3937d7 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2015 23:01:54 +0200 Subject: [PATCH 03/12] Drop text file support --- Makefile.am | 12 +- cmdline/snapraid.c | 7 - cmdline/state.c | 1482 +------------------------------------------- cmdline/state.h | 1 - 4 files changed, 8 insertions(+), 1494 deletions(-) diff --git a/Makefile.am b/Makefile.am index 50880983..61f94523 100644 --- a/Makefile.am +++ b/Makefile.am @@ -532,23 +532,17 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR6) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync - echo --- Delete some files and create some new, sync and check in multiple steps using text content + echo --- Delete some files and create some new, sync and check in multiple steps rm bench/disk1/a/4* rm bench/disk2/a/5* rm bench/disk3/a/6* rm bench/disk4/a/6* - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 10 sync --test-force-content-text - cp bench/content bench/content.txt + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 10 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(CONF) test-rewrite - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS_VERBOSE) -c $(CONF) --test-force-content-text test-rewrite - cmp bench/content bench/content.txt $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check -l test.log $(TESTENV) ./mktest$(EXEEXT) generate 4 6 $(CHECKCOUNT) $(CHECKSIZE) - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 100 sync --test-force-content-text - cp bench/content bench/content.txt + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 100 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-rewrite - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-force-content-text test-rewrite - cmp bench/content bench/content.txt $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -B 1000 sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index a20f7cb1..a80e659f 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -234,7 +234,6 @@ void config(char* conf, size_t conf_size, const char* argv0) #define OPT_TEST_FORCE_SCRUB_AT 271 #define OPT_TEST_FORCE_SCRUB_EVEN 272 #define OPT_TEST_FORCE_CONTENT_WRITE 273 -#define OPT_TEST_FORCE_CONTENT_TEXT 274 #define OPT_TEST_SKIP_CONTENT_CHECK 275 #define OPT_TEST_SKIP_PARITY_ACCESS 276 #define OPT_TEST_EXPECT_FAILURE 277 @@ -327,9 +326,6 @@ struct option long_options[] = { /* Force write of the content file even if no modification is done */ { "test-force-content-write", 0, 0, OPT_TEST_FORCE_CONTENT_WRITE }, - /* Force the use of text content file */ - { "test-force-content-text", 0, 0, OPT_TEST_FORCE_CONTENT_TEXT }, - /* Relax the checks done at the content file */ { "test-skip-content-check", 0, 0, OPT_TEST_SKIP_CONTENT_CHECK }, @@ -684,9 +680,6 @@ int main(int argc, char* argv[]) case OPT_TEST_FORCE_CONTENT_WRITE : opt.force_content_write = 1; break; - case OPT_TEST_FORCE_CONTENT_TEXT : - opt.force_content_text = 1; - break; case OPT_TEST_EXPECT_FAILURE : /* invert the exit codes */ exit_success = 1; diff --git a/cmdline/state.c b/cmdline/state.c index 9c04198c..10db227d 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1455,1475 +1455,6 @@ static int is_block_deleted(struct snapraid_disk* disk, block_off_t pos) return block_state_get(block) == BLOCK_STATE_DELETED; } -static void state_read_text(struct snapraid_state* state, const char* path, STREAM* f, time_t save_time) -{ - struct snapraid_disk* disk; - struct snapraid_file* file; - block_off_t blockidx; - block_off_t blockmax; - unsigned line; - unsigned count_file; - unsigned count_hardlink; - unsigned count_symlink; - unsigned count_dir; - block_off_t b; - - disk = 0; - file = 0; - line = 1; - blockidx = 0; - blockmax = 0; - count_file = 0; - count_hardlink = 0; - count_symlink = 0; - count_dir = 0; - - while (1) { - char buffer[PATH_MAX]; - char tag[PATH_MAX]; - int ret; - int c; - - /* read the command */ - ret = sgettok(f, tag, sizeof(tag)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Error reading the configuration file '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* skip only one space if present */ - c = sgetc(f); - if (c != ' ') { - sungetc(c, f); - } - - if (strcmp(tag, "blk") == 0 || strcmp(tag, "new") == 0 - || strcmp(tag, "chg") == 0 || strcmp(tag, "rep") == 0 - ) { - /* "blk"/""new"/"chg"/"rep" command */ - block_off_t v_pos; - struct snapraid_block* block; - - if (!file) { - /* LCOV_EXCL_START */ - log_fatal("Unexpected '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &v_pos); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (blockidx >= file->blockmax) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - block = &file->blockvec[blockidx]; - - if (block->parity_pos != POS_INVALID) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* set the parity only if present */ - switch (tag[0]) { - case 'b' : - block_state_set(block, BLOCK_STATE_BLK); - break; - case 'n' : - /* deprecated NEW blocks are converted to CHG ones */ - block_state_set(block, BLOCK_STATE_CHG); - break; - case 'c' : - block_state_set(block, BLOCK_STATE_CHG); - break; - case 'r' : - block_state_set(block, BLOCK_STATE_REP); - break; - } - - block->parity_pos = v_pos; - - /* keep track of the max block number used in parity */ - if (v_pos + 1 > blockmax) - blockmax = v_pos + 1; - - /* read the hash only for 'blk/chg/rep', and not for 'new' */ - if (tag[0] != 'n') { - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgethex(f, block->hash, HASH_SIZE); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else { - /* set the ZERO hash for deprecated NEW blocks */ - hash_zero_set(block->hash); - } - - /* if the block contains a hash of past data */ - /* and we are clearing such undeterminated hashes */ - if (state->clear_past_hash - && block_has_past_hash(block) - ) { - /* set the hash value to INVALID */ - hash_invalid_set(block->hash); - } - - /* if we are disabling the copy optimization */ - /* we want also to clear any already previously stored information */ - /* in other sync commands */ - /* note that this is required only in sync, and we detect */ - /* this using the clear_past_hash flag */ - if (state->clear_past_hash - && state->opt.force_nocopy - && block_state_get(block) == BLOCK_STATE_REP - ) { - /* set the hash value to INVALID */ - hash_invalid_set(block->hash); - /* convert from REP to CHG block */ - block_state_set(block, BLOCK_STATE_CHG); - } - - /* if we want a full sync, marks block as invalid parity */ - /* note that we do this after the force_nocopy option */ - /* to avoid to mixup the two things */ - if (state->opt.force_full - && block_state_get(block) == BLOCK_STATE_BLK) { - /* convert from BLK to REP */ - block_state_set(block, BLOCK_STATE_REP); - } - - /* we must not overwrite existing blocks */ - if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for '%s' specification in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* insert the block in the block array */ - fs_par2block_set(disk, v_pos, block); - - /* check for termination of the block list */ - ++blockidx; - if (blockidx == file->blockmax) { - file = 0; - disk = 0; - } - } else if (strcmp(tag, "inf") == 0) { - /* "inf" command */ - block_off_t v_pos; - snapraid_info info; - int rehash; - int bad; - int justsynced; - uint32_t t; - - ret = sgetu32(f, &v_pos); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'inf' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (v_pos >= blockmax) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in position for 'inf' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'inf' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &t); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'inf' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* read extra tags if present */ - rehash = 0; - bad = 0; - justsynced = 0; - c = sgetc(f); - while (c == ' ') { - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'inf' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (strcmp(buffer, "bad") == 0) { - bad = 1; - } else if (strcmp(buffer, "rehash") == 0) { - rehash = 1; - - if (state->prevhash == HASH_UNDEFINED) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for missing previous checksum in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else if (strcmp(buffer, "justsynced") == 0) { - justsynced = 1; - } else { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'inf' specification '%s' in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - } - sungetc(c, f); - - info = info_make(t, bad, rehash, justsynced); - - /* insert the info in the array */ - info_set(&state->infoarr, v_pos, info); - } else if (strcmp(tag, "off") == 0) { - /* "off" command */ - block_off_t v_pos; - struct snapraid_deleted* deleted; - - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Unexpected 'off' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* allocate a new deleted block */ - deleted = deleted_alloc(); - - ret = sgetu32(f, &v_pos); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'off' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - deleted->block.parity_pos = v_pos; - - /* read the hash */ - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'off' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* set the hash only if present */ - ret = sgethex(f, deleted->block.hash, HASH_SIZE); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'off' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* if we are clearing undeterminated hashes */ - if (state->clear_past_hash) { - /* set the hash value to INVALID */ - hash_invalid_set(deleted->block.hash); - } - - /* we must not overwrite existing blocks */ - if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for 'off' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* insert it in the list of deleted blocks */ - tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); - - /* insert the block in the block array */ - fs_par2block_set(disk, v_pos, &deleted->block); - } else if (strcmp(tag, "file") == 0) { - /* file */ - char sub[PATH_MAX]; - uint64_t v_size; - uint64_t v_mtime_sec; - uint32_t v_mtime_nsec; - uint64_t v_inode; - - if (file) { - /* LCOV_EXCL_START */ - log_fatal("Missing 'blk' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu64(f, &v_size); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu64(f, &v_mtime_sec); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c == '.') { /* the nanosecond field is present only from version 1.14 */ - ret = sgetu32(f, &v_mtime_nsec); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - } else { - /* use a special value, meaning that we don't have this information */ - v_mtime_nsec = STAT_NSEC_INVALID; - } - - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu64(f, &v_inode); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetline(f, sub, sizeof(sub)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (!*sub) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'file' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* allocate the file */ - file = file_alloc(state->block_size, sub, v_size, v_mtime_sec, v_mtime_nsec, v_inode, 0); - - /* insert the file in the file containers */ - tommy_hashdyn_insert(&disk->inodeset, &file->nodeset, file, file_inode_hash(file->inode)); - tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); - tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); - tommy_list_insert_tail(&disk->filelist, &file->nodelist, file); - - /* start the block allocation of the file */ - blockidx = 0; - - /* check for empty file */ - if (blockidx == file->blockmax) { - file = 0; - disk = 0; - } - - /* stat */ - ++count_file; - } else if (strcmp(tag, "hole") == 0) { - /* hole */ - - if (file) { - /* LCOV_EXCL_START */ - log_fatal("Missing 'blk' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hole' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else if (strcmp(tag, "symlink") == 0) { - /* symlink */ - char sub[PATH_MAX]; - char linkto[PATH_MAX]; - char tokento[32]; - struct snapraid_link* link; - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetline(f, sub, sizeof(sub)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgeteol(f); - if (c != '\n') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - ++line; - - ret = sgettok(f, tokento, sizeof(tokento)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - if (strcmp(tokento, "to") != 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetline(f, linkto, sizeof(linkto)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (!*sub || !*linkto) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'symlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* allocate the link as symbolic link */ - link = link_alloc(sub, linkto, FILE_IS_SYMLINK); - - /* insert the link in the link containers */ - tommy_hashdyn_insert(&disk->linkset, &link->nodeset, link, link_name_hash(link->sub)); - tommy_list_insert_tail(&disk->linklist, &link->nodelist, link); - - /* stat */ - ++count_symlink; - } else if (strcmp(tag, "hardlink") == 0) { - /* hardlink */ - char sub[PATH_MAX]; - char linkto[PATH_MAX]; - char tokento[32]; - struct snapraid_link* link; - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetline(f, sub, sizeof(sub)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgeteol(f); - if (c != '\n') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - ++line; - - ret = sgettok(f, tokento, sizeof(tokento)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - if (strcmp(tokento, "to") != 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetline(f, linkto, sizeof(linkto)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (!*sub || !*linkto) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'hardlink' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* allocate the link as hard link */ - link = link_alloc(sub, linkto, FILE_IS_HARDLINK); - - /* insert the link in the link containers */ - tommy_hashdyn_insert(&disk->linkset, &link->nodeset, link, link_name_hash(link->sub)); - tommy_list_insert_tail(&disk->linklist, &link->nodelist, link); - - /* stat */ - ++count_hardlink; - } else if (strcmp(tag, "dir") == 0) { - /* dir */ - char sub[PATH_MAX]; - struct snapraid_dir* dir; - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'dir' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'dir' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetline(f, sub, sizeof(sub)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'dir' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (!*sub) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'dir' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* find the disk */ - disk = find_disk(state, buffer); - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Disk named '%s' not found in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* allocate the dir */ - dir = dir_alloc(sub); - - /* insert the dir in the dir containers */ - tommy_hashdyn_insert(&disk->dirset, &dir->nodeset, dir, dir_name_hash(dir->sub)); - tommy_list_insert_tail(&disk->dirlist, &dir->nodelist, dir); - - /* stat */ - ++count_dir; - } else if (strcmp(tag, "checksum") == 0) { - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'checksum' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (strcmp(buffer, "murmur3") == 0) { - state->hash = HASH_MURMUR3; - } else if (strcmp(buffer, "spooky2") == 0) { - state->hash = HASH_SPOOKY2; - } else { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'checksum' specification '%s' in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c == ' ') { - /* read the seed if present */ - ret = sgethex(f, state->hashseed, HASH_SIZE); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'seed' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else { - sungetc(c, f); - memset(state->hashseed, 0, HASH_SIZE); - } - } else if (strcmp(tag, "prevchecksum") == 0) { - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'prevchecksum' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (strcmp(buffer, "murmur3") == 0) { - state->prevhash = HASH_MURMUR3; - } else if (strcmp(buffer, "spooky2") == 0) { - state->prevhash = HASH_SPOOKY2; - } else { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'prevchecksum' specification '%s' in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'prevchecksum' specification '%s' in '%s' at line %u\n", buffer, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* read the seed */ - ret = sgethex(f, state->prevhashseed, HASH_SIZE); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'seed' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else if (strcmp(tag, "blksize") == 0) { - block_off_t blksize; - - ret = sgetu32(f, &blksize); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'blksize' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* without configuration, auto assign the block size */ - if (state->no_conf) { - state->block_size = blksize; - } - - if (blksize != state->block_size) { - /* LCOV_EXCL_START */ - log_fatal("Mismatching 'blksize' and 'blocksize' specification in '%s' at line %u\n", path, line); - log_fatal("Please restore the 'blocksize' value in the configuration file to '%u'\n", blksize / 1024); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else if (strcmp(tag, "map") == 0 || strcmp(tag, "mapping") == 0) { - struct snapraid_map* map; - char uuid[UUID_MAX]; - uint32_t v_pos; - uint32_t v_total_blocks; - uint32_t v_free_blocks; - - ret = sgettok(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &v_pos); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* from SnapRAID 7.0 the 'mapping' command includes the free space */ - if (strcmp(tag, "mapping") == 0) { - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &v_total_blocks); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &v_free_blocks); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else { - v_total_blocks = 0; - v_free_blocks = 0; - } - - /* check if the uuid is present */ - /* uuid must be the latest one as it could be also empty */ - c = sgetc(f); - if (c != ' ') { - sungetc(c, f); - uuid[0] = 0; - } else { - /* read the uuid */ - ret = sgettok(f, uuid, sizeof(uuid)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - - map = map_alloc(buffer, v_pos, v_total_blocks, v_free_blocks, uuid); - - tommy_list_insert_tail(&state->maplist, &map->node, map); - } else if (strcmp(tag, "parity") == 0) { - /* from SnapRAID 7.0 the 'parity' command includes the free space */ - char uuid[UUID_MAX]; - uint32_t v_level; - uint32_t v_total_blocks; - uint32_t v_free_blocks; - - ret = sgetu32(f, &v_level); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'parity' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'parity' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &v_total_blocks); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'parity' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - c = sgetc(f); - if (c != ' ') { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'parity' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ret = sgetu32(f, &v_free_blocks); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'parity' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* check if the uuid is present */ - /* uuid must be the latest one as it could be also empty */ - c = sgetc(f); - if (c != ' ') { - sungetc(c, f); - uuid[0] = 0; - } else { - /* read the uuid */ - ret = sgettok(f, uuid, sizeof(uuid)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'map' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - - /* auto configure if configuration is missing */ - if (state->no_conf && v_level < LEV_MAX && v_level >= state->level) - state->level = v_level + 1; - - /* if we use this parity entry */ - if (v_level < state->level) { - /* set the parity info */ - pathcpy(state->parity[v_level].uuid, sizeof(state->parity[v_level].uuid), uuid); - state->parity[v_level].total_blocks = v_total_blocks; - state->parity[v_level].free_blocks = v_free_blocks; - } - } else if (strcmp(tag, "sign") == 0) { - uint32_t sign; - - ret = sgetu32(f, &sign); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid 'sign' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* sign check not supported anymore for the text content file */ - } else if (tag[0] == 0) { - /* allow empty lines */ - sgetspace(f); - } else if (tag[0] == '#') { - ret = sgetline(f, buffer, sizeof(buffer)); - if (ret < 0) { - /* LCOV_EXCL_START */ - log_fatal("Invalid comment in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } else { - /* LCOV_EXCL_START */ - log_fatal("Invalid command '%s' in '%s' at line %u\n", tag, path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* next line */ - c = sgeteol(f); - if (c == EOF) { - break; - } - if (c != '\n') { - /* LCOV_EXCL_START */ - log_fatal("Extra data in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - ++line; - } - - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error reading the content file '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (file) { - /* LCOV_EXCL_START */ - log_fatal("Missing 'blk' specification in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* check that the stored parity size matches the loaded state */ - if (blockmax != parity_allocated_size(state)) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in parity size in '%s' at line %u\n", path, line); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* to be backward compatible with old context files without info */ - /* create info records if missing */ - for (b = 0; b < blockmax; ++b) { - /* if the position is used */ - if (info_is_required(state, b)) { - snapraid_info info = info_get(&state->infoarr, b); - - /* if no info is present */ - if (!info) { - /* LCOV_EXCL_START */ - /* set a fake info block */ - info = info_make(save_time, 0, 0, 0); - - /* insert the info in the array */ - info_set(&state->infoarr, b, info); - /* LCOV_EXCL_STOP */ - } - } - } - - msg_verbose("%8u files\n", count_file); - msg_verbose("%8u hardlinks\n", count_hardlink); - msg_verbose("%8u symlinks\n", count_symlink); - msg_verbose("%8u empty dirs\n", count_dir); -} - -static void state_write_text(struct snapraid_state* state, STREAM* f) -{ - unsigned count_file; - unsigned count_hardlink; - unsigned count_symlink; - unsigned count_dir; - tommy_node* i; - block_off_t b; - block_off_t blockmax; - int info_has_rehash; - unsigned l; - - count_file = 0; - count_hardlink = 0; - count_symlink = 0; - count_dir = 0; - - /* blocks of all array */ - blockmax = parity_allocated_size(state); - - /* clear the info for unused blocks */ - /* and get some other info */ - info_has_rehash = 0; /* if there is a rehash info */ - for (b = 0; b < blockmax; ++b) { - /* if the position is used */ - if (position_is_required(state, b)) { - snapraid_info info = info_get(&state->infoarr, b); - - /* only if there is some info to store */ - if (info) { - if (info_get_rehash(info)) - info_has_rehash = 1; - } - } else { - /* clear any previous info */ - info_set(&state->infoarr, b, 0); - - /* and clear any deleted blocks */ - position_clear_deleted(state, b); - } - } - - sputsl("blksize ", f); - sputu32(state->block_size, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - if (state->hash == HASH_MURMUR3) { - sputsl("checksum murmur3", f); - } else if (state->hash == HASH_SPOOKY2) { - sputsl("checksum spooky2", f); - } else { - /* LCOV_EXCL_START */ - log_fatal("Unexpected hash when writing the content file '%s'.\n", serrorfile(f)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - sputc(' ', f); - sputhex(state->hashseed, HASH_SIZE, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* previous hash only present */ - if (state->prevhash != HASH_UNDEFINED) { - /* if at least one rehash tag found, we have to save the previous hash */ - if (info_has_rehash) { - if (state->prevhash == HASH_MURMUR3) { - sputsl("prevchecksum murmur3", f); - } else if (state->prevhash == HASH_SPOOKY2) { - sputsl("prevchecksum spooky2", f); - } else { - /* LCOV_EXCL_START */ - log_fatal("Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - sputc(' ', f); - sputhex(state->prevhashseed, HASH_SIZE, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - } - - /* for each map */ - for (i = state->maplist; i != 0; i = i->next) { - struct snapraid_map* map = i->data; - struct snapraid_disk* disk; - - /* find the disk for this mapping */ - disk = find_disk(state, map->name); - if (!disk) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for unmapped disk '%s'\n", map->name); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* save the mapping only for not empty disks */ - if (!disk_is_empty(disk, blockmax)) { - sputsl("mapping ", f); - sputs(map->name, f); - sputc(' ', f); - sputu32(map->position, f); - sputc(' ', f); - sputu32(map->total_blocks, f); - sputc(' ', f); - sputu32(map->free_blocks, f); - - /* if there is an uuid, print it */ - if (map->uuid[0]) { - sputc(' ', f); - sputs(map->uuid, f); - } - - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* mark the disk as with mapping */ - disk->mapping_idx = 0; - } else { - /* mark the disk as without mapping */ - disk->mapping_idx = -1; - } - } - - /* for each parity */ - for (l = 0; l < state->level; ++l) { - sputsl("parity ", f); - sputu32(l, f); - sputc(' ', f); - sputu32(state->parity[l].total_blocks, f); - sputc(' ', f); - sputu32(state->parity[l].free_blocks, f); - - /* if there is an uuid, print it */ - if (state->parity[l].uuid[0]) { - sputc(' ', f); - sputs(state->parity[l].uuid, f); - } - - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - - /* for each disk */ - for (i = state->disklist; i != 0; i = i->next) { - tommy_node* j; - struct snapraid_disk* disk = i->data; - - /* if the disk is not mapped, skip it */ - if (disk->mapping_idx < 0) - continue; - - /* for each file */ - for (j = disk->filelist; j != 0; j = j->next) { - struct snapraid_file* file = j->data; - uint64_t size; - uint64_t mtime_sec; - int32_t mtime_nsec; - uint64_t inode; - - size = file->size; - mtime_sec = file->mtime_sec; - mtime_nsec = file->mtime_nsec; - inode = file->inode; - - sputsl("file ", f); - sputs(disk->name, f); - sputc(' ', f); - sputu64(size, f); - sputc(' ', f); - sputu64(mtime_sec, f); - if (mtime_nsec != STAT_NSEC_INVALID) { - sputc('.', f); - sputu32(mtime_nsec, f); - } - sputc(' ', f); - sputu64(inode, f); - sputc(' ', f); - sputs(file->sub, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - /* for each block in the file */ - for (b = 0; b < file->blockmax; ++b) { - struct snapraid_block* block = &file->blockvec[b]; - unsigned block_state = block_state_get(block); - - /* consistency check */ - if (block != fs_par2block_get(disk, block->parity_pos)) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - switch (block_state) { - case BLOCK_STATE_BLK : - sputsl("blk ", f); - break; - case BLOCK_STATE_CHG : - sputsl("chg ", f); - break; - case BLOCK_STATE_REP : - sputsl("rep ", f); - break; - default : - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in state for block %u state %u\n", block->parity_pos, block_state); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - sputu32(block->parity_pos, f); - - sputc(' ', f); - sputhex(block->hash, HASH_SIZE, f); - - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - - ++count_file; - } - - /* for each link */ - for (j = disk->linklist; j != 0; j = j->next) { - struct snapraid_link* link = j->data; - - switch (link_flag_get(link, FILE_IS_LINK_MASK)) { - case FILE_IS_HARDLINK : - sputsl("hardlink ", f); - ++count_hardlink; - break; - case FILE_IS_SYMLINK : - sputsl("symlink ", f); - ++count_symlink; - break; - } - - sputs(disk->name, f); - sputc(' ', f); - sputs(link->sub, f); - sputeol(f); - sputsl("to ", f); - sputs(link->linkto, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - - /* for each dir */ - for (j = disk->dirlist; j != 0; j = j->next) { - struct snapraid_dir* dir = j->data; - - sputsl("dir ", f); - sputs(disk->name, f); - sputc(' ', f); - sputs(dir->sub, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - ++count_dir; - } - - /* deleted blocks of the disk */ - sputsl("hole ", f); - sputs(disk->name, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - for (b = 0; b < blockmax; ++b) { - if (is_block_deleted(disk, b)) { - struct snapraid_block* block = fs_par2block_get(disk, b); - - /* consistency check */ - if (block->parity_pos != b) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in delete position %u/%u!\n", block->parity_pos, b); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - sputsl("off ", f); - sputu32(b, f); - sputc(' ', f); - sputhex(block->hash, HASH_SIZE, f); - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - } - } - - /* write the info for each block */ - for (b = 0; b < blockmax; ++b) { - snapraid_info info = info_get(&state->infoarr, b); - - /* save only stuffs different than 0 */ - if (info != 0) { - sputsl("inf ", f); - - sputu32(b, f); - sputc(' ', f); - sputu32(info, f); - if (info_get_bad(info)) { - sputsl(" bad", f); - } - if (info_get_rehash(info)) { - sputsl(" rehash", f); - } - if (info_get_justsynced(info)) { - sputsl(" justsynced", f); - } - sputeol(f); - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - } - - if (serror(f)) { - /* LCOV_EXCL_START */ - log_fatal("Error writing the content file '%s'. %s.\n", serrorfile(f), strerror(errno)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - msg_verbose("%8u files\n", count_file); - msg_verbose("%8u hardlinks\n", count_hardlink); - msg_verbose("%8u symlinks\n", count_symlink); - msg_verbose("%8u empty dirs\n", count_dir); -} - /** * Flush the file checking the final CRC. * We exploit the fact that the CRC is always stored in the last 4 bytes. @@ -4509,10 +3040,10 @@ void state_read(struct snapraid_state* state) if (c == 'S') { state_read_binary(state, path, f); } else { - state_read_text(state, path, f, st.st_mtime); - - /* force a rewrite to convert to binary */ - state->need_write = 1; + /* LCOV_EXCL_START */ + log_fatal("The text file format is not supported anymore.\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ } sclose(f); @@ -4578,10 +3109,7 @@ void state_write(struct snapraid_state* state) i = i->next; } - if (state->opt.force_content_text) - state_write_text(state, f); - else - state_write_binary(state, f); + state_write_binary(state, f); /* Use the sequence fflush() -> fsync() -> fclose() -> rename() to ensure */ /* than even in a system crash event we have one valid copy of the file. */ diff --git a/cmdline/state.h b/cmdline/state.h index 13f3a177..01ff0133 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -98,7 +98,6 @@ struct snapraid_option { unsigned force_scrub_at; /**< Force scrub for the specified number of blocks. */ int force_scrub_even; /**< Force scrub of all the even blocks. */ int force_content_write; /**< Force the update of the content file. */ - int force_content_text; /**< Force the use of text version of content file. */ int force_scan_winfind; /**< Force the use of FindFirst/Next in Windows to list directories. */ int force_progress; /**< Force the use of the progress status. */ unsigned force_autosave_at; /**< Force autosave at the specified block. */ From f719ee4f5773c15effb0f0a7ac37e6b8402be3ee Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 1 Oct 2015 23:22:21 +0200 Subject: [PATCH 04/12] Switch from deleted blocks to deleted files This will allow an easier management of the parity allocation. --- cmdline/elem.c | 32 +------------------------------- cmdline/elem.h | 17 +---------------- cmdline/scan.c | 21 +++++++++------------ cmdline/state.c | 35 +++++++++++++++++++++++------------ 4 files changed, 34 insertions(+), 71 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index a101233c..d147e353 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -394,36 +394,6 @@ unsigned block_file_size(struct snapraid_block* block, unsigned block_size) return block_size; } -struct snapraid_deleted* deleted_alloc(void) -{ - struct snapraid_deleted* deleted; - - deleted = malloc_nofail(sizeof(struct snapraid_deleted)); - - /* set the state as deleted */ - deleted->block.file_mixed = BLOCK_STATE_DELETED; - - return deleted; -} - -struct snapraid_deleted* deleted_dup(struct snapraid_block* block) -{ - struct snapraid_deleted* deleted; - - deleted = deleted_alloc(); - - /* copy data from deleted block */ - deleted->block.parity_pos = block->parity_pos; - memcpy(deleted->block.hash, block->hash, HASH_SIZE); - - return deleted; -} - -void deleted_free(struct snapraid_deleted* deleted) -{ - free(deleted); -} - struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical) { struct snapraid_file* file; @@ -752,7 +722,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev void disk_free(struct snapraid_disk* disk) { tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free); - tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)deleted_free); + tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)file_free); tommy_hashdyn_done(&disk->inodeset); tommy_hashdyn_done(&disk->pathset); tommy_hashdyn_done(&disk->stampset); diff --git a/cmdline/elem.h b/cmdline/elem.h index 55864715..5f488fe2 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -288,21 +288,6 @@ struct snapraid_dir { tommy_hashdyn_node nodeset; }; -/** - * Deleted entry. - */ -struct snapraid_deleted { - /** - * Deleted block. - * This block is always in state BLOCK_STATE_DELETED, - * and it's used to keep the old hash during a sync. - */ - struct snapraid_block block; - - /* nodes for data structures */ - tommy_node node; -}; - /** * Disk. */ @@ -344,7 +329,7 @@ struct snapraid_disk { tommy_list filelist; /** - * List of all the snapraid_deleted blocks for the disk. + * List of all the deleted file for the disk. * * These files are kept allocated, because the blocks are still referenced in * the ::blockarr. diff --git a/cmdline/scan.c b/cmdline/scan.c index 0c72fda6..25add9c3 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -266,7 +266,6 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil struct snapraid_block* block = fs_file2block_get(disk, file, i); block_off_t parity_pos = fs_file2par_get(disk, file, i); unsigned block_state; - struct snapraid_deleted* deleted; /* adjust the first free position */ /* note that doing all the deletions before alllocations, */ @@ -275,9 +274,6 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil if (disk->first_free_block > parity_pos) disk->first_free_block = parity_pos; - /* allocated a new deleted block from the block we are going to delete */ - deleted = deleted_dup(block); - /* in case we scan after an aborted sync, */ /* we could get also intermediate states */ block_state = block_state_get(block); @@ -299,12 +295,12 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil /* - File is now deleted after the aborted sync */ /* - Sync again, deleting the blocks (exactly here) */ /* with the hash of CHG block not represeting the real parity state */ - hash_invalid_set(deleted->block.hash); + hash_invalid_set(block->hash); } break; case BLOCK_STATE_REP : /* we just don't know the old hash, and then we set it to invalid */ - hash_invalid_set(deleted->block.hash); + hash_invalid_set(block->hash); break; default : /* LCOV_EXCL_START */ @@ -313,14 +309,15 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil /* LCOV_EXCL_STOP */ } - /* insert it in the list of deleted blocks */ - tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); + /* set the block as deleted */ + block_state_set(block, BLOCK_STATE_DELETED); - /* set the deleted block in the block array */ - fs_par2block_set(disk, parity_pos, &deleted->block); + /* remove the file reference */ + block_file_set(block, 0); } - file_free(file); + /* insert it in the list of deleted blocks */ + tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file); } static void scan_file_delayed_allocate(struct snapraid_scan* scan, struct snapraid_file* file) @@ -510,7 +507,7 @@ static void scan_file_insert(struct snapraid_scan* scan, struct snapraid_file* f tommy_hashdyn_insert(&disk->pathset, &file->pathset, file, file_path_hash(file->sub)); tommy_hashdyn_insert(&disk->stampset, &file->stampset, file, file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec)); - /* delayed allocationg of the parity */ + /* delayed allocation of the parity */ scan_file_delayed_allocate(scan, file); } diff --git a/cmdline/state.c b/cmdline/state.c index 10db227d..e15a3f7b 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1925,6 +1925,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } else if (c == 'h') { /* hole */ uint32_t v_pos; + uint32_t i; struct snapraid_disk* disk; uint32_t mapping; @@ -1941,6 +1942,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST v_pos = 0; while (v_pos < blockmax) { uint32_t v_count; + struct snapraid_file* deleted; ret = sgetb32(f, &v_count); if (ret < 0) { @@ -1964,17 +1966,29 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST switch (c) { case 'o' : /* if it's a run of deleted blocks */ - while (v_count) { - struct snapraid_deleted* deleted; - /* allocate a new deleted block */ - deleted = deleted_alloc(); + /* allocate the deleted file */ + deleted = file_alloc(state->block_size, "", v_count * state->block_size, 0, 0, 0, 0); + + /* insert it in the list of deleted files */ + tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted); + + /* process all blocks */ + i = 0; + while (v_count) { + struct snapraid_block* block = &deleted->blockvec[i]; + + /* set the block as deleted */ + block_state_set(block, BLOCK_STATE_DELETED); + + /* remove the file reference */ + block_file_set(block, 0); /* set the position */ - deleted->block.parity_pos = v_pos; + block->parity_pos = v_pos; /* read the hash */ - ret = sread(f, deleted->block.hash, HASH_SIZE); + ret = sread(f, block->hash, HASH_SIZE); if (ret < 0) { /* LCOV_EXCL_START */ decoding_error(path, f); @@ -1985,7 +1999,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* if we are clearing undeterminated hashes */ if (state->clear_past_hash) { /* set the hash value to INVALID */ - hash_invalid_set(deleted->block.hash); + hash_invalid_set(block->hash); } /* we must not overwrite existing blocks */ @@ -1994,23 +2008,20 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos); if (state->opt.skip_content_check) { log_fatal("Overriding as used.\n"); - deleted_free(deleted); } else { decoding_error(path, f); exit(EXIT_FAILURE); } /* LCOV_EXCL_STOP */ } else { - /* insert it in the list of deleted blocks */ - tommy_list_insert_tail(&disk->deletedlist, &deleted->node, deleted); - /* insert the block in the block array */ - fs_par2block_set(disk, v_pos, &deleted->block); + fs_par2block_set(disk, v_pos, block); } /* go to next block */ ++v_pos; --v_count; + ++i; } break; case 'O' : From fa775d24a8cc92c37ed24bd7b83f9c82dadd88d4 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 3 Oct 2015 10:43:02 +0200 Subject: [PATCH 05/12] More interface changes to prepare to the use of tree mapping --- cmdline/check.c | 119 ++++++++++++++++------------ cmdline/dry.c | 9 ++- cmdline/elem.c | 97 ++++++++++++++--------- cmdline/elem.h | 201 ++++++++++++++++++----------------------------- cmdline/handle.c | 12 +-- cmdline/handle.h | 4 +- cmdline/scan.c | 7 +- cmdline/scrub.c | 37 ++++----- cmdline/search.c | 8 +- cmdline/search.h | 2 +- cmdline/state.c | 21 +++-- cmdline/sync.c | 92 ++++++++++------------ 12 files changed, 295 insertions(+), 314 deletions(-) diff --git a/cmdline/check.c b/cmdline/check.c index c99bf893..878e514f 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -56,26 +56,26 @@ struct failed_struct { int is_outofdate; unsigned index; /**< Index of the failed block. */ - struct snapraid_block* block; /**< The failed block, or BLOCK_DELETED for a deleted block */ - struct snapraid_handle* handle; /**< The file containing the failed block, or 0 for a deleted block */ + struct snapraid_block* block; /**< The failed block */ + struct snapraid_disk* disk; /**< The failed disk. */ + struct snapraid_file* file; /**< The failed file. 0 for DELETED block. */ + block_off_t file_pos; /**< Offset inside the file */ + struct snapraid_handle* handle; /**< The handle containing the failed block, or 0 for a DELETED block */ }; /** * Checks if a block hash matches the specified buffer. * Return ==0 if equal */ -static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned char* buffer, unsigned char* buffer_zero) +static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_block* block, unsigned pos_size, unsigned char* buffer, unsigned char* buffer_zero) { unsigned char hash[HASH_SIZE]; - unsigned size; - - size = block_file_size(block, state->block_size); /* now compute the hash of the valid part */ if (rehash) { - memhash(state->prevhash, state->prevhashseed, hash, buffer, size); + memhash(state->prevhash, state->prevhashseed, hash, buffer, pos_size); } else { - memhash(state->hash, state->hashseed, hash, buffer, size); + memhash(state->hash, state->hashseed, hash, buffer, pos_size); } /* compare the hash */ @@ -84,8 +84,8 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl } /* compare to the end of the block */ - if (size < state->block_size) { - if (memcmp(buffer + size, buffer_zero + size, state->block_size - size) != 0) { + if (pos_size < state->block_size) { + if (memcmp(buffer + pos_size, buffer_zero + pos_size, state->block_size - pos_size) != 0) { return -1; } } @@ -111,7 +111,8 @@ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned d && block_has_updated_hash(failed[failed_map[j]].block) ) { /* if a hash doesn't match, fail the check */ - if (blockcmp(state, rehash, failed[failed_map[j]].block, buffer[failed[failed_map[j]].index], buffer_zero) != 0) { + unsigned pos_size = file_block_size(failed[failed_map[j]].file, failed[failed_map[j]].file_pos, state->block_size); + if (blockcmp(state, rehash, failed[failed_map[j]].block, pos_size, buffer[failed[failed_map[j]].index], buffer_zero) != 0) { log_tag("hash_error: Hash mismatch on entry %u\n", failed_map[j]); return 0; } @@ -188,7 +189,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u for (i = 0; i < failed_count; ++i) { /* if we are expected to recover this block */ if (!failed[failed_map[i]].is_outofdate - /* if the block has a hash to check */ + /* if the block has a hash to check */ && block_has_updated_hash(failed[failed_map[i]].block) ) has_hash = 1; @@ -331,14 +332,12 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign else data = "good"; - if (failed[j].handle) { - const char* disk; - const char* sub; - struct snapraid_handle* handle = failed[j].handle; - disk = handle->disk->name; - sub = block_file_get(block)->sub; + if (failed[j].file) { + struct snapraid_disk* disk = failed[j].disk; + struct snapraid_file* file = failed[j].file; + block_off_t file_pos = failed[j].file_pos; - log_tag("entry:%u:%s:%s:%s:%s:%s:%u:\n", j, desc, hash, data, disk, esc(sub), block_file_pos(block)); + log_tag("entry:%u:%s:%s:%s:%s:%s:%u:\n", j, desc, hash, data, disk->name, esc(file->sub), file_pos); } else { log_tag("entry:%u:%s:%s:%s:\n", j, desc, hash, data); } @@ -372,7 +371,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP) /* try to fetch the block using the known hash */ && (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0 - || state_search_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0) + || state_search_fetch(state, rehash, failed[j].file, failed[j].file_pos, failed[j].block, buffer[failed[j].index]) == 0) ) { /* we already have corrected it! */ log_tag("hash_import: Fixed entry %u\n", j); @@ -442,7 +441,8 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign /* instead, if the hash matches, it could be either that the */ /* block after the sync has this hash, or that */ /* we restored the block before the 'sync'. */ - if (blockcmp(state, rehash, failed[j].block, buffer[failed[j].index], buffer_zero) == 0) { + unsigned pos_size = file_block_size(failed[j].file, failed[j].file_pos, state->block_size); + if (blockcmp(state, rehash, failed[j].block, pos_size, buffer[failed[j].index], buffer_zero) == 0) { /* it may contain garbage */ failed[j].is_outofdate = 1; @@ -610,6 +610,7 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s struct snapraid_disk* disk; struct snapraid_file* collide_file; struct snapraid_file* file; + block_off_t file_pos; char path[PATH_MAX]; uint64_t inode; @@ -625,11 +626,11 @@ static int file_post(struct snapraid_state* state, int fix, unsigned i, struct s continue; } - file = block_file_get(block); + file = fs_par2file_get(disk, i, &file_pos); pathprint(path, sizeof(path), "%s%s", disk->dir, file->sub); /* if it isn't the last block in the file */ - if (!block_is_last(block)) { + if (!file_block_is_last(file, file_pos)) { /* nothing to do */ continue; } @@ -823,11 +824,12 @@ static int block_is_enabled(struct snapraid_state* state, block_off_t i, struct /* try to recover all files, even the ones without hash */ /* because in some cases we can recover also them */ - if (block_has_file(block) - && !file_flag_has(block_file_get(block), FILE_IS_EXCLUDED) /* only if the file is not filtered out */ - ) { - one_tocheck = 1; - break; + if (block_has_file(block)) { + struct snapraid_file* file = fs_par2file_get(handle[j].disk, i, 0); + if (!file_flag_has(file, FILE_IS_EXCLUDED)) { /* only if the file is not filtered out */ + one_tocheck = 1; + break; + } } } @@ -932,6 +934,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna struct snapraid_disk* disk; struct snapraid_block* block; struct snapraid_file* file; + block_off_t file_pos; unsigned block_state; /* if the disk position is not used */ @@ -968,17 +971,20 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna /* store it in the failed set, because potentially */ /* the parity may be still computed with the previous content */ - failed[failed_count].is_bad = 0; + failed[failed_count].is_bad = 0; /* note that is_bad==0 <=> file==0 */ failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; + failed[failed_count].disk = disk; + failed[failed_count].file = 0; + failed[failed_count].file_pos = 0; failed[failed_count].handle = 0; ++failed_count; continue; } /* get the file of this block */ - file = block_file_get(block); + file = fs_par2file_get(disk, i, &file_pos); /* if we are only hashing, we can skip excluded files and don't even read them */ if (state->opt.auditonly && file_flag_has(file, FILE_IS_EXCLUDED)) { @@ -1036,10 +1042,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; + failed[failed_count].disk = disk; + failed[failed_count].file = file; + failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; - log_tag("error:%u:%s:%s: Open error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block)); + log_tag("error:%u:%s:%s: Open error at position %u\n", i, disk->name, esc(file->sub), file_pos); ++error; continue; } @@ -1050,9 +1059,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna && !file_flag_has(file, FILE_IS_EXCLUDED)) { /* check if the file is changed */ - if (handle[j].st.st_size != block_file_get(block)->size - || handle[j].st.st_mtime != block_file_get(block)->mtime_sec - || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec + if (handle[j].st.st_size != file->size + || handle[j].st.st_mtime != file->mtime_sec + || STAT_NSEC(&handle[j].st) != file->mtime_nsec /* don't check the inode to support filesystem without persistent inodes */ ) { /* report that the file is not synced */ @@ -1093,7 +1102,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna } /* read from the file */ - read_size = handle_read(&handle[j], block, buffer[j], state->block_size, + read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, log_error, state->opt.expected_missing ? log_expected : 0); if (read_size == -1) { /* save the failed block for the check/fix */ @@ -1101,10 +1110,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; + failed[failed_count].disk = disk; + failed[failed_count].file = file; + failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; - log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block)); + log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), file_pos); ++error; continue; } @@ -1121,6 +1133,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; + failed[failed_count].disk = disk; + failed[failed_count].file = file; + failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; continue; @@ -1144,10 +1159,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; + failed[failed_count].disk = disk; + failed[failed_count].file = file; + failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; - log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), block_file_pos(block), diff); + log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff); ++error; continue; } @@ -1160,6 +1178,9 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; failed[failed_count].block = block; + failed[failed_count].disk = disk; + failed[failed_count].file = file; + failed[failed_count].file_pos = file_pos; failed[failed_count].handle = &handle[j]; ++failed_count; continue; @@ -1206,13 +1227,13 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna /* print a list of all the errors in files */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad) - log_tag("unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block)); + log_tag("unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].disk->name, esc(failed[j].file->sub), failed[j].file_pos); } /* keep track of damaged files */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad) - file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED); + file_flag_set(failed[j].file, FILE_IS_DAMAGED); } } else { /* now counts partial recovers */ @@ -1224,7 +1245,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad && failed[j].is_outofdate) { ++partial_recover_error; - log_tag("unrecoverable:%u:%s:%s: Unrecoverable unsynced error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block)); + log_tag("unrecoverable:%u:%s:%s: Unrecoverable unsynced error at position %u\n", i, failed[j].disk->name, esc(failed[j].file->sub), failed[j].file_pos); } } if (partial_recover_error != 0) { @@ -1258,15 +1279,15 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna continue; /* do not fix if the file is excluded */ - if (file_flag_has(block_file_get(failed[j].block), FILE_IS_EXCLUDED) - || (state->opt.syncedonly && file_flag_has(block_file_get(failed[j].block), FILE_IS_UNSYNCED))) + if (file_flag_has(failed[j].file, FILE_IS_EXCLUDED) + || (state->opt.syncedonly && file_flag_has(failed[j].file, FILE_IS_UNSYNCED))) continue; - ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size); + ret = handle_write(failed[j].handle, failed[j].file_pos, buffer[failed[j].index], state->block_size); if (ret == -1) { /* LCOV_EXCL_START */ /* mark the file as damaged */ - file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED); + file_flag_set(failed[j].file, FILE_IS_DAMAGED); if (errno == EACCES) { log_fatal("WARNING! Please give write permission to the file.\n"); @@ -1283,15 +1304,15 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna /* if we are not sure that the recovered content is uptodate */ if (failed[j].is_outofdate) { /* mark the file as damaged */ - file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED); + file_flag_set(failed[j].file, FILE_IS_DAMAGED); continue; } /* mark the file as containing some fixes */ /* note that it could be also marked as damaged in other iterations */ - file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED); + file_flag_set(failed[j].file, FILE_IS_FIXED); - log_tag("fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].handle->disk->name, esc(block_file_get(failed[j].block)->sub), block_file_pos(failed[j].block)); + log_tag("fixed:%u:%s:%s: Fixed data error at position %u\n", i, failed[j].disk->name, esc(failed[j].file->sub), failed[j].file_pos); ++recovered_error; } @@ -1323,7 +1344,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna /* meaning that we could fix this file if we try */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad) { - file_flag_set(block_file_get(failed[j].block), FILE_IS_FIXED); + file_flag_set(failed[j].file, FILE_IS_FIXED); } } } @@ -1333,7 +1354,7 @@ static int state_check_process(struct snapraid_state* state, int fix, struct sna /* to report that the file is damaged, and we don't know if we can fix it */ for (j = 0; j < failed_count; ++j) { if (failed[j].is_bad) { - file_flag_set(block_file_get(failed[j].block), FILE_IS_DAMAGED); + file_flag_set(failed[j].file, FILE_IS_DAMAGED); } } } diff --git a/cmdline/dry.c b/cmdline/dry.c index aaa32349..c9a460d2 100644 --- a/cmdline/dry.c +++ b/cmdline/dry.c @@ -58,9 +58,10 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit /* for each disk, process the block */ for (j = 0; j < diskmax; ++j) { int read_size; - struct snapraid_block* block = BLOCK_EMPTY; + struct snapraid_block* block; struct snapraid_disk* disk = handle[j].disk; struct snapraid_file* file; + block_off_t file_pos; if (!disk) { /* if no disk, nothing to do */ @@ -75,7 +76,7 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit } /* get the file of this block */ - file = block_file_get(block); + file = fs_par2file_get(disk, i, &file_pos); /* until now is CPU */ state_usage_cpu(state); @@ -108,9 +109,9 @@ static int state_dry_process(struct snapraid_state* state, struct snapraid_parit } /* read from the file */ - read_size = handle_read(&handle[j], block, buffer_aligned, state->block_size, log_error, 0); + read_size = handle_read(&handle[j], file_pos, buffer_aligned, state->block_size, log_error, 0); if (read_size == -1) { - log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), block_file_pos(block)); + log_tag("error:%u:%s:%s: Read error at position %u\n", i, disk->name, esc(file->sub), file_pos); ++error; continue; } diff --git a/cmdline/elem.c b/cmdline/elem.c index d147e353..8381b8d7 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -355,43 +355,22 @@ int filter_content(tommy_list* contentlist, const char* path) return 0; } -block_off_t block_file_pos(struct snapraid_block* block) +/** + * Sets the file containing the block. + */ +static void block_file_set(struct snapraid_block* block, struct snapraid_file* file) { - struct snapraid_file* file = block_file_get(block); + uintptr_t ptr = (uintptr_t)file; - if (block < file->blockvec || block >= file->blockvec + file->blockmax) { + /* ensure that the pointer doesn't use the flag space */ + if ((ptr & (uintptr_t)BLOCK_STATE_MASK) != 0) { /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in block %u ownership\n", block->parity_pos); + log_fatal("Internal error for pointer not aligned\n"); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } - return block - file->blockvec; -} - -int block_is_last(struct snapraid_block* block) -{ - struct snapraid_file* file = block_file_get(block); - - return block == file->blockvec + file->blockmax - 1; -} - -unsigned block_file_size(struct snapraid_block* block, unsigned block_size) -{ - block_off_t pos = block_file_pos(block); - - /* if it's the last block */ - if (pos + 1 == block_file_get(block)->blockmax) { - unsigned remainder; - if (block_file_get(block)->size == 0) - return 0; - remainder = block_file_get(block)->size % block_size; - if (remainder == 0) - remainder = block_size; - return remainder; - } - - return block_size; + block->private_file_mixed = (block->private_file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr; } struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical) @@ -411,8 +390,8 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); for (i = 0; i < file->blockmax; ++i) { - file->blockvec[i].parity_pos = POS_INVALID; - file->blockvec[i].file_mixed = 0; + file->blockvec[i].private_parity_pos = POS_INVALID; + file->blockvec[i].private_file_mixed = 0; /* set the file */ block_file_set(&file->blockvec[i], file); @@ -442,8 +421,8 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); for (i = 0; i < file->blockmax; ++i) { - file->blockvec[i].file_mixed = copy->blockvec[i].file_mixed; - file->blockvec[i].parity_pos = copy->blockvec[i].parity_pos; + file->blockvec[i].private_file_mixed = copy->blockvec[i].private_file_mixed; + file->blockvec[i].private_parity_pos = copy->blockvec[i].private_parity_pos; memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE); /* set the file to overwrite the copied one */ @@ -515,6 +494,37 @@ const char* file_name(const struct snapraid_file* file) return r; } +unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size) +{ + /* if it's the last block */ + if (file_pos + 1 == file->blockmax) { + unsigned remainder; + if (file->size == 0) + return 0; + remainder = file->size % block_size; + if (remainder == 0) + remainder = block_size; + return remainder; + } + + return block_size; +} + +int file_block_is_last(struct snapraid_file* file, block_off_t file_pos) +{ + if (file_pos == 0 && file->blockmax == 0) + return 1; + + if (file_pos >= file->blockmax) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in file block position\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + return file_pos == file->blockmax - 1; +} + int file_inode_compare_to_arg(const void* void_arg, const void* void_data) { const uint64_t* arg = void_arg; @@ -745,20 +755,29 @@ void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct assert(file_pos < file->blockmax); - file->blockvec[file_pos].parity_pos = parity_pos; + file->blockvec[file_pos].private_parity_pos = parity_pos; } struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) { struct snapraid_block* block = fs_par2block_get(disk, parity_pos); + struct snapraid_file* file = (struct snapraid_file*)(block->private_file_mixed & ~(uintptr_t)BLOCK_STATE_MASK); if (block == BLOCK_EMPTY) return 0; - if (file_pos) - *file_pos = block_file_pos(block); + if (file_pos) { + if (block < file->blockvec || block >= file->blockvec + file->blockmax) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in block %u ownership\n", block->private_parity_pos); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } - return block_file_get(block); + *file_pos = block - file->blockvec; + } + + return file; } block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) @@ -767,7 +786,7 @@ block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* fi assert(file_pos < file->blockmax); - return file->blockvec[file_pos].parity_pos; + return file->blockvec[file_pos].private_parity_pos; } struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) diff --git a/cmdline/elem.h b/cmdline/elem.h index 5f488fe2..e0bcfe3c 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -68,6 +68,17 @@ struct snapraid_filter { tommy_node node; /**< Next node in the list. */ }; +/** + * Block pointer used to represent unused blocks. + */ +#define BLOCK_EMPTY 0 + +/** + * This block is an empty one. + * Note that an empty block is represent with ::BLOCK_EMPTY. + */ +#define BLOCK_STATE_EMPTY 0 + /** * The block has both the hash and the parity computed. * This is the normal state of a saved block. @@ -147,27 +158,15 @@ struct snapraid_filter { */ #define BLOCK_STATE_MASK 7 -/** - * This block is an empty one. - * Note that this state cannot be stored inside the block, - * and it's represented by the block ::BLOCK_EMPTY. - */ -#define BLOCK_STATE_EMPTY 8 - /** * Block of a file. */ struct snapraid_block { - uintptr_t file_mixed; /**< Back pointer to the file owning this block, mixed with some flags. */ - block_off_t parity_pos; /**< Position of the block in the parity. */ + uintptr_t private_file_mixed; /**< Back pointer to the file owning this block, mixed with some flags. */ + block_off_t private_parity_pos; /**< Position of the block in the parity. */ unsigned char hash[HASH_SIZE]; /**< Hash of the block. */ }; -/** - * Block pointer used to mark unused blocks. - */ -#define BLOCK_EMPTY 0 - /** * If a file is present in the disk. * It's used only in scan to detect present and missing files. @@ -465,85 +464,6 @@ int filter_dir(tommy_list* filterlist, struct snapraid_filter** reason, const ch */ int filter_content(tommy_list* contentlist, const char* path); -/** - * Gets the file containing the block. - */ -static inline struct snapraid_file* block_file_get(struct snapraid_block* block) -{ - return (struct snapraid_file*)(block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK); -} - -/** - * Sets the file containing the block. - */ -static inline void block_file_set(struct snapraid_block* block, struct snapraid_file* file) -{ - uintptr_t ptr = (uintptr_t)file; - - /* ensure that the pointer doesn't use the flag space */ - if ((ptr & (uintptr_t)BLOCK_STATE_MASK) != 0) { - /* LCOV_EXCL_START */ - log_fatal("Internal error for pointer not aligned\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - block->file_mixed = (block->file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr; -} - -/** - * Get the state of the block. - * - * For this function, it's allowed to pass a NULL block - * pointer than results in the BLOCK_STATE_EMPTY state. - */ -static inline unsigned block_state_get(const struct snapraid_block* block) -{ - if (block == BLOCK_EMPTY) - return BLOCK_STATE_EMPTY; - - return block->file_mixed & BLOCK_STATE_MASK; -} - -static inline void block_state_set(struct snapraid_block* block, unsigned state) -{ - /* ensure that the state can be stored inside the file pointer */ - if ((state & BLOCK_STATE_MASK) != state) { - /* LCOV_EXCL_START */ - log_fatal("Internal error when setting the block state %u\n", state); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - block->file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK; - block->file_mixed |= state & BLOCK_STATE_MASK; -} - -/** - * Checks if the specified block has an updated hash. - * - * Note that EMPTY / CHG / DELETED return 0. - */ -static inline int block_has_updated_hash(const struct snapraid_block* block) -{ - unsigned state = block_state_get(block); - - return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP; -} - -/** - * Checks if the specified block has a past hash, - * i.e. the hash of the data that it's now overwritten or lost. - * - * Note that EMPTY / BLK / REP return 0. - */ -static inline int block_has_past_hash(const struct snapraid_block* block) -{ - unsigned state = block_state_get(block); - - return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED; -} - /** * Checks if the specified hash is invalid. * @@ -600,6 +520,59 @@ static inline void hash_zero_set(unsigned char* hash) memset(hash, 0xFF, HASH_SIZE); } +/** + * Get the state of the block. + * + * For this function, it's allowed to pass a NULL block + * pointer than results in the BLOCK_STATE_EMPTY state. + */ +static inline unsigned block_state_get(const struct snapraid_block* block) +{ + if (block == BLOCK_EMPTY) + return BLOCK_STATE_EMPTY; + + return block->private_file_mixed & BLOCK_STATE_MASK; +} + +static inline void block_state_set(struct snapraid_block* block, unsigned state) +{ + /* ensure that the state can be stored inside the file pointer */ + if ((state & BLOCK_STATE_MASK) != state) { + /* LCOV_EXCL_START */ + log_fatal("Internal error when setting the block state %u\n", state); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + block->private_file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK; + block->private_file_mixed |= state & BLOCK_STATE_MASK; +} + +/** + * Checks if the specified block has an updated hash. + * + * Note that EMPTY / CHG / DELETED return 0. + */ +static inline int block_has_updated_hash(const struct snapraid_block* block) +{ + unsigned state = block_state_get(block); + + return state == BLOCK_STATE_BLK || state == BLOCK_STATE_REP; +} + +/** + * Checks if the specified block has a past hash, + * i.e. the hash of the data that it's now overwritten or lost. + * + * Note that EMPTY / BLK / REP return 0. + */ +static inline int block_has_past_hash(const struct snapraid_block* block) +{ + unsigned state = block_state_get(block); + + return state == BLOCK_STATE_CHG || state == BLOCK_STATE_DELETED; +} + /** * Checks if the specified block is part of a file. * @@ -638,37 +611,6 @@ static inline int block_has_file_and_valid_parity(const struct snapraid_block* b return state == BLOCK_STATE_BLK; } -/** - * Gets the relative position of a block inside the file. - */ -block_off_t block_file_pos(struct snapraid_block* block); - -/** - * Checks if the block is the last in the file. - */ -int block_is_last(struct snapraid_block* block); - -/** - * Gets the size in bytes of the block. - * If it's the last block of a file it could be less than block_size. - */ -unsigned block_file_size(struct snapraid_block* block, unsigned block_size); - -/** - * Allocates a deleted block. - */ -struct snapraid_deleted* deleted_alloc(void); - -/** - * Allocates a deleted block from a real one. - */ -struct snapraid_deleted* deleted_dup(struct snapraid_block* block); - -/** - * Frees a deleted block. - */ -void deleted_free(struct snapraid_deleted* deleted); - static inline int file_flag_has(const struct snapraid_file* file, unsigned mask) { return (file->flag & mask) == mask; @@ -714,6 +656,17 @@ void file_copy(struct snapraid_file* src_file, struct snapraid_file* dest_file); */ const char* file_name(const struct snapraid_file* file); +/** + * Checks if the block is the last in the file. + */ +int file_block_is_last(struct snapraid_file* file, block_off_t file_pos); + +/** + * Gets the size in bytes of the block. + * If it's the last block of a file it could be less than block_size. + */ +unsigned file_block_size(struct snapraid_file* file, block_off_t file_pos, unsigned block_size); + /** * Compares a file with an inode. */ diff --git a/cmdline/handle.c b/cmdline/handle.c index bfb0506c..0f211814 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -248,14 +248,14 @@ int handle_close(struct snapraid_handle* handle) return 0; } -int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing) +int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing) { ssize_t read_ret; data_off_t offset; unsigned read_size; unsigned count; - offset = block_file_pos(block) * (data_off_t)block_size; + offset = file_pos * (data_off_t)block_size; if (!out_missing) out_missing = out; @@ -270,7 +270,7 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un return -1; } - read_size = block_file_size(block, block_size); + read_size = file_block_size(handle->file, file_pos, block_size); #if !HAVE_PREAD if (lseek(handle->f, offset, SEEK_SET) != offset) { @@ -323,15 +323,15 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un return read_size; } -int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size) +int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size) { ssize_t write_ret; data_off_t offset; unsigned write_size; - offset = block_file_pos(block) * (data_off_t)block_size; + offset = file_pos * (data_off_t)block_size; - write_size = block_file_size(block, block_size); + write_size = file_block_size(handle->file, file_pos, block_size); #if HAVE_PWRITE write_ret = pwrite(handle->f, block_buffer, write_size, offset); diff --git a/cmdline/handle.h b/cmdline/handle.h index 7ef2432a..9a58d5b2 100644 --- a/cmdline/handle.h +++ b/cmdline/handle.h @@ -64,12 +64,12 @@ int handle_close(struct snapraid_handle* handle); * Read a block from a file. * If the read block is shorter, it's padded with 0. */ -int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing); +int handle_read(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size, fptr* out, fptr* out_missing); /** * Writes a block to a file. */ -int handle_write(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size); +int handle_write(struct snapraid_handle* handle, block_off_t file_pos, unsigned char* block_buffer, unsigned block_size); /** * Changes the modification time of the file to the saved value. diff --git a/cmdline/scan.c b/cmdline/scan.c index 25add9c3..0ab47b0f 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -178,7 +178,7 @@ static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* ++parity_pos; /* set the position */ - block->parity_pos = parity_pos; + block->private_parity_pos = parity_pos; /* get block specific info */ info = info_get(&state->infoarr, parity_pos); @@ -304,16 +304,13 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil break; default : /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in deallocating for block %u state %u\n", block->parity_pos, block_state); + log_fatal("Internal inconsistency in deallocating for block %u state %u\n", parity_pos, block_state); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } /* set the block as deleted */ block_state_set(block, BLOCK_STATE_DELETED); - - /* remove the file reference */ - block_file_set(block, 0); } /* insert it in the list of deleted blocks */ diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 89cae70d..1561b946 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -184,6 +184,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par struct snapraid_block* block; int file_is_unsynced; struct snapraid_disk* disk = handle[j].disk; + struct snapraid_file* file; + block_off_t file_pos; /* if the file on this disk is synced */ /* if not, silent errors are assumed as expected error */ @@ -207,6 +209,9 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par continue; } + /* get the file of this block */ + file = fs_par2file_get(disk, i, &file_pos); + /* if the block is unsynced, errors are expected */ if (block_has_invalid_parity(block)) { /* report that the block and the file are not synced */ @@ -219,16 +224,16 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par state_usage_cpu(state); /* if the file is different than the current one, close it */ - if (handle[j].file != 0 && handle[j].file != block_file_get(block)) { + if (handle[j].file != 0 && handle[j].file != file) { /* keep a pointer at the file we are going to close for error reporting */ - struct snapraid_file* file = handle[j].file; + struct snapraid_file* report = handle[j].file; ret = handle_close(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ /* This one is really an unexpected error, because we are only reading */ /* and closing a descriptor should never fail */ if (errno == EIO) { - log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno)); log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to scrub.\n"); log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -236,7 +241,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par goto bail; } - log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno)); log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to scrub.\n"); log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -246,10 +251,8 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } } - ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0); + ret = handle_open(&handle[j], file, state->file_mode, log_error, 0); if (ret == -1) { - /* file we have tried to open for error reporting */ - struct snapraid_file* file = block_file_get(block); if (errno == EIO) { /* LCOV_EXCL_START */ log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); @@ -268,9 +271,9 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par } /* check if the file is changed */ - if (handle[j].st.st_size != block_file_get(block)->size - || handle[j].st.st_mtime != block_file_get(block)->mtime_sec - || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec + if (handle[j].st.st_size != file->size + || handle[j].st.st_mtime != file->mtime_sec + || STAT_NSEC(&handle[j].st) != file->mtime_nsec /* don't check the inode to support filesystem without persistent inodes */ ) { /* report that the block and the file are not synced */ @@ -283,12 +286,10 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par /* from the last sync, as we are expected to return errors if running */ /* in an unsynced array. This is just like the check command. */ - read_size = handle_read(&handle[j], block, buffer[j], state->block_size, log_error, 0); + read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, log_error, 0); if (read_size == -1) { - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); if (errno == EIO) { - log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno)); + log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno)); if (io_error >= state->opt.io_error_limit) { /* LCOV_EXCL_START */ log_fatal("DANGER! Too many input/output read error in a data disk, it isn't possible to scrub.\n"); @@ -299,13 +300,13 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par /* LCOV_EXCL_STOP */ } - log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block)); + log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, file_pos); ++io_error; io_error_on_this_block = 1; continue; } - log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno)); + log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno)); ++error; error_on_this_block = 1; continue; @@ -332,14 +333,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par if (memcmp(hash, block->hash, HASH_SIZE) != 0) { unsigned diff = memdiff(hash, block->hash, HASH_SIZE); - log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(handle[j].file->sub), block_file_pos(block), diff); + log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff); /* it's a silent error only if we are dealing with synced files */ if (file_is_unsynced) { ++error; error_on_this_block = 1; } else { - log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, block_file_pos(block), diff); + log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, file_pos, diff); ++silent_error; silent_error_on_this_block = 1; } diff --git a/cmdline/search.c b/cmdline/search.c index 031c4f03..cd34c136 100644 --- a/cmdline/search.c +++ b/cmdline/search.c @@ -130,7 +130,7 @@ int search_file_compare(const void* void_arg, const void* void_data) return 0; } -int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_block* missing_block, unsigned char* buffer) +int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer) { struct snapraid_search_file* file; tommy_uint32_t file_hash; @@ -138,10 +138,10 @@ int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapra arg.state = state; arg.block = missing_block; - arg.file = block_file_get(missing_block); + arg.file = missing_file; arg.buffer = buffer; - arg.offset = state->block_size * (data_off_t)block_file_pos(missing_block); - arg.read_size = block_file_size(missing_block, state->block_size); + arg.offset = state->block_size * (data_off_t)missing_file_pos; + arg.read_size = file_block_size(missing_file, missing_file_pos, state->block_size); arg.prevhash = prevhash; file_hash = file_stamp_hash(arg.file->size, arg.file->mtime_sec, arg.file->mtime_nsec); diff --git a/cmdline/search.h b/cmdline/search.h index 6664b625..8226a1d7 100644 --- a/cmdline/search.h +++ b/cmdline/search.h @@ -48,7 +48,7 @@ void search_file_free(struct snapraid_search_file* file); * Fetch a file from the size, timestamp and name. * Return ==0 if the block is found, and copied into buffer. */ -int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_block* missing_block, unsigned char* buffer); +int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapraid_file* missing_file, block_off_t missing_file_pos, struct snapraid_block* missing_block, unsigned char* buffer); /** * Import files from the specified directory. diff --git a/cmdline/state.c b/cmdline/state.c index e15a3f7b..2705506b 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1723,7 +1723,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST while (v_count) { struct snapraid_block* block = &file->blockvec[v_idx]; - if (block->parity_pos != POS_INVALID) { + if (block->private_parity_pos != POS_INVALID) { /* LCOV_EXCL_START */ decoding_error(path, f); log_fatal("Internal inconsistency in block position!\n"); @@ -1753,7 +1753,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } - block->parity_pos = v_pos; + block->private_parity_pos = v_pos; /* read the hash only for 'blk/chg/rep', and not for 'new' */ if (c != 'n') { @@ -1981,11 +1981,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* set the block as deleted */ block_state_set(block, BLOCK_STATE_DELETED); - /* remove the file reference */ - block_file_set(block, 0); - /* set the position */ - block->parity_pos = v_pos; + block->private_parity_pos = v_pos; /* read the hash */ ret = sread(f, block->hash, HASH_SIZE); @@ -2669,7 +2666,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) begin = 0; while (begin < file->blockmax) { unsigned block_state = block_state_get(blockvec + begin); - uint32_t v_pos = blockvec[begin].parity_pos; + uint32_t v_pos = blockvec[begin].private_parity_pos; uint32_t v_count; block_off_t end; @@ -2678,7 +2675,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) end = begin + 1; while (end < file->blockmax && block_state == block_state_get(blockvec + end) - && blockvec[end - 1].parity_pos + 1 == blockvec[end].parity_pos + && blockvec[end - 1].private_parity_pos + 1 == blockvec[end].private_parity_pos ) { ++end; } @@ -2710,9 +2707,9 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) struct snapraid_block* block = blockvec + b; /* consistency check */ - if (block != fs_par2block_get(disk, block->parity_pos)) { + if (block != fs_par2block_get(disk, block->private_parity_pos)) { /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->parity_pos, block_state, file->sub); + log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->private_parity_pos, block_state, file->sub); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -2812,9 +2809,9 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) struct snapraid_block* block = fs_par2block_get(disk, begin); /* consistency check */ - if (block->parity_pos != begin) { + if (block->private_parity_pos != begin) { /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in delete position %u/%u!\n", block->parity_pos, begin); + log_fatal("Internal inconsistency in delete position %u/%u!\n", block->private_parity_pos, begin); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/sync.c b/cmdline/sync.c index 936a42ef..71c483b4 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -100,6 +100,8 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta int read_size; unsigned char hash[HASH_SIZE]; unsigned block_state; + struct snapraid_file* file; + block_off_t file_pos; block = fs_par2block_get(disk, i); @@ -110,6 +112,9 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta if (block_state != BLOCK_STATE_REP && block_state != BLOCK_STATE_CHG) continue; + /* get the file of this block */ + file = fs_par2file_get(disk, i, &file_pos); + /* get block specific info */ info = info_get(&state->infoarr, i); @@ -120,16 +125,16 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta state_usage_cpu(state); /* if the file is different than the current one, close it */ - if (handle[j].file != 0 && handle[j].file != block_file_get(block)) { + if (handle[j].file != 0 && handle[j].file != file) { /* keep a pointer at the file we are going to close for error reporting */ - struct snapraid_file* file = handle[j].file; + struct snapraid_file* report = handle[j].file; ret = handle_close(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ /* This one is really an unexpected error, because we are only reading */ /* and closing a descriptor should never fail */ if (errno == EIO) { - log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno)); log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -137,7 +142,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta goto bail; } - log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno)); log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -147,10 +152,8 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta } } - ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0); + ret = handle_open(&handle[j], file, state->file_mode, log_error, 0); if (ret == -1) { - /* file we have tried to open for error reporting */ - struct snapraid_file* file = block_file_get(block); if (errno == EIO) { /* LCOV_EXCL_START */ log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); @@ -194,13 +197,11 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta } /* check if the file is changed */ - if (handle[j].st.st_size != block_file_get(block)->size - || handle[j].st.st_mtime != block_file_get(block)->mtime_sec - || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec - || handle[j].st.st_ino != block_file_get(block)->inode + if (handle[j].st.st_size != file->size + || handle[j].st.st_mtime != file->mtime_sec + || STAT_NSEC(&handle[j].st) != file->mtime_nsec + || handle[j].st.st_ino != file->inode ) { - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc(file->sub)); if (handle[j].st.st_size != file->size) { log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, handle[j].st.st_size); @@ -218,13 +219,11 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta continue; } - read_size = handle_read(&handle[j], block, buffer, state->block_size, log_fatal, 0); + read_size = handle_read(&handle[j], file_pos, buffer, state->block_size, log_fatal, 0); if (read_size == -1) { /* LCOV_EXCL_START */ - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); if (errno == EIO) { - log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno)); + log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno)); log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -232,7 +231,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta goto bail; } - log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno)); + log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno)); log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that file '%s' can be read.\n", handle[j].path); log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub); @@ -256,11 +255,8 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta if (block_state == BLOCK_STATE_REP) { /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); - log_tag("error:%u:%s:%s: Unexpected data change\n", i, disk->name, esc(file->sub)); - log_error("Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block)); + log_error("Data change at file '%s' at position '%u'\n", handle[j].path, file_pos); log_error("WARNING! Unexpected data modification of a file without parity!\n"); if (file_flag_has(file, FILE_IS_COPY)) { @@ -308,14 +304,14 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta /* close the last file in the disk */ if (handle[j].file != 0) { /* keep a pointer at the file we are going to close for error reporting */ - struct snapraid_file* file = handle[j].file; + struct snapraid_file* report = handle[j].file; ret = handle_close(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ /* This one is really an unexpected error, because we are only reading */ /* and closing a descriptor should never fail */ if (errno == EIO) { - log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockmax, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close EIO error. %s\n", blockmax, disk->name, esc(report->sub), strerror(errno)); log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path); log_fatal("Stopping at block %u\n", blockmax); @@ -323,7 +319,7 @@ static int state_hash_process(struct snapraid_state* state, block_off_t blocksta goto bail; } - log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close error. %s\n", blockmax, disk->name, esc(report->sub), strerror(errno)); log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path); log_fatal("Stopping at block %u\n", blockmax); @@ -572,6 +568,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari struct snapraid_block* block; unsigned block_state; struct snapraid_disk* disk = handle[j].disk; + struct snapraid_file* file; + block_off_t file_pos; /* by default no rehash in case of "continue" */ rehandle[j].block = 0; @@ -621,20 +619,23 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari continue; } + /* get the file of this block */ + file = fs_par2file_get(disk, i, &file_pos); + /* until now is CPU */ state_usage_cpu(state); /* if the file is different than the current one, close it */ - if (handle[j].file != 0 && handle[j].file != block_file_get(block)) { + if (handle[j].file != 0 && handle[j].file != file) { /* keep a pointer at the file we are going to close for error reporting */ - struct snapraid_file* file = handle[j].file; + struct snapraid_file* report = handle[j].file; ret = handle_close(&handle[j]); if (ret == -1) { /* LCOV_EXCL_START */ /* This one is really an unexpected error, because we are only reading */ /* and closing a descriptor should never fail */ if (errno == EIO) { - log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close EIO error. %s\n", i, disk->name, esc(report->sub), strerror(errno)); log_fatal("DANGER! Unexpected input/output close error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that disk '%s' is sane and that file '%s' can be accessed.\n", disk->dir, handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -642,7 +643,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari goto bail; } - log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); + log_tag("error:%u:%s:%s: Close error. %s\n", i, disk->name, esc(report->sub), strerror(errno)); log_fatal("WARNING! Unexpected close error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that file '%s' can be accessed.\n", handle[j].path); log_fatal("Stopping at block %u\n", i); @@ -652,10 +653,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } } - ret = handle_open(&handle[j], block_file_get(block), state->file_mode, log_error, 0); + ret = handle_open(&handle[j], file, state->file_mode, log_error, 0); if (ret == -1) { - /* file we have tried to open for error reporting */ - struct snapraid_file* file = block_file_get(block); if (errno == EIO) { /* LCOV_EXCL_START */ log_tag("error:%u:%s:%s: Open EIO error. %s\n", i, disk->name, esc(file->sub), strerror(errno)); @@ -701,13 +700,11 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari } /* check if the file is changed */ - if (handle[j].st.st_size != block_file_get(block)->size - || handle[j].st.st_mtime != block_file_get(block)->mtime_sec - || STAT_NSEC(&handle[j].st) != block_file_get(block)->mtime_nsec - || handle[j].st.st_ino != block_file_get(block)->inode + if (handle[j].st.st_size != file->size + || handle[j].st.st_mtime != file->mtime_sec + || STAT_NSEC(&handle[j].st) != file->mtime_nsec + || handle[j].st.st_ino != file->inode ) { - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); log_tag("error:%u:%s:%s: Unexpected attribute change\n", i, disk->name, esc(file->sub)); if (handle[j].st.st_size != file->size) { log_error("Unexpected size change at file '%s' from %" PRIu64 " to %" PRIu64 ".\n", handle[j].path, file->size, handle[j].st.st_size); @@ -726,13 +723,11 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari continue; } - read_size = handle_read(&handle[j], block, buffer[j], state->block_size, log_error, 0); + read_size = handle_read(&handle[j], file_pos, buffer[j], state->block_size, log_error, 0); if (read_size == -1) { /* LCOV_EXCL_START */ - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); if (errno == EIO) { - log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno)); + log_tag("error:%u:%s:%s: Read EIO error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno)); if (io_error >= state->opt.io_error_limit) { log_fatal("DANGER! Unexpected input/output read error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that disk '%s' is sane and that file '%s' can be read.\n", disk->dir, handle[j].path); @@ -741,13 +736,13 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari goto bail; } - log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, block_file_pos(block)); + log_error("Input/Output error in file '%s' at position '%u'\n", handle[j].path, file_pos); ++io_error; io_error_on_this_block = 1; continue; } - log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), block_file_pos(block), strerror(errno)); + log_tag("error:%u:%s:%s: Read error at position %u. %s\n", i, disk->name, esc(file->sub), file_pos, strerror(errno)); log_fatal("WARNING! Unexpected read error in a data disk, it isn't possible to sync.\n"); log_fatal("Ensure that file '%s' can be read.\n", handle[j].path); log_fatal("Stopping to allow recovery. Try with 'snapraid check -f %s'\n", file->sub); @@ -775,13 +770,10 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari if (block_has_updated_hash(block)) { /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { - /* file we are processing for error reporting */ - struct snapraid_file* file = block_file_get(block); - /* if the file has invalid parity, it's a REP changed during the sync */ if (block_has_invalid_parity(block)) { log_tag("error:%u:%s:%s: Unexpected data change\n", i, disk->name, esc(file->sub)); - log_error("Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block)); + log_error("Data change at file '%s' at position '%u'\n", handle[j].path, file_pos); log_error("WARNING! Unexpected data modification of a file without parity!\n"); if (file_flag_has(file, FILE_IS_COPY)) { @@ -801,8 +793,8 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari continue; } else { /* otherwise it's a BLK with silent error */ unsigned diff = memdiff(hash, block->hash, HASH_SIZE); - log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), block_file_pos(block), diff); - log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, block_file_pos(block), diff); + log_tag("error:%u:%s:%s: Data error at position %u, diff bits %u\n", i, disk->name, esc(file->sub), file_pos, diff); + log_error("Data error in file '%s' at position '%u', diff bits %u\n", handle[j].path, file_pos, diff); /* save the failed block for the fix */ failed[failed_count].index = j; From 9bc5f9e5ced2d298366abd822f6c68134cc15942 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 4 Oct 2015 22:02:05 +0200 Subject: [PATCH 06/12] Change the internal mapping to a tree of chunk This is intended to save memory as now we can use a fixed block of 17 bytes to store the block information, compared to the previous 28/36 bytes. --- Makefile.am | 2 + cmdline/dup.c | 2 +- cmdline/elem.c | 390 ++++++++++++++++++++++++++++++-------------- cmdline/elem.h | 131 +++++++++------ cmdline/parity.c | 4 +- cmdline/scan.c | 34 ++-- cmdline/snapraid.c | 12 +- cmdline/state.c | 83 +++------- cmdline/sync.c | 2 +- tommyds/tommy.c | 1 + tommyds/tommytree.c | 283 ++++++++++++++++++++++++++++++++ tommyds/tommytree.h | 214 ++++++++++++++++++++++++ 12 files changed, 900 insertions(+), 258 deletions(-) create mode 100644 tommyds/tommytree.c create mode 100644 tommyds/tommytree.h diff --git a/Makefile.am b/Makefile.am index 61f94523..e1bdf58e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -59,6 +59,7 @@ noinst_HEADERS = \ tommyds/tommyhash.h \ tommyds/tommyhashdyn.h \ tommyds/tommylist.h \ + tommyds/tommytree.h \ tommyds/tommytypes.h \ tommyds/tommyarray.c \ tommyds/tommyarrayblk.c \ @@ -66,6 +67,7 @@ noinst_HEADERS = \ tommyds/tommyhash.c \ tommyds/tommyhashdyn.c \ tommyds/tommylist.c \ + tommyds/tommytree.c \ cmdline/portable.h \ cmdline/snapraid.h \ cmdline/util.h \ diff --git a/cmdline/dup.c b/cmdline/dup.c index cb148f64..c5d209c3 100644 --- a/cmdline/dup.c +++ b/cmdline/dup.c @@ -50,7 +50,7 @@ struct snapraid_hash* hash_alloc(struct snapraid_state* state, struct snapraid_d /* set the back pointer */ for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = fs_file2block_get(disk, file, i); + struct snapraid_block* block = fs_file2block_get(file, i); memcpy(buf + i * HASH_SIZE, block->hash, HASH_SIZE); diff --git a/cmdline/elem.c b/cmdline/elem.c index 8381b8d7..a56924b4 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -355,24 +355,6 @@ int filter_content(tommy_list* contentlist, const char* path) return 0; } -/** - * Sets the file containing the block. - */ -static void block_file_set(struct snapraid_block* block, struct snapraid_file* file) -{ - uintptr_t ptr = (uintptr_t)file; - - /* ensure that the pointer doesn't use the flag space */ - if ((ptr & (uintptr_t)BLOCK_STATE_MASK) != 0) { - /* LCOV_EXCL_START */ - log_fatal("Internal error for pointer not aligned\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - block->private_file_mixed = (block->private_file_mixed & (uintptr_t)BLOCK_STATE_MASK) | ptr; -} - struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_t size, uint64_t mtime_sec, int mtime_nsec, uint64_t inode, uint64_t physical) { struct snapraid_file* file; @@ -390,13 +372,6 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, data_off_ file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); for (i = 0; i < file->blockmax; ++i) { - file->blockvec[i].private_parity_pos = POS_INVALID; - file->blockvec[i].private_file_mixed = 0; - - /* set the file */ - block_file_set(&file->blockvec[i], file); - - /* set an invalid hash */ block_state_set(&file->blockvec[i], BLOCK_STATE_CHG); hash_invalid_set(file->blockvec[i].hash); } @@ -421,12 +396,8 @@ struct snapraid_file* file_dup(struct snapraid_file* copy) file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); for (i = 0; i < file->blockmax; ++i) { - file->blockvec[i].private_file_mixed = copy->blockvec[i].private_file_mixed; - file->blockvec[i].private_parity_pos = copy->blockvec[i].private_parity_pos; + file->blockvec[i].state = copy->blockvec[i].state; memcpy(file->blockvec[i].hash, copy->blockvec[i].hash, HASH_SIZE); - - /* set the file to overwrite the copied one */ - block_file_set(&file->blockvec[i], file); } return file; @@ -632,6 +603,55 @@ int file_pathstamp_compare(const void* void_a, const void* void_b) return file_stamp_compare(void_a, void_b); } +struct snapraid_chunk* chunk_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count) +{ + struct snapraid_chunk* chunk; + + chunk = malloc_nofail(sizeof(struct snapraid_chunk)); + chunk->parity_pos = parity_pos; + chunk->file = file; + chunk->file_pos = file_pos; + chunk->count = count; + + return chunk; +} + +void chunk_free(struct snapraid_chunk* chunk) +{ + free(chunk); +} + +int chunk_parity_compare(const void* void_a, const void* void_b) +{ + const struct snapraid_chunk* arg_a = void_a; + const struct snapraid_chunk* arg_b = void_b; + + if (arg_a->parity_pos < arg_b->parity_pos) + return -1; + if (arg_a->parity_pos > arg_b->parity_pos) + return 1; + + return 0; +} + +int chunk_file_compare(const void* void_a, const void* void_b) +{ + const struct snapraid_chunk* arg_a = void_a; + const struct snapraid_chunk* arg_b = void_b; + + if (arg_a->file < arg_b->file) + return -1; + if (arg_a->file > arg_b->file) + return 1; + + if (arg_a->file_pos < arg_b->file_pos) + return -1; + if (arg_a->file_pos > arg_b->file_pos) + return 1; + + return 0; +} + struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag) { struct snapraid_link* link; @@ -724,7 +744,8 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev tommy_hashdyn_init(&disk->linkset); tommy_list_init(&disk->dirlist); tommy_hashdyn_init(&disk->dirset); - tommy_arrayblk_init(&disk->blockarr); + tommy_tree_init(&disk->fs_parity, chunk_parity_compare); + tommy_tree_init(&disk->fs_file, chunk_file_compare); return disk; } @@ -733,6 +754,7 @@ void disk_free(struct snapraid_disk* disk) { tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free); tommy_list_foreach(&disk->deletedlist, (tommy_foreach_func*)file_free); + tommy_tree_foreach(&disk->fs_file, (tommy_foreach_func*)chunk_free); tommy_hashdyn_done(&disk->inodeset); tommy_hashdyn_done(&disk->pathset); tommy_hashdyn_done(&disk->stampset); @@ -740,87 +762,32 @@ void disk_free(struct snapraid_disk* disk) tommy_hashdyn_done(&disk->linkset); tommy_list_foreach(&disk->dirlist, (tommy_foreach_func*)dir_free); tommy_hashdyn_done(&disk->dirset); - tommy_arrayblk_done(&disk->blockarr); free(disk); } -block_off_t disk_block_size(struct snapraid_disk* disk) +struct chunk_disk_empty { + block_off_t blockmax; +}; + +/** + * Search for any block inside the specified blockmax. + */ +static int chunk_disk_empty_compare(const void* void_a, const void* void_b) { - return tommy_arrayblk_size(&disk->blockarr); -} + const struct chunk_disk_empty* arg_a = void_a; + const struct snapraid_chunk* arg_b = void_b; -void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos) -{ - (void)disk; - - assert(file_pos < file->blockmax); - - file->blockvec[file_pos].private_parity_pos = parity_pos; -} - -struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) -{ - struct snapraid_block* block = fs_par2block_get(disk, parity_pos); - struct snapraid_file* file = (struct snapraid_file*)(block->private_file_mixed & ~(uintptr_t)BLOCK_STATE_MASK); - - if (block == BLOCK_EMPTY) + /* if the block is inside the specified blockmax, it's found */ + if (arg_a->blockmax > arg_b->parity_pos) return 0; - if (file_pos) { - if (block < file->blockvec || block >= file->blockvec + file->blockmax) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in block %u ownership\n", block->private_parity_pos); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - *file_pos = block - file->blockvec; - } - - return file; -} - -block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) -{ - (void)disk; - - assert(file_pos < file->blockmax); - - return file->blockvec[file_pos].private_parity_pos; -} - -struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) -{ - (void)disk; - - assert(file_pos < file->blockmax); - - return &file->blockvec[file_pos]; -} - -void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos) -{ - tommy_arrayblk_grow(&disk->blockarr, pos + 1); - tommy_arrayblk_set(&disk->blockarr, pos, BLOCK_EMPTY); -} - -void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block) -{ - tommy_arrayblk_grow(&disk->blockarr, pos + 1); - tommy_arrayblk_set(&disk->blockarr, pos, block); -} - -struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos) -{ - if (parity_pos < tommy_arrayblk_size(&disk->blockarr)) - return tommy_arrayblk_get(&disk->blockarr, parity_pos); - else - return BLOCK_EMPTY; + /* otherwise search for a smaller one */ + return -1; } int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) { - block_off_t i; + struct chunk_disk_empty arg = { blockmax }; /* if there is an element, it's not empty */ /* even if links and dirs have no block allocation */ @@ -831,35 +798,208 @@ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax) if (!tommy_list_empty(&disk->dirlist)) return 0; - /* limit the search to the size of the disk */ - if (blockmax > tommy_arrayblk_size(&disk->blockarr)) - blockmax = tommy_arrayblk_size(&disk->blockarr); - - /* checks all the blocks to search for deleted ones */ - /* this search is slow, but it's done only if no file is present */ - for (i = 0; i < blockmax; ++i) { - struct snapraid_block* block = tommy_arrayblk_get(&disk->blockarr, i); - unsigned block_state = block_state_get(block); - - switch (block_state) { - case BLOCK_STATE_EMPTY : - /* empty block are expected for an empty disk */ - break; - case BLOCK_STATE_DELETED : - /* if there is a deleted block, the disk is not empty */ - return 0; - default : - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for used block in disk '%s' without files\n", disk->name); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } + /* search for any chunk inside blockmax */ + if (tommy_tree_search_compare(&disk->fs_parity, chunk_disk_empty_compare, &arg) != 0) + return 0; /* finally, it's empty */ return 1; } +struct chunk_disk_size { + block_off_t size; +}; + +/** + * Search for the chunk with the highest parity position. + * + * The maximum parity position is stored as size. + */ +static int chunk_disk_size_compare(const void* void_a, const void* void_b) +{ + struct chunk_disk_size* arg_a = (void*)void_a; + const struct snapraid_chunk* arg_b = void_b; + + /* get the maximum size */ + if (arg_a->size < arg_b->parity_pos + arg_b->count) + arg_a->size = arg_b->parity_pos + arg_b->count; + + /* search always for a bigger one */ + return 1; +} + +block_off_t disk_size(struct snapraid_disk* disk) +{ + struct chunk_disk_size arg = { 0 }; + tommy_tree_search_compare(&disk->fs_parity, chunk_disk_size_compare, &arg); + return arg.size; +} + +struct chunk_parity_inside { + block_off_t parity_pos; +}; + +/** + * Search for the chunk containing the specified parity position. + */ +static int chunk_parity_inside_compare(const void* void_a, const void* void_b) +{ + const struct chunk_parity_inside* arg_a = void_a; + const struct snapraid_chunk* arg_b = void_b; + + if (arg_a->parity_pos < arg_b->parity_pos) + return -1; + if (arg_a->parity_pos >= arg_b->parity_pos + arg_b->count) + return 1; + + return 0; +} + +struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) +{ + struct chunk_parity_inside arg = { parity_pos }; + struct snapraid_chunk* chunk = tommy_tree_search_compare(&disk->fs_parity, chunk_parity_inside_compare, &arg); + + if (!chunk) + return 0; + + if (file_pos) + *file_pos = chunk->file_pos + (parity_pos - chunk->parity_pos); + + return chunk->file; +} + +struct chunk_file_inside { + struct snapraid_file* file; + block_off_t file_pos; +}; + +/** + * Search for the chunk containing the specified file position. + */ +static int chunk_file_inside_compare(const void* void_a, const void* void_b) +{ + const struct chunk_file_inside* arg_a = void_a; + const struct snapraid_chunk* arg_b = void_b; + + if (arg_a->file < arg_b->file) + return -1; + if (arg_a->file > arg_b->file) + return 1; + + if (arg_a->file_pos < arg_b->file_pos) + return -1; + if (arg_a->file_pos >= arg_b->file_pos + arg_b->count) + return 1; + + return 0; +} + +block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) +{ + struct chunk_file_inside arg = { file, file_pos }; + struct snapraid_chunk* chunk = tommy_tree_search_compare(&disk->fs_file, chunk_file_inside_compare, &arg); + + if (!chunk) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency for a file without parity in disk '%s'\n", disk->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + return chunk->parity_pos + (file_pos - chunk->file_pos); +} + +void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos) +{ + struct snapraid_chunk* chunk; + + if (file_pos > 0) { + /* search an existing chunk for the previous file_pos */ + struct chunk_file_inside arg = { file, file_pos - 1 }; + + chunk = tommy_tree_search_compare(&disk->fs_file, chunk_file_inside_compare, &arg); + + if (chunk != 0 && parity_pos == chunk->parity_pos + chunk->count) { + /* ensure that we are really extending a chunk */ + if (file_pos != chunk->file_pos + chunk->count) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency extending a chunk in disk '%s'\n", disk->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* extend the existing chunk */ + ++chunk->count; + + return; + } + } + + /* a chunk doesn't exist, and we have to create a new one */ + chunk = chunk_alloc(parity_pos, file, file_pos, 1); + + /* insert te chunk in the trees */ + tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk); + tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk); +} + +void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) +{ + struct chunk_parity_inside arg = { parity_pos }; + struct snapraid_chunk* chunk = tommy_tree_search_compare(&disk->fs_parity, chunk_parity_inside_compare, &arg); + + if (!chunk) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency for clearing a not existing block in disk '%s'\n", disk->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* if it's the only block of the chunk, delete it */ + if (chunk->count == 1) { + /* remove from the trees */ + tommy_tree_remove(&disk->fs_parity, chunk); + tommy_tree_remove(&disk->fs_file, chunk); + + /* deallocate */ + chunk_free(chunk); + return; + } + + /* if it's at the start of the chunk, shrink the chunk */ + if (parity_pos == chunk->parity_pos) { + ++chunk->parity_pos; + ++chunk->file_pos; + --chunk->count; + return; + } + + /* if it's at the end of the chunk, shrink the chunk */ + if (parity_pos == chunk->parity_pos + chunk->count - 1) { + --chunk->count; + return; + } + + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency for clearing in the middle of a chunk in disk '%s'\n", disk->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ +} + +struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos) +{ + struct snapraid_file* file; + block_off_t file_pos; + + file = fs_par2file_get(disk, parity_pos, &file_pos); + + if (!file) + return BLOCK_EMPTY; + + return fs_file2block_get(file, file_pos); +} + struct snapraid_map* map_alloc(const char* name, unsigned position, block_off_t total_blocks, block_off_t free_blocks, const char* uuid) { struct snapraid_map* map; diff --git a/cmdline/elem.h b/cmdline/elem.h index e0bcfe3c..d8429eb2 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -20,8 +20,9 @@ #include "util.h" #include "support.h" -#include "tommyds/tommylist.h" #include "tommyds/tommyhash.h" +#include "tommyds/tommylist.h" +#include "tommyds/tommytree.h" #include "tommyds/tommyhashdyn.h" #include "tommyds/tommyarray.h" #include "tommyds/tommyarrayblk.h" @@ -152,19 +153,12 @@ struct snapraid_filter { */ #define BLOCK_STATE_DELETED 4 -/** - * Mask used to store the previous states of the block. - * Used to mix them inside the file pointer. - */ -#define BLOCK_STATE_MASK 7 - /** * Block of a file. */ struct snapraid_block { - uintptr_t private_file_mixed; /**< Back pointer to the file owning this block, mixed with some flags. */ - block_off_t private_parity_pos; /**< Position of the block in the parity. */ unsigned char hash[HASH_SIZE]; /**< Hash of the block. */ + unsigned char state; /**< State of the block. */ }; /** @@ -287,6 +281,20 @@ struct snapraid_dir { tommy_hashdyn_node nodeset; }; +/** + * Chunk. + * + * A chunk represents a fragment of a file mapped into the parity. + */ +struct snapraid_chunk { + struct snapraid_file* file; /* File containing this chunk. */ + block_off_t parity_pos; /**< Parity position. */ + block_off_t file_pos; /**< Position in the file. */ + block_off_t count; /**< Number of sequential blocks in the file and parity. */ + tommy_tree_node parity_node; /**< Tree sorted by . */ + tommy_tree_node file_node; /**< Tree sorter by . */ +}; + /** * Disk. */ @@ -315,12 +323,12 @@ struct snapraid_disk { int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */ int mapping_idx; /**< Index in the mapping vector. Used only as buffer when writing the content file. */ - /**< - * Block array of the disk. - * - * Each element points to a snapraid_block structure, or it's BLOCK_EMPTY. + /** + * Mapping of chunks in the parity. + * Sorted by and by */ - tommy_arrayblk blockarr; + tommy_tree fs_parity; + tommy_tree fs_file; /** * List of all the snapraid_file for the disk. @@ -531,21 +539,15 @@ static inline unsigned block_state_get(const struct snapraid_block* block) if (block == BLOCK_EMPTY) return BLOCK_STATE_EMPTY; - return block->private_file_mixed & BLOCK_STATE_MASK; + return block->state; } +/** + * Set the state of the block. + */ static inline void block_state_set(struct snapraid_block* block, unsigned state) { - /* ensure that the state can be stored inside the file pointer */ - if ((state & BLOCK_STATE_MASK) != state) { - /* LCOV_EXCL_START */ - log_fatal("Internal error when setting the block state %u\n", state); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - - block->private_file_mixed &= ~(uintptr_t)BLOCK_STATE_MASK; - block->private_file_mixed |= state & BLOCK_STATE_MASK; + block->state = state; } /** @@ -724,6 +726,26 @@ static inline tommy_uint32_t file_stamp_hash(data_off_t size, int64_t mtime_sec, return tommy_inthash_u32((tommy_uint32_t)size ^ tommy_inthash_u32(mtime_sec ^ tommy_inthash_u32(mtime_nsec))); } +/** + * Allocate a chunk. + */ +struct snapraid_chunk* chunk_alloc(block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos, block_off_t count); + +/** + * Deallocate a chunk. + */ +void chunk_free(struct snapraid_chunk* chunk); + +/** + * Compare chunk by parity position. + */ +int chunk_parity_compare(const void* void_a, const void* void_b); + +/** + * Compare chunk by file and file position. + */ +int chunk_file_compare(const void* void_a, const void* void_b); + static inline int link_flag_has(const struct snapraid_link* link, unsigned mask) { return (link->flag & mask) == mask; @@ -829,12 +851,41 @@ void disk_free(struct snapraid_disk* disk); /** * Get the size of the disk in blocks. */ -block_off_t disk_block_size(struct snapraid_disk* disk); +block_off_t disk_size(struct snapraid_disk* disk); /** - * Set a filesystem mapping between file and parity positions. + * Check if a disk is totally empty and can be discarded from the content file. + * A disk is empty if it doesn't contain any file, symlink, hardlink or dir + * and without any DELETED block. + * The blockmax is used to limit the search of DELETED block up to blockmax. */ -void fs_par2file_set(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos); +int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax); + +/** + * Allocate a parity position for the specified file position. + * + * After this call you can use the par2file/par2block operations + * to query the relation. + */ +void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos); + +/** + * Deallocate the parity position. + * + * After this call you can the par2file/par2block operations + * won't find anymore the parity association. + */ +void fs_deallocate(struct snapraid_disk* disk, block_off_t pos); + +/** + * Get the block from the file position. + */ +static inline struct snapraid_block* fs_file2block_get(struct snapraid_file* file, block_off_t file_pos) +{ + assert(file_pos < file->blockmax); + + return &file->blockvec[file_pos]; +} /** * Get the file position from the parity position. @@ -847,36 +898,12 @@ struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t pa */ block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos); -/** - * Get the block from the file position. - */ -struct snapraid_block* fs_file2block_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos); - -/** - * Clear a specific block. - */ -void fs_par2block_clear(struct snapraid_disk* disk, block_off_t pos); - -/** - * Set the block from parity position. - * In case of holes, everything is initialized to 0. - */ -void fs_par2block_set(struct snapraid_disk* disk, block_off_t pos, struct snapraid_block* block); - /** * Get the block from the parity position. * Returns BLOCK_EMPTY==0 if the block is over the end of the disk or not used. */ struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos); -/** - * Check if a disk is totally empty and can be discarded from the content file. - * A disk is empty if it doesn't contain any file, symlink, hardlink or dir - * and without any DELETED block. - * The blockmax is used to limit the search of DELETED block up to blockmax. - */ -int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax); - /** * Allocates a disk mapping. * Uses uuid="" if not available. diff --git a/cmdline/parity.c b/cmdline/parity.c index b7617669..cab58ccc 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -37,7 +37,7 @@ block_off_t parity_allocated_size(struct snapraid_state* state) struct snapraid_disk* disk = i->data; /* start from the declared size */ - block_off_t block = disk_block_size(disk); + block_off_t block = disk_size(disk); /* decrease the block until an allocated one, but part of a file */ /* we don't stop at deleted blocks, because we want to have them cleared */ @@ -64,7 +64,7 @@ block_off_t parity_used_size(struct snapraid_state* state) struct snapraid_disk* disk = i->data; /* start from the declared size */ - block_off_t block = disk_block_size(disk); + block_off_t block = disk_size(disk); /* decrease the block until an used one */ while (block > parity_block && !block_has_file_and_valid_parity(fs_par2block_get(disk, block - 1))) diff --git a/cmdline/scan.c b/cmdline/scan.c index 0ab47b0f..ee26c2e7 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -170,19 +170,27 @@ static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* /* allocate the blocks of the file */ parity_pos = disk->first_free_block; for (i = 0; i < file->blockmax; ++i) { + struct snapraid_block* block; + struct snapraid_block* over_block; snapraid_info info; - struct snapraid_block* block = fs_file2block_get(disk, file, i); /* increment the position until the first really free block */ while (block_has_file(fs_par2block_get(disk, parity_pos))) ++parity_pos; - /* set the position */ - block->private_parity_pos = parity_pos; + /* get block we are going to overwrite, if any */ + over_block = fs_par2block_get(disk, parity_pos); + + /* deallocate it */ + if (over_block != BLOCK_EMPTY) + fs_deallocate(disk, parity_pos); /* get block specific info */ info = info_get(&state->infoarr, parity_pos); + /* get the new block we are going to write */ + block = fs_file2block_get(file, i); + /* if the file block already has an updated hash without rehash */ if (block_has_updated_hash(block) && !info_get_rehash(info)) { /* the only possible case is for REP blocks */ @@ -193,26 +201,22 @@ static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* /* and keep the hash as it's */ } else { - struct snapraid_block* over_block; - unsigned block_state; + unsigned over_state; /* convert to a CHG block */ block_state_set(block, BLOCK_STATE_CHG); - /* block to overwrite */ - over_block = fs_par2block_get(disk, parity_pos); - /* state of the block we are going to overwrite */ - block_state = block_state_get(over_block); + over_state = block_state_get(over_block); /* if the block is an empty one */ - if (block_state == BLOCK_STATE_EMPTY) { + if (over_state == BLOCK_STATE_EMPTY) { /* the block was empty and filled with zeros */ /* set the hash to the special ZERO value */ hash_zero_set(block->hash); } else { /* otherwise it's a DELETED one */ - assert(block_state == BLOCK_STATE_DELETED); + assert(over_state == BLOCK_STATE_DELETED); /* copy the past hash of the block */ memcpy(block->hash, over_block->hash, HASH_SIZE); @@ -235,7 +239,7 @@ static void scan_file_allocate(struct snapraid_scan* scan, struct snapraid_file* } /* store in the disk map, after invalidating all the other blocks */ - fs_par2block_set(disk, parity_pos, block); + fs_allocate(disk, parity_pos, file, i); /* set the new free position */ disk->first_free_block = parity_pos + 1; @@ -263,7 +267,7 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil /* free all the blocks of the file */ for (i = 0; i < file->blockmax; ++i) { - struct snapraid_block* block = fs_file2block_get(disk, file, i); + struct snapraid_block* block = fs_file2block_get(file, i); block_off_t parity_pos = fs_file2par_get(disk, file, i); unsigned block_state; @@ -357,7 +361,7 @@ static int file_is_full_invalid_parity_and_stable(struct snapraid_state* state, /* check all blocks */ for (i = 0; i < file->blockmax; ++i) { snapraid_info info; - struct snapraid_block* block = fs_file2block_get(disk, file, i); + struct snapraid_block* block = fs_file2block_get(file, i); block_off_t parity_pos = fs_file2par_get(disk, file, i); /* exclude blocks with parity */ @@ -390,7 +394,7 @@ static int file_is_full_hashed_and_stable(struct snapraid_state* state, struct s /* check all blocks */ for (i = 0; i < file->blockmax; ++i) { snapraid_info info; - struct snapraid_block* block = fs_file2block_get(disk, file, i); + struct snapraid_block* block = fs_file2block_get(file, i); block_off_t parity_pos = fs_file2par_get(disk, file, i); /* exclude blocks without hash */ diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index a80e659f..3f083899 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -78,14 +78,14 @@ void memory(void) { log_tag("memory:used:%" PRIu64 "\n", (uint64_t)malloc_counter()); - /* size of the block + the size of the blockarr pointer to it */ - log_tag("memory:block:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_block) + sizeof(void*))); - - /* size of the file + the size of the 3 hashtables pointers to it, * 2 for the hashtable grow factor */ - log_tag("memory:file:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_file) + 3*2*sizeof(void*))); + /* size of the block */ + log_tag("memory:block:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_block))); + log_tag("memory:chunk:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_chunk))); + log_tag("memory:file:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_file))); + log_tag("memory:link:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_link))); + log_tag("memory:dir:%" PRIu64 "\n", (uint64_t)(sizeof(struct snapraid_dir))); msg_progress("Using %u MiB of memory.\n", (unsigned)(malloc_counter() / (1024 * 1024))); - } /****************************************************************************/ diff --git a/cmdline/state.c b/cmdline/state.c index 2705506b..52f9a630 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1440,7 +1440,7 @@ static void position_clear_deleted(struct snapraid_state* state, block_off_t pos /* if the block is deleted */ if (block_state_get(block) == BLOCK_STATE_DELETED) { /* set it to empty */ - fs_par2block_clear(disk, pos); + fs_deallocate(disk, pos); } } } @@ -1721,15 +1721,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* fill the blocks in the run */ while (v_count) { - struct snapraid_block* block = &file->blockvec[v_idx]; - - if (block->private_parity_pos != POS_INVALID) { - /* LCOV_EXCL_START */ - decoding_error(path, f); - log_fatal("Internal inconsistency in block position!\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } + struct snapraid_block* block = fs_file2block_get(file, v_idx); switch (c) { case 'b' : @@ -1753,8 +1745,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } - block->private_parity_pos = v_pos; - /* read the hash only for 'blk/chg/rep', and not for 'new' */ if (c != 'n') { ret = sread(f, block->hash, HASH_SIZE); @@ -1811,8 +1801,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } - /* insert the block in the block array */ - fs_par2block_set(disk, v_pos, block); + /* set the parity association */ + fs_allocate(disk, v_pos, file, v_idx); /* go to the next block */ ++v_idx; @@ -1925,7 +1915,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } else if (c == 'h') { /* hole */ uint32_t v_pos; - uint32_t i; struct snapraid_disk* disk; uint32_t mapping; @@ -1941,6 +1930,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST v_pos = 0; while (v_pos < blockmax) { + uint32_t v_idx; uint32_t v_count; struct snapraid_file* deleted; @@ -1967,23 +1957,20 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST case 'o' : /* if it's a run of deleted blocks */ - /* allocate the deleted file */ + /* allocate a fake deleted file */ deleted = file_alloc(state->block_size, "", v_count * state->block_size, 0, 0, 0, 0); /* insert it in the list of deleted files */ tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted); /* process all blocks */ - i = 0; + v_idx = 0; while (v_count) { - struct snapraid_block* block = &deleted->blockvec[i]; + struct snapraid_block* block = fs_file2block_get(deleted, v_idx); /* set the block as deleted */ block_state_set(block, BLOCK_STATE_DELETED); - /* set the position */ - block->private_parity_pos = v_pos; - /* read the hash */ ret = sread(f, block->hash, HASH_SIZE); if (ret < 0) { @@ -2012,13 +1999,13 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } else { /* insert the block in the block array */ - fs_par2block_set(disk, v_pos, block); + fs_allocate(disk, v_pos, deleted, v_idx); } /* go to next block */ ++v_pos; + ++v_idx; --v_count; - ++i; } break; case 'O' : @@ -2463,7 +2450,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) unsigned count_symlink; unsigned count_dir; tommy_node* i; - block_off_t b; + block_off_t idx; block_off_t blockmax; block_off_t begin; time_t info_oldest; @@ -2484,10 +2471,10 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* and get some other info */ info_oldest = 0; /* oldest time in info */ info_has_rehash = 0; /* if there is a rehash info */ - for (b = 0; b < blockmax; ++b) { + for (idx = 0; idx < blockmax; ++idx) { /* if the position is used */ - if (position_is_required(state, b)) { - snapraid_info info = info_get(&state->infoarr, b); + if (position_is_required(state, idx)) { + snapraid_info info = info_get(&state->infoarr, idx); /* only if there is some info to store */ if (info) { @@ -2501,10 +2488,10 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) } } else { /* clear any previous info */ - info_set(&state->infoarr, b, 0); + info_set(&state->infoarr, idx, 0); /* and clear any deleted blocks */ - position_clear_deleted(state, b); + position_clear_deleted(state, idx); } } @@ -2633,7 +2620,6 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* for each file */ for (j = disk->filelist; j != 0; j = j->next) { struct snapraid_file* file = j->data; - struct snapraid_block* blockvec = file->blockvec; uint64_t size; uint64_t mtime_sec; int32_t mtime_nsec; @@ -2665,22 +2651,23 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* for all the blocks of the file */ begin = 0; while (begin < file->blockmax) { - unsigned block_state = block_state_get(blockvec + begin); - uint32_t v_pos = blockvec[begin].private_parity_pos; + unsigned v_state = block_state_get(fs_file2block_get(file, begin)); + block_off_t v_pos = fs_file2par_get(disk, file, begin); uint32_t v_count; block_off_t end; /* find the end of run of blocks */ end = begin + 1; - while (end < file->blockmax - && block_state == block_state_get(blockvec + end) - && blockvec[end - 1].private_parity_pos + 1 == blockvec[end].private_parity_pos - ) { + while (end < file->blockmax) { + if (v_state != block_state_get(fs_file2block_get(file, end))) + break; + if (v_pos + (end - begin) != fs_file2par_get(disk, file, end)) + break; ++end; } - switch (block_state) { + switch (v_state) { case BLOCK_STATE_BLK : sputc('b', f); break; @@ -2692,7 +2679,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) break; default : /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in state for block %u state %u\n", v_pos, block_state); + log_fatal("Internal inconsistency in state for block %u state %u\n", v_pos, v_state); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -2703,16 +2690,8 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) sputb32(v_count, f); /* write hashes */ - for (b = begin; b < end; ++b) { - struct snapraid_block* block = blockvec + b; - - /* consistency check */ - if (block != fs_par2block_get(disk, block->private_parity_pos)) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in mismatch for block %u state %u in file '%s'\n", block->private_parity_pos, block_state, file->sub); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } + for (idx = begin; idx < end; ++idx) { + struct snapraid_block* block = fs_file2block_get(file, idx); swrite(block->hash, HASH_SIZE, f); } @@ -2808,14 +2787,6 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) while (begin < end) { struct snapraid_block* block = fs_par2block_get(disk, begin); - /* consistency check */ - if (block->private_parity_pos != begin) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency in delete position %u/%u!\n", block->private_parity_pos, begin); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - swrite(block->hash, HASH_SIZE, f); ++begin; diff --git a/cmdline/sync.c b/cmdline/sync.c index 71c483b4..db444d3d 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1031,7 +1031,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* if it's a deleted block */ if (block_state_get(block) == BLOCK_STATE_DELETED) { /* the parity is now updated without this block, so it's now empty */ - fs_par2block_clear(handle[j].disk, i); + fs_deallocate(handle[j].disk, i); continue; } diff --git a/tommyds/tommy.c b/tommyds/tommy.c index 68316587..d7dc520f 100644 --- a/tommyds/tommy.c +++ b/tommyds/tommy.c @@ -38,5 +38,6 @@ #include "tommyarrayblk.c" #include "tommyarrayblkof.c" #include "tommylist.c" +#include "tommytree.c" #include "tommyhashdyn.c" diff --git a/tommyds/tommytree.c b/tommyds/tommytree.c new file mode 100644 index 00000000..7ea31c21 --- /dev/null +++ b/tommyds/tommytree.c @@ -0,0 +1,283 @@ +/* + * Copyright (c) 2015, Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +#include "tommytree.h" + +#include /* for assert */ + +/******************************************************************************/ +/* tree */ + +void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp) +{ + tree->root = 0; + tree->count = 0; + tree->cmp = cmp; +} + +static int tommy_tree_delta(tommy_tree_node* root) +{ + int left_height = root->prev ? root->prev->key : 0; + int right_height = root->next ? root->next->key : 0; + + return left_height - right_height; +} + +/* AVL tree operations */ +static tommy_tree_node* tommy_tree_balance(tommy_tree_node*); + +static tommy_tree_node* tommy_tree_rotate_left(tommy_tree_node* root) +{ + tommy_tree_node* next = root->next; + + root->next = next->prev; + + next->prev = tommy_tree_balance(root); + + return tommy_tree_balance(next); +} + +static tommy_tree_node* tommy_tree_rotate_right(tommy_tree_node* root) +{ + tommy_tree_node* prev = root->prev; + + root->prev = prev->next; + + prev->next = tommy_tree_balance(root); + + return tommy_tree_balance(prev); +} + +static tommy_tree_node* tommy_tree_move_right(tommy_tree_node* root, tommy_tree_node* node) +{ + if (!root) + return node; + + root->next = tommy_tree_move_right(root->next, node); + + return tommy_tree_balance(root); +} + +static tommy_tree_node* tommy_tree_balance(tommy_tree_node* root) +{ + int delta = tommy_tree_delta(root); + + if (delta < -1) { + if (tommy_tree_delta(root->next) > 0) + root->next = tommy_tree_rotate_right(root->next); + return tommy_tree_rotate_left(root); + } + + if (delta > 1) { + if (tommy_tree_delta(root->prev) < 0) + root->prev = tommy_tree_rotate_left(root->prev); + return tommy_tree_rotate_right(root); + } + + /* recompute key */ + root->key = 0; + + if (root->prev && root->prev->key > root->key) + root->key = root->prev->key; + + if (root->next && root->next->key > root->key) + root->key = root->next->key; + + /* count itself */ + root->key += 1; + + return root; +} + +static tommy_tree_node* tommy_tree_insert_node(tommy_compare_func* cmp, tommy_tree_node* root, tommy_tree_node** let) +{ + int c; + + if (!root) + return *let; + + c = cmp((*let)->data, root->data); + + if (c < 0) { + root->prev = tommy_tree_insert_node(cmp, root->prev, let); + return tommy_tree_balance(root); + } + + if (c > 0) { + root->next = tommy_tree_insert_node(cmp, root->next, let); + return tommy_tree_balance(root); + } + + /* already present, set the return pointer */ + *let = root; + + return root; +} + +void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data) +{ + node->data = data; + node->prev = 0; + node->next = 0; + node->key = 0; + + tree->root = tommy_tree_insert_node(tree->cmp, tree->root, &node); + + ++tree->count; + + return node->data; +} + +static tommy_tree_node* tommy_tree_remove_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data, tommy_tree_node** let) +{ + int c; + + if (!root) + return 0; + + c = cmp(data, root->data); + + if (c < 0) { + root->prev = tommy_tree_remove_node(cmp, root->prev, data, let); + return tommy_tree_balance(root); + } + + if (c > 0) { + root->next = tommy_tree_remove_node(cmp, root->next, data, let); + return tommy_tree_balance(root); + } + + /* found */ + *let = root; + + return tommy_tree_move_right(root->prev, root->next); +} + +void* tommy_tree_remove(tommy_tree* tree, void* data) +{ + tommy_tree_node* node = 0; + + tree->root = tommy_tree_remove_node(tree->cmp, tree->root, data, &node); + + if (!node) + return 0; + + return node->data; +} + +static tommy_tree_node* tommy_tree_search_node(tommy_compare_func* cmp, tommy_tree_node* root, void* data) +{ + int c; + + if (!root) + return 0; + + c = cmp(data, root->data); + + if (c < 0) + return tommy_tree_search_node(cmp, root->prev, data); + + if (c > 0) + return tommy_tree_search_node(cmp, root->next, data); + + return root; +} + +void* tommy_tree_search(tommy_tree* tree, void* data) +{ + tommy_tree_node* node = tommy_tree_search_node(tree->cmp, tree->root, data); + + if (!node) + return 0; + + return node->data; +} + +void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg) +{ + tommy_tree_node* node = tommy_tree_search_node(cmp, tree->root, cmp_arg); + + if (!node) + return 0; + + return node->data; +} + +void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node) +{ + return tommy_tree_remove(tree, node->data); +} + +static void tommy_tree_foreach_node(tommy_tree_node* root, tommy_foreach_func* func) +{ + tommy_tree_node* next; + + if (!root) + return; + + tommy_tree_foreach_node(root->prev, func); + + /* make a copy in case func is free() */ + next = root->next; + + func(root->data); + + tommy_tree_foreach_node(next, func); +} + +void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func) +{ + tommy_tree_foreach_node(tree->root, func); +} + +static void tommy_tree_foreach_arg_node(tommy_tree_node* root, tommy_foreach_arg_func* func, void* arg) +{ + tommy_tree_node* next; + + if (!root) + return; + + tommy_tree_foreach_arg_node(root->prev, func, arg); + + /* make a copy in case func is free() */ + next = root->next; + + func(arg, root->data); + + tommy_tree_foreach_arg_node(next, func, arg); +} + +void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg) +{ + tommy_tree_foreach_arg_node(tree->root, func, arg); +} + +tommy_size_t tommy_tree_memory_usage(tommy_tree* tree) +{ + return tommy_tree_count(tree) * sizeof(tommy_tree_node); +} + diff --git a/tommyds/tommytree.h b/tommyds/tommytree.h new file mode 100644 index 00000000..a4db28b7 --- /dev/null +++ b/tommyds/tommytree.h @@ -0,0 +1,214 @@ +/* + * Copyright (c) 2015, Andrea Mazzoleni. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" + * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE + * LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ + +/** \file + * AVL tree. + * + * This tree is a standard AVL tree implementation that stores elements in the order defined + * by the comparsion function. + * + * \code + * tommy_tree tree; + * + * tommy_tree_init(&tree, cmp); + * \endcode + * + * To insert elements in the tree you have to call tommy_tree_insert() for + * each element. + * In the insertion call you have to specify the address of the node, the + * address of the object. + * The address of the object is used to initialize the tommy_node::data field + * of the node. + * + * \code + * struct object { + * int value; + * // other fields + * tommy_tree_node node; + * }; + * + * struct object* obj = malloc(sizeof(struct object)); // creates the object + * + * obj->value = ...; // initializes the object + * + * tommy_tree_insert(&tree, &obj->node, obj); // inserts the object + * \endcode + * + * To find and element in the tree you have to call tommy_tree_search() providing + * the key to search. + * + * \code + * struct object value_to_find = { 1 }; + * struct object* obj = tommy_tree_search(&tree, &value_to_find); + * if (!obj) { + * // not found + * } else { + * // found + * } + * \endcode + * + * To remove an element from the tree you have to call tommy_tree_remove() + * providing the key to search and remove. + * + * \code + * struct object* obj = tommy_tree_remove(&tree, value_to_remove); + * if (obj) { + * free(obj); // frees the object allocated memory + * } + * \endcode + * + * To destroy the tree you have to remove all the elements. + * + * If you need to iterate over all the elements in the tree, you can use + * tommy_hashtable_foreach() or tommy_hashtable_foreach_arg(). + * If you need a more precise control with a real iteration, you have to insert + * all the elements also in a ::tommy_list, and use the list to iterate. + * See the \ref multiindex example for more detail. + */ + +#ifndef __TOMMYTREE_H +#define __TOMMYTREE_H + +#include "tommytypes.h" + +/******************************************************************************/ +/* tree */ + +/** + * Tree node. + * This is the node that you have to include inside your objects. + */ +typedef tommy_node tommy_tree_node; + +typedef struct tommy_tree_struct { + tommy_tree_node* root; /**< Root node. */ + tommy_count_t count; /**< Number of elements. */ + tommy_compare_func* cmp; /**< Comparison function. */ +} tommy_tree; + +/** + * Initializes the tree. + */ +void tommy_tree_init(tommy_tree* tree, tommy_compare_func* cmp); + +/** + * Inserts an element in the tree. + * If the element is already present, it's not inserted again. + * You have to provide the pointer of the node embedded into the object, + * the pointer at the object. + * \param node Pointer at the node embedded into the object to insert. + * \param data Pointer at the object to insert. + * \return The element in the tree. Either the already existing one, or the one just inserted. + */ +void* tommy_tree_insert(tommy_tree* tree, tommy_tree_node* node, void* data); + +/** + * Searches and removes an element. + * If the element is not found, 0 is returned. + * \param data Element used for comparison. + * \return The removed element, or 0 if not found. + */ +void* tommy_tree_remove(tommy_tree* tree, void* data); + +/** + * Searches an element in the tree. + * If the element is not found, 0 is returned. + * \param data Element used for comparison. + * \return The first element found, or 0 if none. + */ +void* tommy_tree_search(tommy_tree* tree, void* data); + +/** + * Searches an element in the tree with a specific compare function. + * + * Like tommy_tree_search() but you can specify a different comparison function. + * Note that this function must define a suborder of the original one. + * + * The ::data argument is going to always be the first argument of the comparison function, + * and it can be something different than the objects in the tree. + */ +void* tommy_tree_search_compare(tommy_tree* tree, tommy_compare_func* cmp, void* cmp_arg); + +/** + * Removes an element from the tree. + * You must already have the address of the element to remove. + * \return The tommy_node::data field of the node removed. + */ +void* tommy_tree_remove_existing(tommy_tree* tree, tommy_tree_node* node); + +/** + * Calls the specified function for each element in the tree. + * + * The elements are processed in order. + * + * You can use this function to deallocate all the elements inserted. + * + * \code + * tommy_tree tree; + * + * // initializes the tree + * tommy_tree_init(&tree, cmp); + * + * ... + * + * // creates an object + * struct object* obj = malloc(sizeof(struct object)); + * + * ... + * + * // insert it in the hashtable + * tommy_tree_insert(&tree, &obj->node, obj); + * + * ... + * + * // deallocates all the objects iterating the tree + * tommy_tree_foreach(&tree, free); + * \endcode + */ +void tommy_tree_foreach(tommy_tree* tree, tommy_foreach_func* func); + +/** + * Calls the specified function with an argument for each element in the tree. + */ +void tommy_tree_foreach_arg(tommy_tree* tree, tommy_foreach_arg_func* func, void* arg); + +/** + * Gets the number of elements. + */ +tommy_inline tommy_count_t tommy_tree_count(tommy_tree* tree) +{ + return tree->count; +} + +/** + * Gets the size of allocated memory. + * It includes the size of the ::tommy_tree_node of the stored elements. + */ +tommy_size_t tommy_tree_memory_usage(tommy_tree* tree); + +#endif + From 0c3bfb9b48a34982ea18c889f2c79b711b1c6363 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 4 Oct 2015 22:31:12 +0200 Subject: [PATCH 07/12] Remove coverage need for malloc failure functions --- cmdline/support.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/cmdline/support.c b/cmdline/support.c index 95e1933e..34428c6d 100644 --- a/cmdline/support.c +++ b/cmdline/support.c @@ -597,6 +597,9 @@ int fmtime(int f, int64_t mtime_sec, int mtime_nsec) /****************************************************************************/ /* memory */ +/** + * Total amount of memory allocated. + */ static size_t mcounter; size_t malloc_counter(void) @@ -604,6 +607,7 @@ size_t malloc_counter(void) return mcounter; } +/* LCOV_EXCL_START */ static ssize_t malloc_print(int f, const char* str) { ssize_t len = 0; @@ -611,7 +615,9 @@ static ssize_t malloc_print(int f, const char* str) ++len; return write(f, str, len); } +/* LCOV_EXCL_STOP */ +/* LCOV_EXCL_START */ static ssize_t malloc_printn(int f, size_t value) { char buf[32]; @@ -628,11 +634,12 @@ static ssize_t malloc_printn(int f, size_t value) return write(f, buf + i, sizeof(buf) - i); } +/* LCOV_EXCL_STOP */ +/* LCOV_EXCL_START */ void malloc_fail(size_t size) { /* don't use printf to avoid any possible extra allocation */ - /* LCOV_EXCL_START */ int f = 2; /* stderr */ malloc_print(f, "Failed for Low Memory!\n"); @@ -646,8 +653,8 @@ void malloc_fail(size_t size) malloc_print(f, "You are currently using a 32 bits executable.\n"); malloc_print(f, "If you have more than 4GB of memory, please upgrade to a 64 bits one.\n"); } - /* LCOV_EXCL_STOP */ } +/* LCOV_EXCL_STOP */ void* malloc_nofail(size_t size) { From cb17124cec5bdc2dcd42da04bc4453f7f4ace291 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 4 Oct 2015 22:31:47 +0200 Subject: [PATCH 08/12] Remove unused text stream functions --- cmdline/stream.c | 228 ----------------------------------------------- cmdline/stream.h | 43 --------- 2 files changed, 271 deletions(-) diff --git a/cmdline/stream.c b/cmdline/stream.c index 9f9059d8..6efd4948 100644 --- a/cmdline/stream.c +++ b/cmdline/stream.c @@ -21,31 +21,6 @@ #include "util.h" #include "stream.h" -/****************************************************************************/ -/* hex conversion table */ - -static char strhexset[16] = "0123456789abcdef"; - -static unsigned strdecset[256] = -{ - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, - -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1, -1 -}; - /****************************************************************************/ /* stream */ @@ -470,108 +445,6 @@ int sgetu32(STREAM* f, uint32_t* value) } } -int sgetu64(STREAM* f, uint64_t* value) -{ - int c; - - c = sgetc(f); - if (c >= '0' && c <= '9') { - uint64_t v; - - v = c - '0'; - - c = sgetc(f); - while (c >= '0' && c <= '9') { - v *= 10; - v += c - '0'; - c = sgetc(f); - } - - *value = v; - - sungetc(c, f); - return 0; - } else { - /* LCOV_EXCL_START */ - /* nothing read */ - return -1; - /* LCOV_EXCL_STOP */ - } -} - -int sgethex(STREAM* f, void* void_data, int size) -{ - unsigned char* data = void_data; - - /* if there is enough data in memory */ - if (sptrlookup(f, size * 2)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - unsigned x = 0; - - while (size--) { - unsigned b0; - unsigned b1; - unsigned b; - - b0 = strdecset[pos[0]]; - b1 = strdecset[pos[1]]; - pos += 2; - - b = (b0 << 4) | b1; - - x |= b; - - *data++ = b; - } - - /* at the end check if a digit was wrong */ - if (x > 0xFF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - sptrset(f, pos); - } else { - /* standard version using sgetc() */ - while (size--) { - unsigned b0; - unsigned b1; - unsigned b; - int c; - - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - b0 = strdecset[(unsigned char)c]; - - c = sgetc(f); - if (c == EOF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - b1 = strdecset[(unsigned char)c]; - - b = (b0 << 4) | b1; - - if (b > 0xFF) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - *data++ = b; - } - } - - return 0; -} - int sgetb32(STREAM* f, uint32_t* value) { uint32_t v; @@ -680,19 +553,6 @@ int sgetbs(STREAM* f, char* str, int size) return sread(f, str, (int)len); } -int sputs(const char* str, STREAM* f) -{ - while (*str) { - if (sputc(*str++, f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - } - - return 0; -} - int swrite(const void* void_data, unsigned size, STREAM* f) { const unsigned char* data = void_data; @@ -723,94 +583,6 @@ int swrite(const void* void_data, unsigned size, STREAM* f) return 0; } -int sputu32(uint32_t value, STREAM* s) -{ - char buf[16]; - int i; - - if (!value) - return sputc('0', s); - - i = sizeof(buf); - - while (value) { - buf[--i] = (value % 10) + '0'; - value /= 10; - } - - return swrite(buf + i, sizeof(buf) - i, s); -} - -int sputu64(uint64_t value, STREAM* s) -{ - char buf[32]; - uint32_t value32; - int i; - - if (!value) - return sputc('0', s); - - i = sizeof(buf); - - while (value > 0xFFFFFFFF) { - buf[--i] = (value % 10) + '0'; - value /= 10; - } - - value32 = (uint32_t)value; - - while (value32) { - buf[--i] = (value32 % 10) + '0'; - value32 /= 10; - } - - return swrite(buf + i, sizeof(buf) - i, s); -} - -int sputhex(const void* void_data, int size, STREAM* f) -{ - const unsigned char* data = void_data; - - /* if there is enough space in memory */ - if (sptrlookup(f, size * 2)) { - /* optimized version with all the data in memory */ - unsigned char* pos = sptrget(f); - - while (size) { - unsigned b = *data; - - *pos++ = strhexset[b >> 4]; - *pos++ = strhexset[b & 0xF]; - - ++data; - --size; - } - - sptrset(f, pos); - } else { - /* standard version using sputc() */ - while (size) { - unsigned b = *data; - - if (sputc(strhexset[b >> 4], f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - if (sputc(strhexset[b & 0xF], f) != 0) { - /* LCOV_EXCL_START */ - return -1; - /* LCOV_EXCL_STOP */ - } - - ++data; - --size; - } - } - - return 0; -} - int sputb32(uint32_t value, STREAM* s) { unsigned char b; diff --git a/cmdline/stream.h b/cmdline/stream.h index a26fa20f..2c0a9407 100644 --- a/cmdline/stream.h +++ b/cmdline/stream.h @@ -305,19 +305,6 @@ int sgetlasttok(STREAM* f, char* str, int size); */ int sgetu32(STREAM* f, uint32_t* value); -/** - * Reads a 64 bit number. - * Stops at the first not digit char. - * Returns <0 if there isn't enough to read. - */ -int sgetu64(STREAM* f, uint64_t* value); - -/** - * Reads an hexadecimal string of fixed length. - * Returns <0 if there isn't enough to read. - */ -int sgethex(STREAM* f, void* data, int size); - /****************************************************************************/ /* binary get */ @@ -376,42 +363,12 @@ static inline int sputeol(STREAM* s) return sputc('\n', s); } -/** - * Writes a string. - * Returns 0 on success or -1 on error. - */ -int sputs(const char* str, STREAM* f); - /** * Write a sized string. * Returns 0 on success or -1 on error. */ int swrite(const void* data, unsigned size, STREAM* f); -/** - * Write a string literal (constant string). - * Returns 0 on success or -1 on error. - */ -#define sputsl(str, f) swrite(str, sizeof(str) - 1, f) - -/** - * Writes a 32 bit number. - * Returns 0 on success or -1 on error. - */ -int sputu32(uint32_t value, STREAM* s); - -/** - * Writes a 64 bit number. - * Returns 0 on success or -1 on error. - */ -int sputu64(uint64_t value, STREAM* s); - -/** - * Writes a hexadecimal string of fixed length. - * Returns 0 on success or -1 on error. - */ -int sputhex(const void* void_data, int size, STREAM* s); - /****************************************************************************/ /* binary put */ From 20043778e47c11251c654bbacefdda1a94726f25 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 4 Oct 2015 23:11:05 +0200 Subject: [PATCH 09/12] Update manual and history for the new memory usage --- HISTORY | 3 +++ TODO | 52 ---------------------------------------------------- snapraid.1 | 11 +++++------ snapraid.d | 11 +++++------ snapraid.txt | 11 +++++------ 5 files changed, 18 insertions(+), 70 deletions(-) diff --git a/HISTORY b/HISTORY index faf61de4..1773139d 100644 --- a/HISTORY +++ b/HISTORY @@ -14,6 +14,9 @@ SnapRAID HISTORY parity during sync is really correct. The "status" command reports the amount of scrub necessary to ensure to scrub all the synced blocks. + * Reduced the RAM memory usage, expecially with big files. + For each block, it now uses 17 bytes, instead of 28 bytes + (for 32 bit) and 36 bytes (for 64 bit). * The -p, --percentage option supports the new the arguments: "bad", "sync" and "full" to define the kind of scrub to do. * Extends the -h, --pre-hash option to also verify moved and diff --git a/TODO b/TODO index 07fdef35..4964e1c9 100644 --- a/TODO +++ b/TODO @@ -16,58 +16,6 @@ The content file is instead missing in all platforms. - Minor -* Check how to reduce memory occupation using smaller memory block. -We can reduce the memory occupation from 28 (32bit)/36 (64bit) to 17 bytes. -16 for the hash and 1 for the type. - -+ We can store the parity_position<->file,pos relation using some form of binary tree, -and then drop completely blockarr, and the extra data in memory_block. - -struct node { - uint32 parity_pos /* parity position */ - file* file_ptr; /* pointer at the file */ - uint32 file_pos; /* position in the file */ - uint32 count; /* number of sequential blocks */ - void* left_by_pp; /* tree by parity_pos */ - void* right_by_pp; - void* left_by_fp; /* tree by file,offset */ - void* right_by_fp; -}; - -4*8*number_of_fragments = 32*30000 = 960 KB -but we have to add the extra allocation for each small block, so it could be -(4*8+32)**number_of_fragments = 64*30000 = 1920 KB = 1.9 MB -Low memory occupation. -We can likely make a fast insertion at the end when loading, putting nodes in a list, -and later insert them in the tree. - -+ Or instead of a tree we can use two vectors with binary search. - -parity_pos -> file,pos -[parity_pos,file_ptr,file_pos] 4+4+4 - -file,pos -> parity_pos -[file_ptr,file_pos,parity_pos] 4+4+4 - -(3*4*2)*number_of_fragments=24*30000 = 720 KB = 0.7 MB -Less memory occupation, but slow to change in the middle, like removing and adding files. -Very fast to load from disk as we insert only at the end. - -+ We can also get rid of deleted blocks, as the hash will be stored anyway. -+ We can put the state into the hash, dropping 3 bits of the hash of the 128 ones. - -- A problem is that we are going to move hash from file to parity. -How to keep the hash of the file when the file is not in the parity ? -We'll have to allocate hash also in the file, and move data. -We can keep the hash ALWAYS on the file! -But how to handle DELETED blocks ? At now we handle them one-by-one, -We have to change to handle them like files, to allow to have DELETED files, inside the tree. -Here the problem is how to handle when DELETED blocks are overwritten. -Note that we need the UNDELETE state to allow to recover in FIX. -The HASH is also used to search for DELETED blocks. - -TODO CONVERT DELETED BLOCKS TO DELETE FILES - * Restore ownership and permissions, at least in Unix. * Restore directory timestamps. diff --git a/snapraid.1 b/snapraid.1 index fb44e22b..21159db0 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -1268,15 +1268,14 @@ A reason to use a different blocksize is if your system has less than 4 GiB of memory. As a rule of thumb, with 4 GiB or more memory use the default 256, with 2 GiB use 512, and with 1 GiB use 1024. .PP -In more details SnapRAID requires about TS*28/BS bytes -of RAM memory to run in the 32 bits version, and TS*36/BS -in the 64 bits one. Where TS is the total size in bytes of -your disk array, and BS is the block size in bytes. +In more details SnapRAID requires about TS*17/BS bytes of RAM memory. +Where TS is the total size in bytes of your disk array, and BS is the +block size in bytes. .PP -For example with 8 disk of 4 TB and a block size of 256 KiB +For example with 8 disks of 4 TB and a block size of 256 KiB (1 KiB = 1024 bytes) you have: .PP -RAM = (8 * 4 * 10^12) * 28 / (256 * 2^10) = 3.2 GiB +RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB .PD 0 .PP .PD diff --git a/snapraid.d b/snapraid.d index c656d086..98381241 100644 --- a/snapraid.d +++ b/snapraid.d @@ -985,15 +985,14 @@ Configuration than 4 GiB of memory. As a rule of thumb, with 4 GiB or more memory use the default 256, with 2 GiB use 512, and with 1 GiB use 1024. - In more details SnapRAID requires about TS*28/BS bytes - of RAM memory to run in the 32 bits version, and TS*36/BS - in the 64 bits one. Where TS is the total size in bytes of - your disk array, and BS is the block size in bytes. + In more details SnapRAID requires about TS*17/BS bytes of RAM memory. + Where TS is the total size in bytes of your disk array, and BS is the + block size in bytes. - For example with 8 disk of 4 TB and a block size of 256 KiB + For example with 8 disks of 4 TB and a block size of 256 KiB (1 KiB = 1024 bytes) you have: - :RAM = (8 * 4 * 10^12) * 28 / (256 * 2^10) = 3.2 GiB + :RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB Another reason to use a different blocksize is if you have a lot of small files. In the order of many millions. diff --git a/snapraid.txt b/snapraid.txt index be6c8544..785cefc3 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -1063,15 +1063,14 @@ A reason to use a different blocksize is if your system has less than 4 GiB of memory. As a rule of thumb, with 4 GiB or more memory use the default 256, with 2 GiB use 512, and with 1 GiB use 1024. -In more details SnapRAID requires about TS*28/BS bytes -of RAM memory to run in the 32 bits version, and TS*36/BS -in the 64 bits one. Where TS is the total size in bytes of -your disk array, and BS is the block size in bytes. +In more details SnapRAID requires about TS*17/BS bytes of RAM memory. +Where TS is the total size in bytes of your disk array, and BS is the +block size in bytes. -For example with 8 disk of 4 TB and a block size of 256 KiB +For example with 8 disks of 4 TB and a block size of 256 KiB (1 KiB = 1024 bytes) you have: -RAM = (8 * 4 * 10^12) * 28 / (256 * 2^10) = 3.2 GiB +RAM = (8 * 4 * 10^12) * 17 / (256 * 2^10) = 1.9 GiB Another reason to use a different blocksize is if you have a lot of small files. In the order of many millions. From 74ca71600d1933d0d5f23d8f1d232a3425eac050 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 4 Oct 2015 23:47:06 +0200 Subject: [PATCH 10/12] Optimize the access at the mapping trees We access them sequentially, and keeping the last accessed block is enough to improve the speed. --- cmdline/elem.c | 80 ++++++++++++++++++++++++++++++++++++++++++-------- cmdline/elem.h | 6 ++++ 2 files changed, 73 insertions(+), 13 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index a56924b4..5f3c1373 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -746,6 +746,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev tommy_hashdyn_init(&disk->dirset); tommy_tree_init(&disk->fs_parity, chunk_parity_compare); tommy_tree_init(&disk->fs_file, chunk_file_compare); + disk->fs_last = 0; return disk; } @@ -855,18 +856,28 @@ static int chunk_parity_inside_compare(const void* void_a, const void* void_b) return 0; } -struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) +static struct snapraid_chunk* fs_par2chunk_get(struct snapraid_disk* disk, block_off_t parity_pos) { - struct chunk_parity_inside arg = { parity_pos }; - struct snapraid_chunk* chunk = tommy_tree_search_compare(&disk->fs_parity, chunk_parity_inside_compare, &arg); + struct snapraid_chunk* chunk; + + /* check if the last accessed chunk matches */ + if (disk->fs_last + && parity_pos >= disk->fs_last->parity_pos + && parity_pos < disk->fs_last->parity_pos + disk->fs_last->count + ) { + chunk = disk->fs_last; + } else { + struct chunk_parity_inside arg = { parity_pos }; + chunk = tommy_tree_search_compare(&disk->fs_parity, chunk_parity_inside_compare, &arg); + } if (!chunk) return 0; - if (file_pos) - *file_pos = chunk->file_pos + (parity_pos - chunk->parity_pos); + /* store the last accessed chunk */ + disk->fs_last = chunk; - return chunk->file; + return chunk; } struct chunk_file_inside { @@ -895,11 +906,50 @@ static int chunk_file_inside_compare(const void* void_a, const void* void_b) return 0; } +static struct snapraid_chunk* fs_file2chunk_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) +{ + struct snapraid_chunk* chunk; + + /* check if the last accessed chunk matches */ + if (disk->fs_last + && file == disk->fs_last->file + && file_pos >= disk->fs_last->file_pos + && file_pos < disk->fs_last->file_pos + disk->fs_last->count + ) { + chunk = disk->fs_last; + } else { + struct chunk_file_inside arg = { file, file_pos }; + chunk = tommy_tree_search_compare(&disk->fs_file, chunk_file_inside_compare, &arg); + } + + if (!chunk) + return 0; + + /* store the last accessed chunk */ + disk->fs_last = chunk; + + return chunk; +} + +struct snapraid_file* fs_par2file_get(struct snapraid_disk* disk, block_off_t parity_pos, block_off_t* file_pos) +{ + struct snapraid_chunk* chunk; + + chunk = fs_par2chunk_get(disk, parity_pos); + if (!chunk) + return 0; + + if (file_pos) + *file_pos = chunk->file_pos + (parity_pos - chunk->parity_pos); + + return chunk->file; +} + block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) { - struct chunk_file_inside arg = { file, file_pos }; - struct snapraid_chunk* chunk = tommy_tree_search_compare(&disk->fs_file, chunk_file_inside_compare, &arg); + struct snapraid_chunk* chunk; + chunk = fs_file2chunk_get(disk, file, file_pos); if (!chunk) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for a file without parity in disk '%s'\n", disk->name); @@ -916,9 +966,7 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap if (file_pos > 0) { /* search an existing chunk for the previous file_pos */ - struct chunk_file_inside arg = { file, file_pos - 1 }; - - chunk = tommy_tree_search_compare(&disk->fs_file, chunk_file_inside_compare, &arg); + chunk = fs_file2chunk_get(disk, file, file_pos - 1); if (chunk != 0 && parity_pos == chunk->parity_pos + chunk->count) { /* ensure that we are really extending a chunk */ @@ -942,13 +990,16 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap /* insert te chunk in the trees */ tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk); tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk); + + /* store the last accessed chunk */ + disk->fs_last = chunk; } void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) { - struct chunk_parity_inside arg = { parity_pos }; - struct snapraid_chunk* chunk = tommy_tree_search_compare(&disk->fs_parity, chunk_parity_inside_compare, &arg); + struct snapraid_chunk* chunk; + chunk = fs_par2chunk_get(disk, parity_pos); if (!chunk) { /* LCOV_EXCL_START */ log_fatal("Internal inconsistency for clearing a not existing block in disk '%s'\n", disk->name); @@ -964,6 +1015,9 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) /* deallocate */ chunk_free(chunk); + + /* clear the last accessed chunk */ + disk->fs_last = 0; return; } diff --git a/cmdline/elem.h b/cmdline/elem.h index d8429eb2..b6e56b24 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -330,6 +330,12 @@ struct snapraid_disk { tommy_tree fs_parity; tommy_tree fs_file; + /** + * Last chunk we accessed. + * It's used to optimize access of sequential blocks. + */ + struct snapraid_chunk* fs_last; + /** * List of all the snapraid_file for the disk. */ From b17e618d49c26e5486d9bf98810967d7e37b3a1e Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 5 Oct 2015 21:40:44 +0200 Subject: [PATCH 11/12] Add an extensive parity filesystem check With the new mapping it's now possible to check everything in a very fast way. --- cmdline/elem.c | 208 +++++++++++++++++++++++++++++++++++++++++++++--- cmdline/elem.h | 16 ++++ cmdline/scan.c | 6 ++ cmdline/state.c | 54 +++++++------ cmdline/state.h | 6 ++ cmdline/sync.c | 9 ++- 6 files changed, 260 insertions(+), 39 deletions(-) diff --git a/cmdline/elem.c b/cmdline/elem.c index 5f3c1373..2b5628b3 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -771,7 +771,7 @@ struct chunk_disk_empty { }; /** - * Search for any block inside the specified blockmax. + * Compare the chunk if inside the specified blockmax. */ static int chunk_disk_empty_compare(const void* void_a, const void* void_b) { @@ -812,7 +812,7 @@ struct chunk_disk_size { }; /** - * Search for the chunk with the highest parity position. + * Compare the chunk by highest parity position. * * The maximum parity position is stored as size. */ @@ -836,12 +836,179 @@ block_off_t disk_size(struct snapraid_disk* disk) return arg.size; } +struct chunk_check { + const struct snapraid_chunk* prev; + int result; +}; + +void chunk_parity_check_foreach(void* void_arg, void* void_obj) +{ + struct chunk_check* arg = void_arg; + const struct snapraid_chunk* obj = void_obj; + const struct snapraid_chunk* prev = arg->prev; + + /* set the next previous block */ + arg->prev = obj; + + /* stop reporting if too many errors */ + if (arg->result > 100) { + /* LCOV_EXCL_START */ + return; + /* LCOV_EXCL_STOP */ + } + + /* check only if there is a previous block */ + if (!prev) + return; + + /* check the order */ + if (prev->parity_pos >= obj->parity_pos) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in parity order for files '%s' at '%u:%u' and '%s' at '%u:%u'\n", + prev->file->sub, prev->parity_pos, prev->count, obj->file->sub, obj->parity_pos, obj->count); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + + /* check that the chunks don't overlap */ + if (prev->parity_pos + prev->count > obj->parity_pos) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency for parity overlap for files '%s' at '%u:%u' and '%s' at '%u:%u'\n", + prev->file->sub, prev->parity_pos, prev->count, obj->file->sub, obj->parity_pos, obj->count); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } +} + +void chunk_file_check_foreach(void* void_arg, void* void_obj) +{ + struct chunk_check* arg = void_arg; + const struct snapraid_chunk* obj = void_obj; + const struct snapraid_chunk* prev = arg->prev; + + /* set the next previous block */ + arg->prev = obj; + + /* stop reporting if too many errors */ + if (arg->result > 100) { + /* LCOV_EXCL_START */ + return; + /* LCOV_EXCL_STOP */ + } + + /* note that for deleted files, some chunks may be missing */ + + /* if the files are different */ + if (!prev || prev->file != obj->file) { + if (prev != 0) { + if (file_flag_has(prev->file, FILE_IS_DELETED)) { + /* check that the chunk doesn't overflow the file */ + if (prev->file_pos + prev->count > prev->file->blockmax) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in delete end for file '%s' at '%u:%u' overflowing size '%u'\n", + prev->file->sub, prev->file_pos, prev->count, prev->file->blockmax); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + } else { + /* check that the chunk ends the file */ + if (prev->file_pos + prev->count != prev->file->blockmax) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in file end for file '%s' at '%u:%u' instead of size '%u'\n", + prev->file->sub, prev->file_pos, prev->count, prev->file->blockmax); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + } + } + + if (file_flag_has(obj->file, FILE_IS_DELETED)) { + /* check that the chunk doesn't overflow the file */ + if (obj->file_pos + obj->count > obj->file->blockmax) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in delete start for file '%s' at '%u:%u' overflowing size '%u'\n", + obj->file->sub, obj->file_pos, obj->count, obj->file->blockmax); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + } else { + /* check that the chunk starts the file */ + if (obj->file_pos != 0) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in file start for file '%s' at '%u:%u'\n", + obj->file->sub, obj->file_pos, obj->count); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + } + } else { + /* check the order */ + if (prev->file_pos >= obj->file_pos) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in file order for file '%s' at '%u:%u' and at '%u:%u'\n", + prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + + if (file_flag_has(obj->file, FILE_IS_DELETED)) { + /* check that the chunks don't overlap */ + if (prev->file_pos + prev->count > obj->file_pos) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in delete sequence for file '%s' at '%u:%u' and at '%u:%u'\n", + prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + } else { + /* check that the chunks are sequential */ + if (prev->file_pos + prev->count != obj->file_pos) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in file sequence for file '%s' at '%u:%u' and at '%u:%u'\n", + prev->file->sub, prev->file_pos, prev->count, obj->file_pos, obj->count); + ++arg->result; + return; + /* LCOV_EXCL_STOP */ + } + } + } +} + +int fs_check(struct snapraid_disk* disk) +{ + struct chunk_check arg; + + /* error count starts from 0 */ + arg.result = 0; + + /* check parity sequence */ + arg.prev = 0; + tommy_tree_foreach_arg(&disk->fs_parity, chunk_parity_check_foreach, &arg); + + /* check file sequence */ + arg.prev = 0; + tommy_tree_foreach_arg(&disk->fs_file, chunk_file_check_foreach, &arg); + + if (arg.result != 0) + return -1; + + return 0; +} + struct chunk_parity_inside { block_off_t parity_pos; }; /** - * Search for the chunk containing the specified parity position. + * Compare the chunk if containing the specified parity position. */ static int chunk_parity_inside_compare(const void* void_a, const void* void_b) { @@ -856,6 +1023,11 @@ static int chunk_parity_inside_compare(const void* void_a, const void* void_b) return 0; } +/** + * Seach the chunk at the specified parity position. + * The search is optimized for sequential accesses. + * \return If not found return 0 + */ static struct snapraid_chunk* fs_par2chunk_get(struct snapraid_disk* disk, block_off_t parity_pos) { struct snapraid_chunk* chunk; @@ -886,7 +1058,7 @@ struct chunk_file_inside { }; /** - * Search for the chunk containing the specified file position. + * Compare the chunk if containing the specified file position. */ static int chunk_file_inside_compare(const void* void_a, const void* void_b) { @@ -906,6 +1078,11 @@ static int chunk_file_inside_compare(const void* void_a, const void* void_b) return 0; } +/** + * Seach the chunk at the specified file position. + * The search is optimized for sequential accesses. + * \return If not found return 0 + */ static struct snapraid_chunk* fs_file2chunk_get(struct snapraid_disk* disk, struct snapraid_file* file, block_off_t file_pos) { struct snapraid_chunk* chunk; @@ -952,7 +1129,7 @@ block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* fi chunk = fs_file2chunk_get(disk, file, file_pos); if (!chunk) { /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for a file without parity in disk '%s'\n", disk->name); + log_fatal("Internal inconsistency when resolving a file to a not existing chunk in disk '%s'\n", disk->name); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -963,16 +1140,18 @@ block_off_t fs_file2par_get(struct snapraid_disk* disk, struct snapraid_file* fi void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snapraid_file* file, block_off_t file_pos) { struct snapraid_chunk* chunk; + struct snapraid_chunk* parity_chunk; + struct snapraid_chunk* file_chunk; if (file_pos > 0) { /* search an existing chunk for the previous file_pos */ chunk = fs_file2chunk_get(disk, file, file_pos - 1); if (chunk != 0 && parity_pos == chunk->parity_pos + chunk->count) { - /* ensure that we are really extending a chunk */ + /* ensure that we are extending the chunk at the end */ if (file_pos != chunk->file_pos + chunk->count) { /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency extending a chunk in disk '%s'\n", disk->name); + log_fatal("Internal inconsistency extending a chunk in the middle in disk '%s'\n", disk->name); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -988,8 +1167,15 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap chunk = chunk_alloc(parity_pos, file, file_pos, 1); /* insert te chunk in the trees */ - tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk); - tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk); + parity_chunk = tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk); + file_chunk = tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk); + + if (parity_chunk != chunk || file_chunk != chunk) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency when allocating a chunk that already exists in disk '%s'\n", disk->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } /* store the last accessed chunk */ disk->fs_last = chunk; @@ -1002,7 +1188,7 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) chunk = fs_par2chunk_get(disk, parity_pos); if (!chunk) { /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for clearing a not existing block in disk '%s'\n", disk->name); + log_fatal("Internal inconsistency when deallocating a not existing chunk in disk '%s'\n", disk->name); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } @@ -1036,7 +1222,7 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) } /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for clearing in the middle of a chunk in disk '%s'\n", disk->name); + log_fatal("Internal inconsistency when deallocating a chunk in the middle in disk '%s'\n", disk->name); exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } diff --git a/cmdline/elem.h b/cmdline/elem.h index b6e56b24..fb15a9f3 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -229,6 +229,16 @@ struct snapraid_block { */ #define FILE_IS_WITHOUT_INODE 0x200 +/** + * The file is deleted. + * This happens when a file is deleted from the array, + * but it's keep inside the parity until the next sync. + * + * During the filesystem check we needs this information, + * because deleted files may be present only partially. + */ +#define FILE_IS_DELETED 0x400 + #define FILE_IS_HARDLINK 0x1000 /**< If it's an hardlink. */ #define FILE_IS_SYMLINK 0x2000 /**< If it's a file symlink. */ #define FILE_IS_SYMDIR 0x4000 /**< If it's a dir symlink for Windows. Not yet supported. */ @@ -867,6 +877,12 @@ block_off_t disk_size(struct snapraid_disk* disk); */ int disk_is_empty(struct snapraid_disk* disk, block_off_t blockmax); +/** + * Check the filesystem for errors. + * Return 0 if it's OK. + */ +int fs_check(struct snapraid_disk* disk); + /** * Allocate a parity position for the specified file position. * diff --git a/cmdline/scan.c b/cmdline/scan.c index ee26c2e7..94b0e9a4 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -317,6 +317,9 @@ static void scan_file_deallocate(struct snapraid_scan* scan, struct snapraid_fil block_state_set(block, BLOCK_STATE_DELETED); } + /* mark the file as deleted */ + file_flag_set(file, FILE_IS_DELETED); + /* insert it in the list of deleted blocks */ tommy_list_insert_tail(&disk->deletedlist, &file->nodelist, file); } @@ -1772,6 +1775,9 @@ static int state_diffscan(struct snapraid_state* state, int is_diff) tommy_list_foreach(&scanlist, (tommy_foreach_func*)free); + /* check the filesystem on all disks */ + state_fscheck(state, "after scan"); + if (is_diff) { /* check for file difference */ if (!no_difference) diff --git a/cmdline/state.c b/cmdline/state.c index 52f9a630..6cc1f4fe 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1792,15 +1792,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST block_state_set(block, BLOCK_STATE_REP); } - /* we must not overwrite existing blocks */ - if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { - /* LCOV_EXCL_START */ - decoding_error(path, f); - log_fatal("Internal inconsistency in block existence!\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - /* set the parity association */ fs_allocate(disk, v_pos, file, v_idx); @@ -1958,7 +1949,10 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* if it's a run of deleted blocks */ /* allocate a fake deleted file */ - deleted = file_alloc(state->block_size, "", v_count * state->block_size, 0, 0, 0, 0); + deleted = file_alloc(state->block_size, "", v_count * state->block_size, 0, 0, 0, 0); + + /* mark the file as deleted */ + file_flag_set(deleted, FILE_IS_DELETED); /* insert it in the list of deleted files */ tommy_list_insert_tail(&disk->deletedlist, &deleted->nodelist, deleted); @@ -1986,21 +1980,8 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST hash_invalid_set(block->hash); } - /* we must not overwrite existing blocks */ - if (fs_par2block_get(disk, v_pos) != BLOCK_EMPTY) { - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency for used hole at pos %u!\n", v_pos); - if (state->opt.skip_content_check) { - log_fatal("Overriding as used.\n"); - } else { - decoding_error(path, f); - exit(EXIT_FAILURE); - } - /* LCOV_EXCL_STOP */ - } else { - /* insert the block in the block array */ - fs_allocate(disk, v_pos, deleted, v_idx); - } + /* insert the block in the block array */ + fs_allocate(disk, v_pos, deleted, v_idx); /* go to next block */ ++v_pos; @@ -2424,6 +2405,9 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* LCOV_EXCL_STOP */ } + /* check the filesystem on all disks */ + state_fscheck(state, "after read"); + /* check that the stored parity size matches the loaded state */ if (blockmax != parity_allocated_size(state)) { /* LCOV_EXCL_START */ @@ -2467,6 +2451,9 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) /* blocks of all array */ blockmax = parity_allocated_size(state); + /* check the filesystem on all disks */ + state_fscheck(state, "before write"); + /* clear the info for unused blocks */ /* and get some other info */ info_oldest = 0; /* oldest time in info */ @@ -3474,6 +3461,23 @@ void state_usage_print(struct snapraid_state* state) } } +void state_fscheck(struct snapraid_state* state, const char* ope) +{ + tommy_node* i; + + /* check the filesystem on all disks */ + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + + if (fs_check(disk) != 0) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency in filesystem for disk '%s' %s\n", disk->name, ope); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } +} + void generate_configuration(const char* path) { struct snapraid_state state; diff --git a/cmdline/state.h b/cmdline/state.h index 01ff0133..b5ce33bf 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -372,6 +372,12 @@ static inline void state_usage_parity(struct snapraid_state* state, unsigned lev */ void state_usage_print(struct snapraid_state* state); +/** + * Check the filesystem on all disks. + * On error it aborts. + */ +void state_fscheck(struct snapraid_state* state, const char* ope); + /****************************************************************************/ /* misc */ diff --git a/cmdline/sync.c b/cmdline/sync.c index db444d3d..7c9471e2 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1019,9 +1019,12 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari if (!io_error_on_this_block) { /* for each disk, mark the blocks as processed */ for (j = 0; j < diskmax; ++j) { - struct snapraid_block* block = BLOCK_EMPTY; - if (handle[j].disk) - block = fs_par2block_get(handle[j].disk, i); + struct snapraid_block* block; + + if (!handle[j].disk) + continue; + + block = fs_par2block_get(handle[j].disk, i); if (block == BLOCK_EMPTY) { /* nothing to do */ From 25fe6a252c4c7d861b58594b0647d34d8541de63 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 5 Oct 2015 22:48:16 +0200 Subject: [PATCH 12/12] Extend filesystem support for deallocation in the middle This could happen in some special cases --- Makefile.am | 27 +++++++++++++++++++++++++++ cmdline/elem.c | 33 ++++++++++++++++++++++++++++----- cmdline/elem.h | 2 +- cmdline/state.c | 2 +- 4 files changed, 57 insertions(+), 7 deletions(-) diff --git a/Makefile.am b/Makefile.am index e1bdf58e..c406f2b7 100644 --- a/Makefile.am +++ b/Makefile.am @@ -227,6 +227,33 @@ if HAVE_SYMLINK endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -a check + echo -- Filesystem allocation test + head -c 8192 /dev/zero > bench/disk1/TEST + head -c 8192 /dev/zero > bench/disk2/TEST + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync +# Now delete the file, and do a partial sync to force a deallocation in the middle of a file + rm bench/disk1/TEST + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E -S 1 -B 1 sync +# Now force a deallocation at the end + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E -S 7 -B 1 sync + rm bench/disk2/TEST + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E sync + echo -- Filesystem fragmentation test + head -c 8192 /dev/zero > bench/disk1/TEST1 + head -c 8192 /dev/zero > bench/disk1/TEST2 + head -c 8192 /dev/zero > bench/disk1/TEST3 + head -c 8192 /dev/zero > bench/disk1/TEST4 + head -c 8192 /dev/zero > bench/disk1/TEST5 + head -c 8192 /dev/zero > bench/disk1/TEST6 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync +# Now delete some files, and create a bigger one to fill the holes + rm bench/disk1/TEST1 + rm bench/disk1/TEST3 + rm bench/disk1/TEST5 + head -c 65536 /dev/zero > bench/disk1/TESTX + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync + rm -r bench/disk1/TEST* + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -E sync echo --- Create some files $(TESTENV) ./mktest$(EXEEXT) generate 1 6 $(CHECKCOUNT) $(CHECKSIZE) echo DUP > bench/disk1/DUP1 diff --git a/cmdline/elem.c b/cmdline/elem.c index 2b5628b3..6e79039a 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -1166,7 +1166,7 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap /* a chunk doesn't exist, and we have to create a new one */ chunk = chunk_alloc(parity_pos, file, file_pos, 1); - /* insert te chunk in the trees */ + /* insert the chunk in the trees */ parity_chunk = tommy_tree_insert(&disk->fs_parity, &chunk->parity_node, chunk); file_chunk = tommy_tree_insert(&disk->fs_file, &chunk->file_node, chunk); @@ -1184,6 +1184,10 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) { struct snapraid_chunk* chunk; + struct snapraid_chunk* second_chunk; + struct snapraid_chunk* parity_chunk; + struct snapraid_chunk* file_chunk; + block_off_t first_count, second_count; chunk = fs_par2chunk_get(disk, parity_pos); if (!chunk) { @@ -1221,10 +1225,29 @@ void fs_deallocate(struct snapraid_disk* disk, block_off_t parity_pos) return; } - /* LCOV_EXCL_START */ - log_fatal("Internal inconsistency when deallocating a chunk in the middle in disk '%s'\n", disk->name); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + /* otherwise it's in the middle */ + first_count = parity_pos - chunk->parity_pos; + second_count = chunk->count - first_count - 1; + + /* adjust the first chunk */ + chunk->count = first_count; + + /* allocate the second chunk */ + second_chunk = chunk_alloc(chunk->parity_pos + first_count + 1, chunk->file, chunk->file_pos + first_count + 1, second_count); + + /* insert the chunk in the trees */ + parity_chunk = tommy_tree_insert(&disk->fs_parity, &second_chunk->parity_node, second_chunk); + file_chunk = tommy_tree_insert(&disk->fs_file, &second_chunk->file_node, second_chunk); + + if (parity_chunk != second_chunk || file_chunk != second_chunk) { + /* LCOV_EXCL_START */ + log_fatal("Internal inconsistency when splitting a chunk in disk '%s'\n", disk->name); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + /* store the last accessed chunk */ + disk->fs_last = second_chunk; } struct snapraid_block* fs_par2block_get(struct snapraid_disk* disk, block_off_t parity_pos) diff --git a/cmdline/elem.h b/cmdline/elem.h index fb15a9f3..bac5dd34 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -894,7 +894,7 @@ void fs_allocate(struct snapraid_disk* disk, block_off_t parity_pos, struct snap /** * Deallocate the parity position. * - * After this call you can the par2file/par2block operations + * After this call the par2file/par2block operations * won't find anymore the parity association. */ void fs_deallocate(struct snapraid_disk* disk, block_off_t pos); diff --git a/cmdline/state.c b/cmdline/state.c index 6cc1f4fe..ba176bb2 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1403,7 +1403,7 @@ static int position_is_required(struct snapraid_state* state, block_off_t pos) * Checks if the info block is REQUIREQ. * * This is used to ensure that we keep the last check used for scrubbing. - * and the we add it when importing old context files. + * and that we add it when importing old context files. * * Note that you can have position without info blocks, for example * if all the blocks are not synced.