From ad57aafd8fdbed9f2f843f2bf516bc928fc9c8b7 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 1 Feb 2015 18:48:50 +0100 Subject: [PATCH] Refactor the parity size detection before sync. Now we resize the parity only after checking the size of all the parity file. --- cmdline/check.c | 2 +- cmdline/dry.c | 2 +- cmdline/elem.h | 15 +++++++++++++- cmdline/parity.c | 33 +++++++++++++++++++++++++++---- cmdline/parity.h | 20 +++++++++++++++++-- cmdline/rehash.c | 2 +- cmdline/scrub.c | 2 +- cmdline/state.c | 31 ++++++----------------------- cmdline/state.h | 14 ------------- cmdline/status.c | 2 +- cmdline/sync.c | 51 +++++++++++++++++++++++++++++++++++++----------- 11 files changed, 112 insertions(+), 62 deletions(-) diff --git a/cmdline/check.c b/cmdline/check.c index 7f1978b9..e7de94d5 100644 --- a/cmdline/check.c +++ b/cmdline/check.c @@ -1855,7 +1855,7 @@ int state_check(struct snapraid_state* state, int fix, block_off_t blockstart, b printf("Initializing...\n"); - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); size = blockmax * (data_off_t)state->block_size; if (blockstart > blockmax) { diff --git a/cmdline/dry.c b/cmdline/dry.c index 8b3b612e..0f915b6e 100644 --- a/cmdline/dry.c +++ b/cmdline/dry.c @@ -187,7 +187,7 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t printf("Drying...\n"); - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); if (blockstart > blockmax) { /* LCOV_EXCL_START */ diff --git a/cmdline/elem.h b/cmdline/elem.h index 0ac0c8bb..9a16aafb 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -544,7 +544,8 @@ static inline int block_has_updated_hash(const struct snapraid_block* block) } /** - * Checks if the specified block has an a past hash. + * Checks if the specified block has a past hash, + * i.e. the hash of the data that it's now overwritten or lost. * * Note that EMPTY / BLK / REP return 0. */ @@ -637,6 +638,18 @@ static inline int block_has_invalid_parity(const struct snapraid_block* block) || state == BLOCK_STATE_CHG || state == BLOCK_STATE_REP; } +/** + * Checks if the block is part of a file with valid parity. + * + * Note that anything different than BLK return 0. + */ +static inline int block_has_file_and_valid_parity(const struct snapraid_block* block) +{ + unsigned state = block_state_get(block); + + return state == BLOCK_STATE_BLK; +} + /** * Gets the relative position of a block inside the file. */ diff --git a/cmdline/parity.c b/cmdline/parity.c index 880541df..c7df2c62 100644 --- a/cmdline/parity.c +++ b/cmdline/parity.c @@ -26,7 +26,34 @@ /****************************************************************************/ /* parity */ -block_off_t parity_size(struct snapraid_state* state) +block_off_t parity_allocated_size(struct snapraid_state* state) +{ + block_off_t parity_block; + tommy_node* i; + + /* compute the size of the parity file */ + parity_block = 0; + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + + /* start from the declared size */ + block_off_t block = tommy_arrayblk_size(&disk->blockarr); + + /* decrease the block until an allocated one, but part of a file */ + /* we don't stop at deleted blocks, because we want to have them cleared */ + /* if they are at the end of the parity */ + while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1))) + --block; + + /* get the highest value */ + if (block > parity_block) + parity_block = block; + } + + return parity_block; +} + +block_off_t parity_used_size(struct snapraid_state* state) { block_off_t parity_block; tommy_node* i; @@ -40,9 +67,7 @@ block_off_t parity_size(struct snapraid_state* state) block_off_t block = tommy_arrayblk_size(&disk->blockarr); /* decrease the block until an used one */ - /* we don't stop at deleted blocks, because we want to have them cleared */ - /* if they are at the end of the parity */ - while (block > parity_block && !block_has_file(tommy_arrayblk_get(&disk->blockarr, block - 1))) + while (block > parity_block && !block_has_file_and_valid_parity(tommy_arrayblk_get(&disk->blockarr, block - 1))) --block; /* get the highest value */ diff --git a/cmdline/parity.h b/cmdline/parity.h index 26afbada..3f7a7377 100644 --- a/cmdline/parity.h +++ b/cmdline/parity.h @@ -29,9 +29,25 @@ struct snapraid_parity_handle { }; /** - * Computes the size of the parity data in number of blocks. + * Computes the size of the allocated parity data in number of blocks. + * + * This includes parity blocks not yet written and still invalid. */ -block_off_t parity_size(struct snapraid_state* state); +block_off_t parity_allocated_size(struct snapraid_state* state); + +/** + * Computes the size of the used parity data in number of blocks. + * + * This includes only parity blocks used for files, not counting + * potential invalid parity at the end. + * + * If the array is fully synched there is no difference between + * parity_allocate_size() and parity_used_size(). + * But if the sync is interrupted, the parity_used_size() returns + * the position of the latest BLK block, ignoring CHG, REL and DELETED ones, + * because their parity may be still not even written in the parity file. + */ +block_off_t parity_used_size(struct snapraid_state* state); /** * Reports all the files outside the specified parity size. diff --git a/cmdline/rehash.c b/cmdline/rehash.c index 2116d572..6151cb09 100644 --- a/cmdline/rehash.c +++ b/cmdline/rehash.c @@ -33,7 +33,7 @@ void state_rehash(struct snapraid_state* state) block_off_t blockmax; block_off_t i; - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); /* check if a rehash is already in progress */ if (state->prevhash != HASH_UNDEFINED) { diff --git a/cmdline/scrub.c b/cmdline/scrub.c index 3961dcd9..3b042744 100644 --- a/cmdline/scrub.c +++ b/cmdline/scrub.c @@ -567,7 +567,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan) printf("Initializing...\n"); - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); if (state->opt.force_scrub_even) { /* no limit */ diff --git a/cmdline/state.c b/cmdline/state.c index c085a005..cb9b103f 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -99,7 +99,6 @@ void state_init(struct snapraid_state* state) state->pool_device = 0; state->lockfile[0] = 0; state->level = 1; /* default is the lowest protection */ - state->loaded_paritymax = 0; state->clear_past_hash = 0; state->no_conf = 0; @@ -1338,7 +1337,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE struct snapraid_file* file; block_off_t blockidx; block_off_t blockmax; - block_off_t paritymax; unsigned line; unsigned count_file; unsigned count_hardlink; @@ -1351,7 +1349,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE line = 1; blockidx = 0; blockmax = 0; - paritymax = 0; count_file = 0; count_hardlink = 0; count_symlink = 0; @@ -1420,10 +1417,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE switch (tag[0]) { case 'b' : block_state_set(block, BLOCK_STATE_BLK); - - /* keep track of the required parity size */ - if (v_pos + 1 > paritymax) - paritymax = v_pos + 1; break; case 'n' : /* deprecated NEW blocks are converted to CHG ones */ @@ -2383,7 +2376,7 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } /* check that the stored parity size matches the loaded state */ - if (blockmax != parity_size(state)) { + if (blockmax != parity_allocated_size(state)) { /* LCOV_EXCL_START */ fprintf(stderr, "Internal inconsistency in parity size in '%s' at line %u\n", path, line); exit(EXIT_FAILURE); @@ -2410,9 +2403,6 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE } } - /* set the required parity size */ - state->loaded_paritymax = paritymax; - if (state->opt.verbose) { printf("%8u files\n", count_file); printf("%8u hardlinks\n", count_hardlink); @@ -2439,7 +2429,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f) count_dir = 0; /* blocks of all array */ - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); /* clear the info for unused blocks */ /* and get some other info */ @@ -2883,7 +2873,6 @@ static void decoding_error(const char* path, STREAM* f) static void state_read_binary(struct snapraid_state* state, const char* path, STREAM* f) { block_off_t blockmax; - block_off_t paritymax; unsigned count_file; unsigned count_hardlink; unsigned count_symlink; @@ -2895,7 +2884,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST uint32_t mapping_max; blockmax = 0; - paritymax = 0; count_file = 0; count_hardlink = 0; count_symlink = 0; @@ -3098,10 +3086,6 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST switch (c) { case 'b' : block_state_set(block, BLOCK_STATE_BLK); - - /* keep track of the required parity size */ - if (v_pos + 1 > paritymax) - paritymax = v_pos + 1; break; case 'n' : /* deprecated NEW blocks are converted to CHG ones */ @@ -3809,21 +3793,18 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST } /* check that the stored parity size matches the loaded state */ - if (blockmax != parity_size(state)) { + if (blockmax != parity_allocated_size(state)) { /* LCOV_EXCL_START */ fprintf(stderr, "Internal inconsistency in parity size in '%s' at offset %" PRIi64 "\n", path, stell(f)); if (state->opt.skip_content_check) { - fprintf(stderr, "Overriding from %u to %u.\n", blockmax, parity_size(state)); - blockmax = parity_size(state); + fprintf(stderr, "Overriding from %u to %u.\n", blockmax, parity_allocated_size(state)); + blockmax = parity_allocated_size(state); } else { exit(EXIT_FAILURE); } /* LCOV_EXCL_STOP */ } - /* set the required parity size */ - state->loaded_paritymax = paritymax; - if (state->opt.verbose) { printf("%8u files\n", count_file); printf("%8u hardlinks\n", count_hardlink); @@ -3853,7 +3834,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f) count_dir = 0; /* blocks of all array */ - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); /* clear the info for unused blocks */ /* and get some other info */ diff --git a/cmdline/state.h b/cmdline/state.h index 1e391323..2a74f54a 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -150,20 +150,6 @@ struct snapraid_state { */ uint64_t tick_last; - /** - * Required size of the parity file, computed from the loaded state. - * - * This size only counts BLK blocks, ignoring CHG, REL and DELETED ones, - * because in such case the parity may be still not updated to contain them. - * - * In normal case it's also the blockmax size returned by parity_size(). - * In case of interrupted sync, this is the position + 1 of the last BLK block. - * Potentionally smaller than parity_size(). - * - * This value is used to check the minimal parity size before starting a sync. - */ - block_off_t loaded_paritymax; - int clear_past_hash; /**< Clear all the hash from CHG and DELETED blocks when reading the state from an incomplete sync. */ time_t progress_whole_start; /**< Initial start of the whole process. */ diff --git a/cmdline/status.c b/cmdline/status.c index 20051b10..64d5a90f 100644 --- a/cmdline/status.c +++ b/cmdline/status.c @@ -80,7 +80,7 @@ int state_status(struct snapraid_state* state) /* keep track if at least a free info is available */ free_not_zero = 0; - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); fprintf(stdlog, "summary:block_size:%u\n", state->block_size); fprintf(stdlog, "summary:parity_block_count:%u\n", blockmax); diff --git a/cmdline/sync.c b/cmdline/sync.c index daa0a398..5452adca 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -1181,7 +1181,8 @@ bail: int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount) { block_off_t blockmax; - data_off_t loaded_size; + block_off_t used_paritymax; + block_off_t file_paritymax; data_off_t size; data_off_t out_size; int ret; @@ -1195,9 +1196,14 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t printf("Initializing...\n"); - blockmax = parity_size(state); + blockmax = parity_allocated_size(state); size = blockmax * (data_off_t)state->block_size; - loaded_size = state->loaded_paritymax * (data_off_t)state->block_size; + + /* minum size of the parity files we expect */ + used_paritymax = parity_used_size(state); + + /* effective size of the parity files */ + file_paritymax = 0; if (blockstart > blockmax) { /* LCOV_EXCL_START */ @@ -1212,6 +1218,8 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t } for (l = 0; l < state->level; ++l) { + block_off_t parityblocks; + /* create the file and open for writing */ parity_ptr[l] = &parity[l]; ret = parity_create(parity_ptr[l], state->parity[l].path, &out_size, state->file_mode); @@ -1222,20 +1230,41 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t /* LCOV_EXCL_STOP */ } - /* if we do a full sync, having a wrong parity size is expected */ - if (!state->opt.force_full) { - /* if the file is too small */ - if (out_size < loaded_size) { - /* LCOV_EXCL_START */ - fprintf(stderr, "DANGER! The %s file %s is smaller than the expected %" PRId64 ".\n", lev_name(l), state->parity[l].path, loaded_size); + /* number of block in the parity file */ + parityblocks = out_size / state->block_size; + + /* if the file is too small */ + if (parityblocks < used_paritymax) { + fprintf(stderr, "WARNING! The %s file %s has data only %u blocks instead of %u.\n", lev_name(l), state->parity[l].path, parityblocks, used_paritymax); + } + + /* keep the smallest parity number of blocks */ + if (l == 0 || file_paritymax > parityblocks) + file_paritymax = parityblocks; + } + + /* if we do a full sync, having a wrong parity size is expected */ + if (!state->opt.force_full) { + /* if the parities are too small */ + if (file_paritymax < used_paritymax) { + /* LCOV_EXCL_START */ + fprintf(stderr, "DANGER! One or more the parity files are smaller than expected!\n"); + if (file_paritymax != 0) { fprintf(stderr, "If this happens because you are using an old content file,\n"); fprintf(stderr, "you can 'sync' anyway using 'snapraid --force-full sync'\n"); fprintf(stderr, "to force a full rebuild of the parity.\n"); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ + } else { + fprintf(stderr, "It's possible that the parity disks are not mounted.\n"); + fprintf(stderr, "If instead you are adding a new parity level, you can 'sync' using\n"); + fprintf(stderr, "'snapraid --force-full sync' to force a full rebuild of the parity.\n"); } + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ } + } + /* now change the size of all parities */ + for (l = 0; l < state->level; ++l) { /* change the size of the parity file, truncating or extending it */ /* from this point all the DELETED blocks after the end of the parity are invalid */ /* and they are automatically removed when we save the new content file */