diff --git a/cmdline/locate.c b/cmdline/locate.c index 29aece03..169c5eeb 100644 --- a/cmdline/locate.c +++ b/cmdline/locate.c @@ -24,7 +24,7 @@ #include "locate.h" struct snapraid_parity_entry { - block_off_t block_size; + uint32_t block_size; block_off_t low; /**< Lower position in the parity */ block_off_t high; /**< Higher position in the parity */ block_off_t fragments; /**< Number of fragments in the parity */ @@ -33,6 +33,13 @@ struct snapraid_parity_entry { tommy_node node; }; +struct snapraid_locate_info { + block_off_t block_max; /**< Number of blocks in the parity */ + data_off_t parity_size; /**< Size in bytes of the parity */ + block_off_t tail_block; /**< Number of blocks of the tail of the parity */ + block_off_t low_to_free_tail_block; /**< First block that has to be freed */ +}; + static int parity_entry_compare(const void* void_a, const void* void_b) { const struct snapraid_parity_entry* entry_a = void_a; @@ -69,7 +76,7 @@ static void add_size_to_sum(void* arg, void* void_entry) *size_sum += entry->file->size; } -static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk* disk, struct snapraid_file* file, tommy_list* file_list, block_off_t low_occupied_block_number) +static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk* disk, struct snapraid_file* file, tommy_list* file_list, block_off_t low_to_free_tail_block) { block_off_t parity_low = 0; /* lower block */ block_off_t parity_high = 0; /* higher block */ @@ -98,8 +105,8 @@ static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk* if (parity_fragments == 0) return; /* not allocated */ - if (low_occupied_block_number > 0 && parity_high < low_occupied_block_number) - return; /* entry not relevant */ + if (low_to_free_tail_block > 0 && parity_high < low_to_free_tail_block) + return; /* entry not in the range to free */ /* found a relevant block so add the corresponding file */ struct snapraid_parity_entry* entry = malloc_nofail(sizeof(struct snapraid_parity_entry)); @@ -112,38 +119,44 @@ static void collect_parity_block_file(uint32_t block_size, struct snapraid_disk* tommy_list_insert_tail(file_list, &entry->node, entry); } +void state_locate_info(struct snapraid_state* state, uint64_t parity_tail, struct snapraid_locate_info* info) +{ + uint32_t block_size = state->block_size; + + info->block_max = parity_allocated_size(state); + info->parity_size = info->block_max * (uint64_t)block_size; + info->tail_block = (parity_tail + block_size - 1) / block_size; + info->low_to_free_tail_block = info->block_max - info->tail_block; +} + void state_locate(struct snapraid_state* state, uint64_t parity_tail) { char buf[64]; - uint64_t block_size = state->block_size; - - block_off_t min_occupied_block_number; + uint32_t block_size = state->block_size; + block_off_t low_to_free_tail_block; printf("SnapRAID locate report:\n"); printf("\n"); if (parity_tail == 0) { printf("Locate all files\n\n"); - min_occupied_block_number = 0; + low_to_free_tail_block = 0; } else { printf("Locate files within the tail of %sB of the parity\n\n", fmt_size(parity_tail, buf, sizeof(buf))); - data_off_t block_max = parity_allocated_size(state); - data_off_t parity_size = block_max * block_size; + struct snapraid_locate_info info; + state_locate_info(state, parity_tail, &info); - printf("Current parity size is %sB\n", fmt_size(parity_size, buf, sizeof(buf))); - - block_off_t tail_block = (parity_tail + block_size - 1) / block_size; - - if (tail_block >= block_max) { + printf("Current parity size is %sB\n", fmt_size(info.parity_size, buf, sizeof(buf))); + if (info.tail_block >= info.block_max) { printf("Specified tail greater than the parity size!\n"); return; } - min_occupied_block_number = block_max - tail_block; + low_to_free_tail_block = info.low_to_free_tail_block; } - msg_progress("Collecting files with offset greater or equal to %" PRIu64 "\n", min_occupied_block_number * block_size); + msg_progress("Collecting files with offset greater or equal to %" PRIu64 "\n", low_to_free_tail_block * (uint64_t)block_size); tommy_list files; tommy_list_init(&files); @@ -151,7 +164,7 @@ void state_locate(struct snapraid_state* state, uint64_t parity_tail) struct snapraid_disk* disk = i->data; for (tommy_node* j = tommy_list_head(&disk->filelist); j != 0; j = j->next) { struct snapraid_file* file = j->data; - collect_parity_block_file(block_size, disk, file, &files, min_occupied_block_number); + collect_parity_block_file(block_size, disk, file, &files, low_to_free_tail_block); } } @@ -179,3 +192,61 @@ void state_locate(struct snapraid_state* state, uint64_t parity_tail) tommy_list_foreach(&files, free); } +void state_locate_mark_tail_blocks_for_resync(struct snapraid_state* state, uint64_t parity_tail) +{ + char buf[64]; + uint32_t block_size = state->block_size; + + struct snapraid_locate_info info; + state_locate_info(state, parity_tail, &info); + block_off_t low_to_free_tail_block = info.low_to_free_tail_block; + + msg_progress("Forcing reallocation of all files within the tail of %sB of the parity\n\n", fmt_size(parity_tail, buf, sizeof(buf))); + + msg_progress("Collecting files with offset greater or equal to %" PRIu64 "\n", low_to_free_tail_block * (uint64_t)block_size); + + tommy_list files; + tommy_list_init(&files); + for (tommy_node* i = tommy_list_head(&state->disklist); i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + for (tommy_node* j = tommy_list_head(&disk->filelist); j != 0; j = j->next) { + struct snapraid_file* file = j->data; + collect_parity_block_file(block_size, disk, file, &files, low_to_free_tail_block); + } + } + + printf("\n"); + + if (tommy_list_count(&files) == 0) { + printf("No files located in the specified parity tail.\n"); + } else { + /* process all the files partiall or fully overlapping the free zone */ + for (tommy_node* j = tommy_list_head(&files); j != 0; j = j->next) { + struct snapraid_parity_entry* entry = j->data; + struct snapraid_file* file = entry->file; + struct snapraid_disk* disk = entry->disk; + + /* reallocate the full file, not only the part of in the free zone */ + /* this is required because the files blocks have to be in order in */ + /* in the parity file. */ + /* not reallocating the file head will prevent the reallocation of the file tail */ + for (block_off_t f = 0; f < file->blockmax; ++f) { + block_off_t parity_pos = fs_file2par_find(disk, file, f); + + if (parity_pos == POS_NULL) { + /* block not yet allocated */ + continue; + } + + struct snapraid_block* block = fs_file2block_get(file, f); + if (block_state_get(block) == BLOCK_STATE_BLK) { + /* convert from BLK to REP */ + block_state_set(block, BLOCK_STATE_REP); + } + } + } + } + + tommy_list_foreach(&files, free); +} + diff --git a/cmdline/locate.h b/cmdline/locate.h index b9c99ddf..0e6f4cfa 100644 --- a/cmdline/locate.h +++ b/cmdline/locate.h @@ -25,5 +25,7 @@ void state_locate(struct snapraid_state* state, uint64_t parity_tail); +void state_locate_mark_tail_blocks_for_resync(struct snapraid_state* state, uint64_t parity_tail); + #endif diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index c6793075..85bcff9d 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -74,8 +74,7 @@ void usage(const char* conf) printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n"); printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands with inaccessible/shared disks\n"); printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n"); - printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force a full parity computation in sync\n"); - printf(" " SWITCH_GETOPT_LONG("-R, --force-realloc ", "-R") " Force a full parity reallocation in sync\n"); + /* --force-full, --force-realloc and --force-realloc-tail are not listed as they are dangerous */ printf(" " SWITCH_GETOPT_LONG("-w, --bw-limit RATE ", "-w") " Limit IO bandwidth (M|G)\n"); printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); printf("\n"); @@ -712,6 +711,7 @@ static struct option long_options[] = { { "force-nocopy", 0, 0, 'N' }, { "force-full", 0, 0, 'F' }, { "force-realloc", 0, 0, 'R' }, + { "force-realloc-tail", 1, 0, 'W' }, { "bw-limit", 1, 0, 'w' }, { "audit-only", 0, 0, 'a' }, { "pre-hash", 0, 0, 'h' }, @@ -870,12 +870,12 @@ static struct option long_options[] = { #endif /* - * Free letters: gIjJkKMnPQruxXWz + * Free letters: gIjJkKMnPQruxXz * * The 's' letter is used in main.c * The 'G' letter is free but only from 14.0 */ -#define OPTIONS "t:c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRahTC:vqHVw:" +#define OPTIONS "t:c:f:d:mebp:o:S:B:L:i:l:AZEUDNFRW:ahTC:vqHVw:" int parse_option_size(const char* arg, uint64_t* out_size) { @@ -983,7 +983,6 @@ int snapraid_main(int argc, char* argv[]) int speed_test_blocks_size; time_t t; struct tm* tm; - uint64_t parity_tail = 0; #if HAVE_LOCALTIME_R struct tm tm_res; #endif @@ -1145,7 +1144,7 @@ int snapraid_main(int argc, char* argv[]) import_timestamp = optarg; break; case 't' : - if (parse_option_size(optarg, &parity_tail) != 0) { + if (parse_option_size(optarg, &opt.parity_tail) != 0) { /* LCOV_EXCL_START */ log_fatal(EUSER, "Invalid tail size '%s'\n", optarg); exit(EXIT_FAILURE); @@ -1194,6 +1193,15 @@ int snapraid_main(int argc, char* argv[]) case 'R' : opt.force_realloc = 1; break; + case 'W' : + opt.force_realloc = 1; + if (parse_option_size(optarg, &opt.parity_tail) != 0) { + /* LCOV_EXCL_START */ + log_fatal(EUSER, "Invalid tail size '%s'\n", optarg); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + break; case 'a' : opt.auditonly = 1; break; @@ -1810,6 +1818,11 @@ int snapraid_main(int argc, char* argv[]) state_read(&state); + /* mark the files that have to be reallocated */ + /* it will happen in inside scan_file_keep() called in state_scan() */ + if (state.opt.force_realloc && opt.parity_tail != 0) + state_locate_mark_tail_blocks_for_resync(&state, opt.parity_tail); + state_scan(&state); /* refresh the size info before the content write */ @@ -1924,7 +1937,7 @@ int snapraid_main(int argc, char* argv[]) } else if (operation == OPERATION_LOCATE) { state_read(&state); - state_locate(&state, parity_tail); + state_locate(&state, opt.parity_tail); } else if (operation == OPERATION_DUP) { state_read(&state); diff --git a/cmdline/state.c b/cmdline/state.c index 34bd792b..9723f2fa 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2235,7 +2235,7 @@ static void state_read_content(struct snapraid_state* state, const char* path, S /* if we want a full reallocation, marks block as invalid parity */ /* note that we do this after the force_nocopy option */ /* to avoid to mixup the two things */ - if (state->opt.force_realloc + if (state->opt.force_realloc && state->opt.parity_tail == 0 && block_state_get(block) == BLOCK_STATE_BLK) { /* convert from BLK to REP */ block_state_set(block, BLOCK_STATE_REP); diff --git a/cmdline/state.h b/cmdline/state.h index 2709e63d..1851bdfe 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -81,6 +81,7 @@ struct snapraid_option { int force_nocopy; /**< Force dangerous operations of syncing files without using copy detection. */ int force_full; /**< Force a full parity update. */ int force_realloc; /**< Force a full reallocation and parity update. */ + uint64_t parity_tail; /**< Limit the reallocation of the location at the specified parity tail */ int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */ int expect_recoverable; /**< Expect presence of recoverable error in checking. */ int skip_device; /**< Skip devices matching checks. */