diff --git a/HISTORY b/HISTORY index c0dcf35c..312c5f39 100644 --- a/HISTORY +++ b/HISTORY @@ -23,6 +23,10 @@ SnapRAID HISTORY * In Windows allows to use the escape char '^' to handle file patterns containing real characters matching the globbing '*?[]' ones. In Unix it was already possible to do the same escaping with '\'. + * Added a new -R, --force-realloc option to reallocate all the + parity information keeping the precomputed hash. + This is the previous -F, --force-full that instead now maintains the + same parity organization and just recomputes it. * Added test options for selecting the file advise mode to use: --test-io-advise-none for standard mode --test-io-advise-sequential advise sequential access (Linux/Windows) diff --git a/Makefile.am b/Makefile.am index c0bd988a..76afc13e 100644 --- a/Makefile.am +++ b/Makefile.am @@ -540,8 +540,10 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check --test-expect-recoverable $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) fix $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -p full scrub - $(MSG) Full sync forced + $(MSG) Full sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -F sync + $(MSG) Realloc sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -R sync #### SYNC PARTIAL #### $(MSG) Abort sync with additions. Delete some of them, and add others and sync again. $(MSG) This triggers files reallocation inside parity diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 6b70252d..5cfe2803 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -71,7 +71,8 @@ void usage(void) printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n"); printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands with inaccessible/shared disks\n"); printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n"); - printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force commands requiring a full sync\n"); + printf(" " SWITCH_GETOPT_LONG("-F, --force-full ", "-F") " Force a full parity computation in sync\n"); + printf(" " SWITCH_GETOPT_LONG("-R, --force-realloc ", "-R") " Force a full parity reallocation in sync\n"); printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); } @@ -313,6 +314,7 @@ struct option long_options[] = { { "force-device", 0, 0, 'D' }, { "force-nocopy", 0, 0, 'N' }, { "force-full", 0, 0, 'F' }, + { "force-realloc", 0, 0, 'R' }, { "audit-only", 0, 0, 'a' }, { "pre-hash", 0, 0, 'h' }, { "speed-test", 0, 0, 'T' }, /* undocumented speed test command */ @@ -412,7 +414,7 @@ struct option long_options[] = { /* Match first UUID */ { "test-match-first-uuid", 0, 0, OPT_TEST_MATCH_FIRST_UUID }, - /* Force parity update even if all the data is not changed */ + /* Force parity update even if all the data hash is already matching */ { "test-force-parity-update", 0, 0, OPT_TEST_FORCE_PARITY_UPDATE }, /* Number of IO buffers */ @@ -455,7 +457,7 @@ struct option long_options[] = { }; #endif -#define OPTIONS "c:f:d:mep:o:S:B:L:i:l:ZEUDNFahTC:vqHVG" +#define OPTIONS "c:f:d:mep:o:S:B:L:i:l:ZEUDNFRahTC:vqHVG" volatile int global_interrupt = 0; @@ -705,6 +707,9 @@ int main(int argc, char* argv[]) case 'F' : opt.force_full = 1; break; + case 'R' : + opt.force_realloc = 1; + break; case 'a' : opt.auditonly = 1; break; @@ -1006,6 +1011,13 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } + + if (opt.force_realloc) { + /* LCOV_EXCL_START */ + log_fatal("You cannot use -R, --force-realloc with the '%s' command\n", command); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } } if (opt.force_full && opt.force_nocopy) { @@ -1015,6 +1027,20 @@ int main(int argc, char* argv[]) /* LCOV_EXCL_STOP */ } + if (opt.force_realloc && opt.force_nocopy) { + /* LCOV_EXCL_START */ + log_fatal("You cannot use the -R, --force-realloc and -N, --force-nocopy options at the same time\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + if (opt.force_realloc && opt.force_full) { + /* LCOV_EXCL_START */ + log_fatal("You cannot use the -R, --force-realloc and -F, --force-full options at the same time\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + if (opt.prehash && opt.force_nocopy) { /* LCOV_EXCL_START */ log_fatal("You cannot use the -h, --pre-hash and -N, --force-nocopy options at the same time\n"); diff --git a/cmdline/state.c b/cmdline/state.c index 0e1b31d6..58ee503e 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -2013,10 +2013,10 @@ static void state_read_content(struct snapraid_state* state, const char* path, S block_state_set(block, BLOCK_STATE_CHG); } - /* if we want a full sync, marks block as invalid parity */ + /* if we want a full reallocation, marks block as invalid parity */ /* note that we do this after the force_nocopy option */ /* to avoid to mixup the two things */ - if (state->opt.force_full + if (state->opt.force_realloc && block_state_get(block) == BLOCK_STATE_BLK) { /* convert from BLK to REP */ block_state_set(block, BLOCK_STATE_REP); diff --git a/cmdline/state.h b/cmdline/state.h index 12e48b9b..188d0a22 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -82,7 +82,8 @@ struct snapraid_option { int force_uuid; /**< Forced dangerous operations of syncing disks with uuid changed. */ int force_device; /**< Forced dangerous operations of using disks with save device id. */ int force_nocopy; /**< Force dangerous operations of syncing files without using copy detection. */ - int force_full; /**< Force a full sync when using an old content file. */ + int force_full; /**< Force a full parity update. */ + int force_realloc; /**< Force a full reallocation and parity update. */ int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */ int expect_recoverable; /**< Expect presence of recoverable error in checking. */ int skip_device; /**< Skip devices matching checks. */ diff --git a/cmdline/sync.c b/cmdline/sync.c index 6f00d7ab..7d4b6c8c 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -399,6 +399,7 @@ finish: struct snapraid_plan { unsigned handle_max; struct snapraid_handle* handle_map; + int force_full; }; /** @@ -460,7 +461,7 @@ static int block_is_enabled(void* void_plan, block_off_t i) if (block_has_file(block)) one_valid = 1; - if (block_has_invalid_parity(block)) + if (block_has_invalid_parity(block) || plan->force_full) one_invalid = 1; } @@ -733,6 +734,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari countmax = 0; plan.handle_max = diskmax; plan.handle_map = handle; + plan.force_full = state->opt.force_full; for (blockcur = blockstart; blockcur < blockmax; ++blockcur) { if (!block_is_enabled(&plan, blockcur)) continue; @@ -805,7 +807,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* Note that CHG/DELETED blocks already present in the content file loaded */ /* have the hash cleared (::clear_past_hash flag), and then they won't never match the hash. */ /* We are treating only CHG blocks created at runtime. */ - parity_needs_to_be_updated = state->opt.force_parity_update; + parity_needs_to_be_updated = state->opt.force_full || state->opt.force_parity_update; /* if the parity is going to be updated */ parity_going_to_be_updated = 0; @@ -1468,8 +1470,8 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t file_paritymax = parityblocks; } - /* if we do a full sync, having a wrong parity size is expected */ - if (!state->opt.force_full) { + /* if we do a full parity realloc or computation, having a wrong parity size is expected */ + if (!state->opt.force_realloc && !state->opt.force_full) { /* if the parities are too small */ if (file_paritymax < used_paritymax) { /* LCOV_EXCL_START */ diff --git a/snapraid.1 b/snapraid.1 index 0bf34f3f..5d6178ba 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -37,6 +37,10 @@ snapraid [\-c, \-\-conf CONFIG] [\-N, \-\-force\-nocopy] [\-F, \-\-force\-full] .PD 0 .PP +.PD + [\-R, \-\-force\-realloc] +.PD 0 +.PP .PD [\-S, \-\-start BLKSTART] [\-B, \-\-count BLKCOUNT] .PD 0 @@ -1143,12 +1147,21 @@ This option can be used only with \[dq]sync\[dq], \[dq]check\[dq] and \[dq]fix\[ .TP .B \-F, \-\-force\-full In \[dq]sync\[dq] forces a full rebuild of the parity. -This option can be used when you reverted back to an old content -file, but using a more recent parity data. +This option can be used when you add a new parity level, or if +you reverted back to an old content file using a more recent parity data. Instead of recomputing the parity from scratch, this allows to reuse the hashes present in the content file to validate data, and to maintain data protection during the \[dq]sync\[dq] process using -the old content file and the parity data you have. +the parity data you have. +This option can be used only with \[dq]sync\[dq]. +.TP +.B \-R, \-\-force\-realloc +In \[dq]sync\[dq] forces a full reallocation of files and rebuild of the parity. +This option can be used to completely reallocate all the files +removing the fragmentation, but reusing the hashes present in the content +file to validate data. +Compared to \-F, \-\-force\-full, this option reallocates all the parity +not having data protection during the operation. This option can be used only with \[dq]sync\[dq]. .TP .B \-l, \-\-log FILE diff --git a/snapraid.d b/snapraid.d index 5c16fe8f..82c78620 100644 --- a/snapraid.d +++ b/snapraid.d @@ -11,6 +11,7 @@ Synopsis : [-Z, --force-zero] [-E, --force-empty] : [-U, --force-uuid] [-D, --force-device] : [-N, --force-nocopy] [-F, --force-full] + : [-R, --force-realloc] : [-S, --start BLKSTART] [-B, --count BLKCOUNT] : [-L, --error-limit NUMBER] : [-v, --verbose] [-q, --quiet] @@ -844,12 +845,21 @@ Options -F, --force-full In "sync" forces a full rebuild of the parity. - This option can be used when you reverted back to an old content - file, but using a more recent parity data. + This option can be used when you add a new parity level, or if + you reverted back to an old content file using a more recent parity data. Instead of recomputing the parity from scratch, this allows to reuse the hashes present in the content file to validate data, and to maintain data protection during the "sync" process using - the old content file and the parity data you have. + the parity data you have. + This option can be used only with "sync". + + -R, --force-realloc + In "sync" forces a full reallocation of files and rebuild of the parity. + This option can be used to completely reallocate all the files + removing the fragmentation, but reusing the hashes present in the content + file to validate data. + Compared to -F, --force-full, this option reallocates all the parity + not having data protection during the operation. This option can be used only with "sync". -l, --log FILE diff --git a/snapraid.txt b/snapraid.txt index fe29b41c..86067808 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -15,6 +15,7 @@ snapraid [-c, --conf CONFIG] [-Z, --force-zero] [-E, --force-empty] [-U, --force-uuid] [-D, --force-device] [-N, --force-nocopy] [-F, --force-full] + [-R, --force-realloc] [-S, --start BLKSTART] [-B, --count BLKCOUNT] [-L, --error-limit NUMBER] [-v, --verbose] [-q, --quiet] @@ -905,12 +906,21 @@ SnapRAID provides the following options: -F, --force-full In "sync" forces a full rebuild of the parity. - This option can be used when you reverted back to an old content - file, but using a more recent parity data. + This option can be used when you add a new parity level, or if + you reverted back to an old content file using a more recent parity data. Instead of recomputing the parity from scratch, this allows to reuse the hashes present in the content file to validate data, and to maintain data protection during the "sync" process using - the old content file and the parity data you have. + the parity data you have. + This option can be used only with "sync". + + -R, --force-realloc + In "sync" forces a full reallocation of files and rebuild of the parity. + This option can be used to completely reallocate all the files + removing the fragmentation, but reusing the hashes present in the content + file to validate data. + Compared to -F, --force-full, this option reallocates all the parity + not having data protection during the operation. This option can be used only with "sync". -l, --log FILE