From f64047c412c850930bacd7c32c1f28ac09f48bc9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 10 Jan 2016 23:00:11 +0100 Subject: [PATCH] Save the content file only after --pre-hash This allow recovering of moved files. --- HISTORY | 3 ++ TODO | 8 ----- cmdline/snapraid.c | 17 ---------- cmdline/sync.c | 81 ++++++++++++++++++++++++++++------------------ 4 files changed, 53 insertions(+), 56 deletions(-) diff --git a/HISTORY b/HISTORY index bbeba5a2..b618d9ee 100644 --- a/HISTORY +++ b/HISTORY @@ -11,6 +11,9 @@ SnapRAID HISTORY You can control the number of blocks to cache with the option --test-io-cache=NUMBER, where the number is between 3 and 128. The default is 32. + * The -h, --pre-hash command, saves the content file only after having + verified all the hashes. This allows recovering of moved files in case + a silent error is found during the hash verification check. * In the configuration file 'data' is now a synonymous of 'disk'. 9.2 2016/01 diff --git a/TODO b/TODO index 9f5b02f2..9866ec45 100644 --- a/TODO +++ b/TODO @@ -5,14 +5,6 @@ This is the list of TODO items for SnapRAID. - Next -* Handle someway moved file to a different disk. -In case of a silent error it should be possible to recover. -Likely an automatic prehash that verifies *only* moved/copied files, -and delay the content file saving only if everything is good. -It can be very similar at prehash, but checking only REP blocks -that already have a hash. -https://sourceforge.net/p/snapraid/discussion/1677233/thread/6615f0e3/ - * smart/up/down working without a configuration file - Minor diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 29674857..ba4d00fd 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -1129,23 +1129,6 @@ int main(int argc, char* argv[]) /* intercept signals while operating */ signal_init(); - /* save the new state before the sync */ - /* this allow to recover the case of the changes in the array after an aborted sync. */ - - /* for example, think at this case: */ - /* - add some files at the array */ - /* - run a sync command, it will recompute the parity adding the new files */ - /* - abort the sync command before it stores the new content file */ - /* - delete the not yet synced files from the array */ - /* - run a new sync command */ - - /* the new sync command has now way to know that the parity file was modified */ - /* because the files triggering these changes are now deleted */ - /* and they aren't listed in the content file */ - - if (state.need_write) - state_write(&state); - /* run a test command if required */ if (run != 0) { ret = system(run); /* ignore error */ diff --git a/cmdline/sync.c b/cmdline/sync.c index bbc59553..37371ac9 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -28,14 +28,6 @@ /****************************************************************************/ /* hash */ -/** - * Sync plan to use. - */ -struct snapraid_plan { - unsigned handle_max; - struct snapraid_handle* handle_map; -}; - static int state_hash_process(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax, int* skip_sync) { struct snapraid_handle* handle; @@ -396,6 +388,14 @@ finish: /****************************************************************************/ /* sync */ +/** + * Sync plan to use. + */ +struct snapraid_plan { + unsigned handle_max; + struct snapraid_handle* handle_map; +}; + /** * A block that failed the hash check, or that was deleted. */ @@ -1455,24 +1455,6 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t } } - /* now change the size of all parities */ - for (l = 0; l < state->level; ++l) { - /* change the size of the parity file, truncating or extending it */ - /* from this point all the DELETED blocks after the end of the parity are invalid */ - /* and they are automatically removed when we save the new content file */ - ret = parity_chsize(&parity_handle[l], size, &out_size, state->opt.skip_fallocate); - if (ret == -1) { - /* LCOV_EXCL_START */ - parity_overflow(state, out_size); - log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l)); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - } - - /* after resizing parity files, refresh again the free info */ - state_refresh(state); - unrecoverable_error = 0; if (state->opt.prehash) { @@ -1482,16 +1464,53 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t if (ret == -1) { /* LCOV_EXCL_START */ ++unrecoverable_error; - /* continue, in case also doing the sync */ + /* continue, in case also doing the sync if ::skip_sync is not set */ /* LCOV_EXCL_STOP */ } - - /* save with the new hash information */ - if (state->need_write) - state_write(state); } if (!skip_sync) { + /** + * Save the new state before the sync but after the hashing phase + * + * This allows to recover after an aborted sync, and at the same time + * it allows to recover broken copied/moved files identified in the + * hashing phase. + * + * For example, think at this case: + * - Add some files at the array + * - Run a sync command, it will recompute the parity adding the new files + * - Abort the sync command before it stores the new content file + * - Delete the not yet synced files from the array + * - Run a new sync command + * + * The sync command has no way to know that the parity file was modified + * because the files triggering these changes are now deleted and they aren't + * listed in the content file. + * Instead, saving the new content file in advance, keeps track of all the parity + * that may be modified. + */ + if (state->need_write) + state_write(state); + + /* now change the size of all parities */ + for (l = 0; l < state->level; ++l) { + /* change the size of the parity file, truncating or extending it */ + /* from this point all the DELETED blocks after the end of the parity are invalid */ + /* and they are automatically removed when we save the new content file */ + ret = parity_chsize(&parity_handle[l], size, &out_size, state->opt.skip_fallocate); + if (ret == -1) { + /* LCOV_EXCL_START */ + parity_overflow(state, out_size); + log_fatal("WARNING! Without an accessible %s file, it isn't possible to sync.\n", lev_name(l)); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + + /* after resizing parity files, refresh again the free info */ + state_refresh(state); + msg_progress("Syncing...\n"); /* skip degenerated cases of empty parity, or skipping all */