From 447f1f1294acf235a456b129f4135a5b45ea110a Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 19 Oct 2014 22:25:30 +0200 Subject: [PATCH] Makes fix to autodetect moved files and use them in recovering. --- HISTORY | 13 +++++++++++-- Makefile.am | 7 +++++++ TODO | 12 ------------ cmdline/elem.h | 29 +++++++++++++++++++++-------- cmdline/search.c | 14 ++++++++++++++ cmdline/search.h | 5 +++++ cmdline/snapraid.c | 37 ++++++++++++++++++++++++++----------- cmdline/state.c | 10 ++++++++-- cmdline/sync.c | 2 +- snapraid.1 | 23 +++++++++++++---------- snapraid.d | 23 +++++++++++++---------- snapraid.txt | 23 +++++++++++++---------- 12 files changed, 132 insertions(+), 66 deletions(-) diff --git a/HISTORY b/HISTORY index 3c31f6a9..8f4c7ce2 100644 --- a/HISTORY +++ b/HISTORY @@ -3,10 +3,19 @@ SnapRAID HISTORY 7.0 2014/10 =========== - * Adds AVX2 support to improve parity generation speed. - * A lot faster directory listing for Windows. + * In 'check' and 'fix' the array is scanned to find any moved files + that could be used to recover missing data. Files are identified by + timestamp, and then they are recognized also if moved to a different + disk. Note that even if there are false positive they are identified + checking the hash, so they have not effect, besides making the + process a little slower. To disable this new behaviour you can use + the -N, --force-nocopy option. + * The -i, --import command now identifies files by timestamp making it + very fast in importing directories. * More detailed 'status' report with single disk stats and free space available. + * A lot faster directory listing for Windows. + * Adds AVX2 support to improve parity generation speed. * Prints the time spent waiting for each disk also in 'scrub'. * The CPU usage, speed and ETA estimations are now based on the last 100 seconds rather than from the start. diff --git a/Makefile.am b/Makefile.am index 72f7d2e4..4893df34 100644 --- a/Makefile.am +++ b/Makefile.am @@ -524,6 +524,13 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -i bench/a -c $(PAR2) fix -l test.log rm -r bench/a $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync + echo --- Delete files from three disks and check/fix with automatic import in PAR2 + rm -r bench/disk1/a + rm -r bench/disk2/a + mv bench/disk3/a bench/disk1/a_from_disk3 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR2) fix -l test.log + rm -r bench/disk1/a_from_disk3 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync echo --- Abort sync late with additions, delete them and recover with PAR2 echo --- This triggers the recovering with q using p to check the validity cp -pR bench/disk1/a bench/disk1/a_copy diff --git a/TODO b/TODO index 25c97af8..31c3c1fa 100644 --- a/TODO +++ b/TODO @@ -104,18 +104,6 @@ File with the same name will get an extra extension like ".1", ".2". * Use threads to scan all the disks at the same time. -* When checking/fixing opens someway renamed files. -We have various report of failed fix for rename operations. -Most of the time the log files became so big, that users cannot even open it, -and then have no way to understand the problem. -This could be fixed opening the file by inode. -This is possible in Windows Vista with OpenFileById, but not in *nix. -But we can anyway scan all files, and search for inode. -Btrfs has BTRFS_IOC_INO_PATHS for root use. -http://stackoverflow.com/questions/4606774/why-cant-files-be-manipulated-by-inode -Besides that, scanning all files and comparing inode and/or nano timestamp -is likely better anyway as it leave more options open. - * Create a filesystem snapshot at every sync, and use it in all the other commands automatically. At the next sync, drop the old snapshot and create a new one. diff --git a/cmdline/elem.h b/cmdline/elem.h index c921e8a7..175377c7 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -92,10 +92,12 @@ struct snapraid_filter { * data used to compute the parity, because the sync process was interrupted at an unknown point, * and the parity may or may not be updated. * - * For this reson we clear all such hashes when reading the state from an incomplete sync when - * starting a new sync, because sync is affected by sych hashes. - * Check and fix are instead able to work with unsynced hashes. - * Scrub is not affected because it ignores CHG/DELETED blocks. + * For this reson we clear all such hashes when reading the state from an incomplete sync before + * starting a new sync, because sync is affected by such hashes, as sync updates the parity, only + * if the new data read for CHG blocks has a mismatching hash. + * Clearing is done setting the ::clear_past_hash flag before reading the state. + * No clearing is done in other commands, as check and fix are instead able to work with unsynced + * hashes, and scrub ignores CHG/DELETED blocks. */ #define BLOCK_STATE_CHG 2 @@ -103,6 +105,15 @@ struct snapraid_filter { * The block is new and hashed. * This happens when a new block overwrite a just removed block, or an empty space. * + * Note that when the file copy heuristic is enabled, the REP blocks may be set + * using this euristic, meaning that the hash may be wrong. + * + * For this reason, when the ::force_nocopy flag is enabled in sync, we convert all the REP blocks + * to CHG, invalidating the stored hash. + * Clearing is done setting the ::clear_past_hash flag before reading the state. + * No cleaering is done in other commands, as they don't stop the process like in sync + * when there is a false silent error. + * * The block hash field IS set, and it represents the hash of the new data. * The parity for this disk is not updated. */ @@ -120,10 +131,12 @@ struct snapraid_filter { * data used to compute the parity, because the sync process was interrupted at an unknown point, * and the parity may or may not be updated. * - * For this reson we clear all such hashes when reading the state from an incomplete sync when - * starting a new sync, because sync is affected by sych hashes. - * Check and fix are instead able to work with unsynced hashes. - * Scrub is not affected because it ignores CHG/DELETED blocks. + * A now the sync process is not affected by DELETED hash, so clearing won't be really needed, + * but considering that we have to do it for CHG blocks, we do it also for DELETED ones, + * clearing all the past hashes. + * Clearing is done setting the ::clear_past_hash flag before reading the state. + * No clearing is done in other commands, as check and fix are instead able to work with unsynced + * hashes, and scrub ignores CHG/DELETED blocks. */ #define BLOCK_STATE_DELETED 4 diff --git a/cmdline/search.c b/cmdline/search.c index 945be6f6..46485863 100644 --- a/cmdline/search.c +++ b/cmdline/search.c @@ -238,3 +238,17 @@ void state_search(struct snapraid_state* state, const char* dir) search_dir(state, dir); } +void state_search_array(struct snapraid_state* state) +{ + tommy_node* i; + + /* import from all the disks */ + for (i = state->disklist; i != 0; i = i->next) { + struct snapraid_disk* disk = i->data; + + printf("Scanning disk %s...\n", disk->name); + + search_dir(state, disk->dir); + } +} + diff --git a/cmdline/search.h b/cmdline/search.h index 285cfb83..6664b625 100644 --- a/cmdline/search.h +++ b/cmdline/search.h @@ -55,5 +55,10 @@ int state_search_fetch(struct snapraid_state* state, int prevhash, struct snapra */ void state_search(struct snapraid_state* state, const char* dir); +/** + * Import files from all the data disks. + */ +void state_search_array(struct snapraid_state* state); + #endif diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 532c759c..038c8bbc 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -722,6 +722,20 @@ int main(int argc, char* argv[]) } } + switch (operation) { + case OPERATION_SYNC : + case OPERATION_CHECK : + case OPERATION_FIX : + break; + default : + if (opt.force_nocopy) { + /* LCOV_EXCL_START */ + fprintf(stderr, "You cannot use -N, --force-nocopy with the '%s' command\n", command); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + } + switch (operation) { case OPERATION_SYNC : break; @@ -733,13 +747,6 @@ int main(int argc, char* argv[]) /* LCOV_EXCL_STOP */ } - if (opt.force_nocopy) { - /* LCOV_EXCL_START */ - fprintf(stderr, "You cannot use -N, --force-nocopy with the '%s' command\n", command); - exit(EXIT_FAILURE); - /* LCOV_EXCL_STOP */ - } - if (opt.force_full) { /* LCOV_EXCL_START */ fprintf(stderr, "You cannot use -G, --force-full with the '%s' command\n", command); @@ -1027,10 +1034,18 @@ int main(int argc, char* argv[]) } else { state_read(&state); - if (import_timestamp != 0) - state_search(&state, import_timestamp); - if (import_content != 0) - state_import(&state, import_content); + /* if we are also trying to recover */ + if (!state.opt.auditonly) { + /* import the user specified dirs */ + if (import_timestamp != 0) + state_search(&state, import_timestamp); + if (import_content != 0) + state_import(&state, import_content); + + /* import from all the array */ + if (!state.opt.force_nocopy) + state_search_array(&state); + } /* apply the command line filter */ state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing, filter_error); diff --git a/cmdline/state.c b/cmdline/state.c index 3d6b5a2b..0a3582ee 100644 --- a/cmdline/state.c +++ b/cmdline/state.c @@ -1356,7 +1356,10 @@ static void state_read_text(struct snapraid_state* state, const char* path, STRE /* if we are disabling the copy optimization */ /* we want also to clear any already previously stored information */ /* in other sync commands */ - if (state->opt.force_nocopy + /* note that this is required only in sync, and we detect */ + /* this using the clear_past_hash flag */ + if (state->clear_past_hash + && state->opt.force_nocopy && block_state_get(block) == BLOCK_STATE_REP ) { /* set the hash value to INVALID */ @@ -2880,7 +2883,10 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST /* if we are disabling the copy optimization */ /* we want also to clear any already previously stored information */ /* in other sync commands */ - if (state->opt.force_nocopy + /* note that this is required only in sync, and we detect */ + /* this using the clear_past_hash flag */ + if (state->clear_past_hash + && state->opt.force_nocopy && block_state_get(block) == BLOCK_STATE_REP ) { /* set the hash value to INVALID */ diff --git a/cmdline/sync.c b/cmdline/sync.c index a67b63d3..4dfd787c 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -479,7 +479,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari /* it could happens that all the blocks are EMPTY/BLK and CHG but with the hash */ /* still matching because the specific CHG block was not modified. */ /* Note that CHG/DELETED blocks already present in the content file loaded */ - /* have the hash cleared, and then they won't never match the hash. */ + /* have the hash cleared (::clear_past_hash flag), and then they won't never match the hash. */ /* We are treating only CHG blocks created at runtime. */ /* In such case, we can avoid to update parity, because it would be the same as before */ parity_needs_to_be_updated = 0; diff --git a/snapraid.1 b/snapraid.1 index a635197d..03ff0639 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -822,16 +822,19 @@ option allows you to continue anyway. This option can be used only with the \[dq]fix\[dq] command. .TP .B \-N, \-\-force\-nocopy -In the \[dq]sync\[dq] command, disables the copy detection heuristic. -Without this option SnapRAID assumes that files with the -same name, size, and nanosecond timestamp are copies with the +In the \[dq]sync\[dq], \[dq]check and \[dq]fix\[dq] commands, disables the copy +detection heuristic. +Without this option SnapRAID assumes that files with same +attributes, like name, size and timestamp are copies with the same data. -This allows to identify copied files from one dir to another, -and to reuse the already computed hash information to detect -silent errors in the copy. +This allows to identify copied or moved files from one disk +to another, and to reuse the already computed hash information +to detect silent errors or to recover missing files. This behavior, in some rare cases, may result in false positives, -and this option allow to resolve them. -This option can be used only with the \[dq]sync\[dq] command. +or in a slow process due the many hash verifications, and this +option allows to resolve them. +This option can be used only with the \[dq]sync\[dq], \[dq]check\[dq] and \[dq]fix\[dq] +commands. .TP .B \-F, \-\-force\-full In the \[dq]sync\[dq] command, forces a full rebuild of the parity. @@ -1363,8 +1366,8 @@ You can get a detailed list of all the unrecoverable blocks in the fix.log file checking all the lines starting with \[dq]unrecoverable:\[dq] .PP If you are not satisfied of the recovering, you can retry it as many time you wish. -For example, if you have moved away some files from other disks after the last \[dq]sync\[dq], -you can retry to put them inplace, and retry the \[dq]fix\[dq]. +For example, if you have remove some files after the last \[dq]sync\[dq], you can retry +to put them inplace, and retry the \[dq]fix\[dq]. .PP If you are satisfied of the recovering, you can now proceed further, but take care that after syncing you will no more able to retry the diff --git a/snapraid.d b/snapraid.d index 176b463f..8aed6957 100644 --- a/snapraid.d +++ b/snapraid.d @@ -585,16 +585,19 @@ Options This option can be used only with the "fix" command. -N, --force-nocopy - In the "sync" command, disables the copy detection heuristic. - Without this option SnapRAID assumes that files with the - same name, size, and nanosecond timestamp are copies with the + In the "sync", "check and "fix" commands, disables the copy + detection heuristic. + Without this option SnapRAID assumes that files with same + attributes, like name, size and timestamp are copies with the same data. - This allows to identify copied files from one dir to another, - and to reuse the already computed hash information to detect - silent errors in the copy. + This allows to identify copied or moved files from one disk + to another, and to reuse the already computed hash information + to detect silent errors or to recover missing files. This behavior, in some rare cases, may result in false positives, - and this option allow to resolve them. - This option can be used only with the "sync" command. + or in a slow process due the many hash verifications, and this + option allows to resolve them. + This option can be used only with the "sync", "check" and "fix" + commands. -F, --force-full In the "sync" command, forces a full rebuild of the parity. @@ -988,8 +991,8 @@ Recovering checking all the lines starting with "unrecoverable:" If you are not satisfied of the recovering, you can retry it as many time you wish. - For example, if you have moved away some files from other disks after the last "sync", - you can retry to put them inplace, and retry the "fix". + For example, if you have remove some files after the last "sync", you can retry + to put them inplace, and retry the "fix". If you are satisfied of the recovering, you can now proceed further, but take care that after syncing you will no more able to retry the diff --git a/snapraid.txt b/snapraid.txt index 5109a019..dc991fd3 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -630,16 +630,19 @@ SnapRAID provides the following options: This option can be used only with the "fix" command. -N, --force-nocopy - In the "sync" command, disables the copy detection heuristic. - Without this option SnapRAID assumes that files with the - same name, size, and nanosecond timestamp are copies with the + In the "sync", "check and "fix" commands, disables the copy + detection heuristic. + Without this option SnapRAID assumes that files with same + attributes, like name, size and timestamp are copies with the same data. - This allows to identify copied files from one dir to another, - and to reuse the already computed hash information to detect - silent errors in the copy. + This allows to identify copied or moved files from one disk + to another, and to reuse the already computed hash information + to detect silent errors or to recover missing files. This behavior, in some rare cases, may result in false positives, - and this option allow to resolve them. - This option can be used only with the "sync" command. + or in a slow process due the many hash verifications, and this + option allows to resolve them. + This option can be used only with the "sync", "check" and "fix" + commands. -F, --force-full In the "sync" command, forces a full rebuild of the parity. @@ -1064,8 +1067,8 @@ You can get a detailed list of all the unrecoverable blocks in the fix.log file checking all the lines starting with "unrecoverable:" If you are not satisfied of the recovering, you can retry it as many time you wish. -For example, if you have moved away some files from other disks after the last "sync", -you can retry to put them inplace, and retry the "fix". +For example, if you have remove some files after the last "sync", you can retry +to put them inplace, and retry the "fix". If you are satisfied of the recovering, you can now proceed further, but take care that after syncing you will no more able to retry the