From 6da21cdbd9dc7b5004a53249ac8af5b6e1d7e934 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 27 Feb 2013 21:07:16 +0100 Subject: [PATCH] Adds the -i, --import options to import deleted files for fix. --- HISTORY | 9 ++- TODO | 6 -- check.c | 163 +++++++++++++++++++++++++++++++-------------------- elem.h | 2 +- scan.c | 2 +- snapraid.1 | 17 +++++- snapraid.c | 25 +++++++- snapraid.d | 17 +++++- snapraid.txt | 17 +++++- state.c | 5 ++ state.h | 2 + 11 files changed, 183 insertions(+), 82 deletions(-) diff --git a/HISTORY b/HISTORY index 8a158f0d..b657e656 100644 --- a/HISTORY +++ b/HISTORY @@ -5,11 +5,14 @@ SnapRAID HISTORY ========== * Added pooling support with the new 'pool' command. It creates a virtual view of the array using symbolic links pointing at the original files. - * Added a new -m, -.filter-missing options that allow to undelete files, + * Added a new -m, --filter-missing option that allow to undelete files, without checking/fixing the others. + * Added a new -i, --import option to automatically import deleted files + when fixing. * Added support also for hardlinks and empty directories. - * Added symlinks support to Windows. Note that Windows junctions are still - not supported. + * Added support to save symlinks in Windows. Note that only the symlink + is saved and not the linked file or directory. + Junctions are still not supported. * Files without read permission generates now an error instead of a warning. You now must explicitely exclude them in the configuration file with exclusion rules. diff --git a/TODO b/TODO index 75d6a170..7096998c 100644 --- a/TODO +++ b/TODO @@ -3,12 +3,6 @@ SnapRAID TODO This is the list of TODO items for SnapRAID. -- v3.0 - -* We can extend the fix command to read know deleted files from a specific directory, -even outside the array, compute the hash of all the blocks, and use it as a bag of data, -indexed by hash, to find missing data for DELETED and CHG blocks. - - v3.1 * If a content file is missing, sync should rewrite it, even if no change was done at the content. diff --git a/check.c b/check.c index 98778ee8..7b98d5bd 100644 --- a/check.c +++ b/check.c @@ -19,6 +19,7 @@ #include "util.h" #include "elem.h" +#include "import.h" #include "state.h" #include "parity.h" #include "handle.h" @@ -234,35 +235,59 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, int error; unsigned j; int n; - int has_bad; + int something_to_recover; error = 0; - /* here we have to try two different strategies to recover, because in case the 'sync' */ - /* process is aborted, we don't know if the parity data is really updated or */ - /* still represents the state before the 'sync' */ + /* Here we have to try two different strategies to recover, because in case the 'sync' */ + /* process is aborted, we don't know if the parity data is really updated just like after 'sync', */ + /* or if it still represents the state before the 'sync'. */ - /* now we assume that the parity is already computed for the current state */ + /* As first, we assume that the parity is already computed for the current state */ /* and that we are going to recover the state after the last 'sync'. */ - /* we need to put in the recovering process only the bad blocks, because all the */ - /* others already contains the correct data, and the parity is correctly computed for them. */ - /* this is the normal condition. */ + /* This is the normal condition. */ + /* In this case, parity contains correct info for BLK, CHG (new version) and NEW blocks, */ + /* and not for DELETED ones. */ + /* We need to put in the recovering process only the bad blocks, because all the */ + /* others already contains the correct data read frin disk, and the parity is correctly computed for them. */ + /* We are interested to recover BLK, CHG and NEW blocks if they are marked as bad, */ + /* but we are not interested in DELETED ones. */ + n = 0; - has_bad = 0; /* keep track if there is at least one block to fix */ + something_to_recover = 0; /* keep track if there is at least one block to fix */ for(j=0;jhash, buffer[failed[j].index]) == 0) { + /* we have corrected it! */ + } else { + /* otherwise try to recover it */ + failed_map[n] = j; + ++n; + + /* we have something to try to recover */ + something_to_recover = 1; + } + } else { + assert(block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_NEW); + + /* otherwise it's CHG or NEW and we try to recover it */ + failed_map[n] = j; + ++n; + + /* we have something to try to recover */ + something_to_recover = 1; + } } } /* if nothing to fix */ - if (!has_bad) { + if (!something_to_recover) { /* recompute only parity and qarity */ raid_gen(state->level, buffer, diskmax, state->block_size); return 0; @@ -289,9 +314,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, /* it may contain garbage */ failed[j].is_outofdate = 1; } - } - - if (block_state == BLOCK_STATE_CHG) { + } else if (block_state == BLOCK_STATE_CHG) { /* if the hash is a bogus value we cannot check the result */ /* this could happen if we have lost this information */ /* after an aborted sync */ @@ -299,7 +322,7 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, /* it may contain garbage */ failed[j].is_outofdate = 1; } else - /* if the hash is different the previous one, we are sure to have */ + /* if the hash is different than the previous one, we are sure to have */ /* restored it to the state after the 'sync' */ /* instead, if the hash matches, it could be either that the */ /* block after the sync has this hash, or that */ @@ -317,31 +340,51 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, if (ret > 0) error += ret; - /* now assume that the parity computation was not updated at the current state */ + /* Now assume that the parity computation was not updated at the current state, */ /* but still represent the state before the last 'sync' process. */ - /* this may happen only if a 'sync' is aborted by a crash or manually */ + /* This may happen only if a 'sync' is aborted by a crash or manually. */ + /* In this case, parity contains info for BLK, CHG (old version) and DELETED blocks, */ + /* but not for CHG (new version) and NEW ones. */ + /* We are interested to recover BLK ones marked as bad, */ + /* but we are not interested to recover NEW and CHG (new version) blocks, even if marked as bad, */ + /* because we don't have parity for them and it's just impossible, */ + /* and we are not interested to recover DELETED ones. */ n = 0; - has_bad = 0; /* keep track if there is at least one block to fix */ + something_to_recover = 0; /* keep track if there is at least one block to fix */ for(j=0;jhash, buffer[failed[j].index]) == 0) { + /* note that from now the buffer is definitively lost */ + /* we can do this only because it's the last retry of recovering */ + } else { + /* otherwise try to recover it */ + failed_map[n] = j; + ++n; + + /* note that we don't set something_to_recover, because we are */ + /* not really interested to recover *only* old blocks. */ + } + + /* mark that we have restored or we will restore an old state */ + /* note that if the block is not marked as is_bad, */ + /* we are not going to write it in the disk */ + failed[j].is_outofdate = 1; + } else if (block_state == BLOCK_STATE_NEW) { + /* If the block is NEW, restore it to the original 0 as before the 'sync' */ + /* Also in this case, we do this to just allow recovering of other BLK ones */ - /* if the block is new, restore it to the original 0 as before the 'sync' */ - if (block_state == BLOCK_STATE_NEW) { memset(buffer[failed[j].index], 0, state->block_size); /* note that from now the buffer is definitively lost */ /* we can do this only because it's the last retry of recovering */ @@ -350,36 +393,31 @@ static int repair(struct snapraid_state* state, unsigned pos, unsigned diskmax, /* note that if the block is not marked as is_bad, */ /* we are not going to write it in the disk */ failed[j].is_outofdate = 1; + } else if (failed[j].is_bad) { + /* If the block is bad we don't know its content, and we try to recover it */ + /* At this point, we can have only BLK ones */ + + assert(block_state == BLOCK_STATE_BLK); + + /* we have something we are interested to recover */ + something_to_recover = 1; + + /* we try to recover it */ + failed_map[n] = j; + ++n; } } - /* if nothing to fix */ - if (!has_bad) { - /* recompute only parity and qarity */ - raid_gen(state->level, buffer, diskmax, state->block_size); - return 0; - } - - ret = repair_step(state, pos, diskmax, failed, failed_map, n, buffer, buffer_parity, buffer_qarity, buffer_zero); - if (ret == 0) { - /* reprocess the blocks for NEW and CHG ones, for which we don't have a hash to check */ - - /* we know for sure that, even if we have restored them, that we have restored an old state */ - /* so, we mark all of them as outofdate */ - for(j=0;j 0) + error += ret; } - if (ret > 0) - error += ret; /* return the number of failed attempts, or -1 if no strategy */ if (error) @@ -641,7 +679,6 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, if (block_state == BLOCK_STATE_NEW || block_state == BLOCK_STATE_CHG) { /* store it in the failed set, because potentially */ /* the parity may be still computed with the previous content */ - /* note that these blocks are never recovered, because we cannot check them */ failed[failed_count].is_bad = 0; failed[failed_count].is_outofdate = 0; failed[failed_count].index = j; diff --git a/elem.h b/elem.h index 6e73cae5..d3eefbf9 100644 --- a/elem.h +++ b/elem.h @@ -435,7 +435,7 @@ struct snapraid_deleted* deleted_alloc(void); struct snapraid_deleted* deleted_dup(struct snapraid_block* block); /** - * Frees a deleted block. + * Frees a deleted block. */ void deleted_free(struct snapraid_deleted* deleted); diff --git a/scan.c b/scan.c index 816491c4..fa32fe43 100644 --- a/scan.c +++ b/scan.c @@ -573,7 +573,7 @@ static int scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, in fprintf(stderr, "You can exclude it in the config file with:\n\texclude /%s\n", sub); exit(EXIT_FAILURE); } - if (dd == 0 && errno == 0) { + if (dd == 0) { break; /* finished */ } diff --git a/snapraid.1 b/snapraid.1 index afe4b240..06e0f518 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -10,7 +10,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] .PD 0 .PP .PD - [\(hya, \(hy\(hyaudit\(hyonly] + [\(hya, \(hy\(hyaudit\(hyonly] [\(hyi, \(hy\(hyimport DIR] .PD 0 .PP .PD @@ -388,6 +388,7 @@ Nothing is modified. Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved in the previous \(a"sync\(a" command. +.PP If you use the \(hya, \(hy\(hyaudit\(hyonly option, only the file data is checked, and the redundandy data is ignored. .PP @@ -442,7 +443,7 @@ This option can be used only with the \(a"check\(a" and \(a"fix\(a" commands. Note that it cannot be used with \(a"sync\(a", because \(a"sync\(a" always process the whole array. .TP -.B \(hyn, \(hy\(hyfileter\(hymissing +.B \(hyn, \(hy\(hyfilter\(hymissing Filters the files to process in the \(a"check\(a" and \(a"fix\(a" commands. Only the files missing/deleted from the array are processed. @@ -456,8 +457,20 @@ process the whole array. .B \(hya, \(hy\(hyaudit\(hyonly When checking, only verify the hash of the files, without doing any kind of check on the redundancy data. +If you are interested in checking only the file data this +option can speedup a lot the checking process. This option can be used only with the \(a"check\(a" command. .TP +.B \(hyi, \(hy\(hyimport DIR +When fixing imports from the specified directory any file +that you deleted from the array after the last \(a"sync\(a" +commmand. +If you still have such files, they could be used by the \(a"fix\(a" +command to improve the recover process. +The files are read also in subdirectories and they are +identified regardless of their name. +This option can be used only with the \(a"check\(a" and \(a"fix\(a" command. +.TP .B \(hyN, \(hy\(hyfind\(hyby\(hyname When syncing finds the files by path instead than by inode, and ignores the nanosecond part of timestamp. diff --git a/snapraid.c b/snapraid.c index 54b023c2..221bc841 100644 --- a/snapraid.c +++ b/snapraid.c @@ -20,9 +20,9 @@ #include "snapraid.h" #include "util.h" #include "elem.h" +#include "import.h" #include "state.h" #include "raid.h" -#include "os.h" /****************************************************************************/ /* main */ @@ -60,6 +60,7 @@ void usage(void) printf(" " SWITCH_GETOPT_LONG("-f, --filter PATTERN ", "-f") " Process only files matching the pattern\n"); printf(" " SWITCH_GETOPT_LONG("-d, --filter-dist NAME", "-f") " Process only files in the disk\n"); printf(" " SWITCH_GETOPT_LONG("-m, --filter-missing ", "-m") " Process only missing/delete files\n"); + printf(" " SWITCH_GETOPT_LONG("-i, --import DIR ", "-i") " Import deleted files\n"); printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-A") " Check only file data and not parity\n"); printf(" " SWITCH_GETOPT_LONG("-N, --find-by-name ", "-N") " Find the files by name instead than by inode\n"); printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n"); @@ -86,6 +87,7 @@ struct option long_options[] = { { "filter-missing", 0, 0, 'm' }, { "start", 1, 0, 's' }, { "count", 1, 0, 't' }, + { "import", 1, 0, 'i' }, { "force-zero", 0, 0, 'Z' }, { "force-empty", 0, 0, 'E' }, { "force-second", 0, 0, 'S' }, @@ -106,7 +108,7 @@ struct option long_options[] = { }; #endif -#define OPTIONS "c:f:d:ms:t:ZESNaTvhVG" +#define OPTIONS "c:f:d:ms:t:i:ZESNaTvhVG" volatile int global_interrupt = 0; @@ -152,6 +154,7 @@ int main(int argc, char* argv[]) int filter_missing; char* e; const char* command; + char* import; os_init(); @@ -173,6 +176,7 @@ int main(int argc, char* argv[]) tommy_list_init(&filterlist_file); tommy_list_init(&filterlist_disk); filter_missing = 0; + import = 0; opterr = 0; while ((c = @@ -219,6 +223,13 @@ int main(int argc, char* argv[]) exit(EXIT_FAILURE); } break; + case 'i' : + if (import) { + fprintf(stderr, "Import directory '%s' already specified as '%s'\n", optarg, import); + exit(EXIT_FAILURE); + } + import = strdup_nofail(optarg); + break; case 'Z' : force_zero = 1; break; @@ -324,6 +335,10 @@ int main(int argc, char* argv[]) fprintf(stderr, "You cannot filter with the '%s' command\n", command); exit(EXIT_FAILURE); } + if (import != 0) { + fprintf(stderr, "You cannot import with the '%s' command\n", command); + exit(EXIT_FAILURE); + } } raid_init(); @@ -335,6 +350,11 @@ int main(int argc, char* argv[]) state_config(&state, conf, verbose, gui, force_zero, force_empty, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_device); + if (import != 0) { + printf("%s\n", import); + state_import(&state, import); + } + if (operation == OPERATION_DIFF) { state_read(&state); @@ -420,6 +440,7 @@ int main(int argc, char* argv[]) state_done(&state); tommy_list_foreach(&filterlist_file, (tommy_foreach_func*)filter_free); tommy_list_foreach(&filterlist_disk, (tommy_foreach_func*)filter_free); + free(import); os_done(); diff --git a/snapraid.d b/snapraid.d index 4b2233f3..bd8386c6 100644 --- a/snapraid.d +++ b/snapraid.d @@ -4,7 +4,7 @@ Name{number} Synopsis :snapraid [-c, --conf CONFIG] : [-f, --filter PATTERN] [-d, --filter-disk NAME] [-m, --filter-missing] - : [-a, --audit-only] + : [-a, --audit-only] [-i, --import DIR] : [-N, --find-by-name] : [-Z, --force-zero] [-E, --force-empty] : [-s, --start BLKSTART] [-t, --count BLKCOUNT] @@ -243,6 +243,7 @@ Commands Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved in the previous "sync" command. + If you use the -a, --audit-only option, only the file data is checked, and the redundandy data is ignored. @@ -299,7 +300,7 @@ Options Note that it cannot be used with "sync", because "sync" always process the whole array. - -n, --fileter-missing + -n, --filter-missing Filters the files to process in the "check" and "fix" commands. Only the files missing/deleted from the array are processed. @@ -313,8 +314,20 @@ Options -a, --audit-only When checking, only verify the hash of the files, without doing any kind of check on the redundancy data. + If you are interested in checking only the file data this + option can speedup a lot the checking process. This option can be used only with the "check" command. + -i, --import DIR + When fixing imports from the specified directory any file + that you deleted from the array after the last "sync" + commmand. + If you still have such files, they could be used by the "fix" + command to improve the recover process. + The files are read also in subdirectories and they are + identified regardless of their name. + This option can be used only with the "check" and "fix" command. + -N, --find-by-name When syncing finds the files by path instead than by inode, and ignores the nanosecond part of timestamp. diff --git a/snapraid.txt b/snapraid.txt index 19cf2ca2..87c13d89 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -8,7 +8,7 @@ snapraid [-c, --conf CONFIG] [-f, --filter PATTERN] [-d, --filter-disk NAME] [-m, --filter-missing] - [-a, --audit-only] + [-a, --audit-only] [-i, --import DIR] [-N, --find-by-name] [-Z, --force-zero] [-E, --force-empty] [-s, --start BLKSTART] [-t, --count BLKCOUNT] @@ -273,6 +273,7 @@ Nothing is modified. Checks all the files and the redundancy data. All the files are hashed and compared with the snapshot saved in the previous "sync" command. + If you use the -a, --audit-only option, only the file data is checked, and the redundandy data is ignored. @@ -334,7 +335,7 @@ SnapRAID provides the following options: Note that it cannot be used with "sync", because "sync" always process the whole array. - -n, --fileter-missing + -n, --filter-missing Filters the files to process in the "check" and "fix" commands. Only the files missing/deleted from the array are processed. @@ -348,8 +349,20 @@ SnapRAID provides the following options: -a, --audit-only When checking, only verify the hash of the files, without doing any kind of check on the redundancy data. + If you are interested in checking only the file data this + option can speedup a lot the checking process. This option can be used only with the "check" command. + -i, --import DIR + When fixing imports from the specified directory any file + that you deleted from the array after the last "sync" + commmand. + If you still have such files, they could be used by the "fix" + command to improve the recover process. + The files are read also in subdirectories and they are + identified regardless of their name. + This option can be used only with the "check" and "fix" command. + -N, --find-by-name When syncing finds the files by path instead than by inode, and ignores the nanosecond part of timestamp. diff --git a/state.c b/state.c index ab6274e6..78ad7fab 100644 --- a/state.c +++ b/state.c @@ -18,6 +18,7 @@ #include "portable.h" #include "elem.h" +#include "import.h" #include "state.h" #include "util.h" @@ -46,6 +47,8 @@ void state_init(struct snapraid_state* state) tommy_list_init(&state->maplist); tommy_list_init(&state->contentlist); tommy_list_init(&state->filterlist); + tommy_list_init(&state->importlist); + tommy_hashdyn_init(&state->importset); state->loaded_blockmax = 0; } @@ -55,6 +58,8 @@ void state_done(struct snapraid_state* state) tommy_list_foreach(&state->maplist, (tommy_foreach_func*)map_free); tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free); tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free); + tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free); + tommy_hashdyn_done(&state->importset); } /** diff --git a/state.h b/state.h index d919f3a0..18d5cea8 100644 --- a/state.h +++ b/state.h @@ -50,6 +50,8 @@ struct snapraid_state { tommy_list disklist; /**< List of all the disks. */ tommy_list maplist; /**< List of all the disk mappings. */ tommy_list filterlist; /**< List of inclusion/exclusion. */ + tommy_list importlist; /**< List of import file. */ + tommy_hashdyn importset; /**< Hashtable by hash of all the import blocks. */ block_off_t loaded_blockmax; /**< Previous size of the parity file, computed from the loaded state. */ time_t progress_start; /**< Start of processing for progress visualization. */ time_t progress_last; /**< Last update of progress visualization. */