From acad6aad0479bed764cfe9a73944b766167a07f8 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 24 Feb 2013 11:33:09 +0100 Subject: [PATCH] Adds the new -m, --filter-missing options to undelete files. --- HISTORY | 9 +++++++-- Makefile.am | 8 ++++---- elem.c | 5 +++++ elem.h | 18 ++++++++++++------ snapraid.1 | 33 +++++++++++++++++++++++++++------ snapraid.c | 22 +++++++++++++++------- snapraid.d | 28 ++++++++++++++++++++++------ state.c | 15 ++++++++++----- state.h | 2 +- 9 files changed, 103 insertions(+), 37 deletions(-) diff --git a/HISTORY b/HISTORY index a21fcb15..2d1c6818 100644 --- a/HISTORY +++ b/HISTORY @@ -3,11 +3,16 @@ SnapRAID HISTORY 2.2 2013/2 ========== - * Added pooling support with the new 'pool' command. It creates a virtual view - of the array using symbolic links pointing at the original files. + * Added pooling support with the new 'pool' command. It creates a virtual + view of the array using symbolic links pointing at the original files. + * Added a new -m, -.filter-missing options that allow to undelete files, + without checking/fixing the others. * Added support also for hardlinks and empty directories. * Added symlinks support to Windows. Note that Windows junctions are still not supported. + * Files without read permission generates now an error instead of a warning. + You now must explicitely exclude them in the configuration file with + exclusion rules. 2.1 2013/1 ========== diff --git a/Makefile.am b/Makefile.am index 416d71ff..fe4b8690 100644 --- a/Makefile.am +++ b/Makefile.am @@ -273,16 +273,16 @@ endif mv test/_a test/disk1/_a mv test/_b test/disk1/_b $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync - echo --- Delete some files, fix and check with raid5 + echo --- Delete some files, fix with -m and check with raid5 rm test/disk1/8* $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf -m fix 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check - echo --- Delete some dirs in a disk, fix and check with raid5 + echo --- Delete some dirs in a disk, fix with -m and check with raid5 rm -r test/disk2/_a rm -r test/disk2/_b $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --test-expect-recoverable -c $(srcdir)/test-raid5.conf check 2>> test.log - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf fix 2>> test.log + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf -m fix 2>> test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check echo --- Delete some dirs in two disks, fix and check with raid6 using the -d option for each disk rm -r test/disk2/_a diff --git a/elem.c b/elem.c index 4ff35576..fe68d609 100644 --- a/elem.c +++ b/elem.c @@ -230,6 +230,11 @@ int filter_dir(tommy_list* filterlist, const char* disk, const char* sub) return filter_element(filterlist, disk, sub, 1); } +int filter_existence(int filter_missing, unsigned flag) +{ + return filter_missing && (flag & FILE_IS_PRESENT) == FILE_IS_PRESENT; +} + int filter_content(tommy_list* contentlist, const char* path) { tommy_node* i; diff --git a/elem.h b/elem.h index 7c4b54a6..c3e21398 100644 --- a/elem.h +++ b/elem.h @@ -133,7 +133,7 @@ struct snapraid_block { #define FILE_IS_PRESENT 1 /**< If it's seen as present. */ #define FILE_IS_EXCLUDED 2 /**< If it's an excluded file from the processing. */ -#define FILE_IS_LARGER 4 /**< If a larger file was already detected. */ +#define FILE_IS_LARGER 4 /**< If a larger file was already detected. Just to avoid to report it more times. */ #define FILE_IS_DAMAGED 8 /**< If a fix was attempted but it failed. */ #define FILE_IS_FIXED 16 /**< If a fix was done. */ #define FILE_IS_HARDLINK 32 /**< If it's an hardlink. */ @@ -265,6 +265,12 @@ static inline int filter_hidden(int enable, struct dirent* dd) */ int filter_path(tommy_list* filterlist, const char* disk, const char* sub); +/** + * Filter a file/link/dir if missing. + * Return !=0 if it matches and it should be excluded. + */ +int filter_existence(int filter_missing, unsigned flag); + /** * Filters a dir using a list of filters. * For each element of the path all the filters are applied, until the first one that matches. @@ -294,7 +300,7 @@ static inline void block_file_set(struct snapraid_block* block, struct snapraid_ block->file_mixed = (block->file_mixed & ~(uintptr_t)BLOCK_STATE_MASK) | (uintptr_t)file; } -static inline unsigned block_state_get(struct snapraid_block* block) +static inline unsigned block_state_get(const struct snapraid_block* block) { return block->file_mixed & BLOCK_STATE_MASK; } @@ -314,7 +320,7 @@ static inline void block_clear_parity(struct snapraid_block* block) } } -static inline int block_has_hash(struct snapraid_block* block) +static inline int block_has_hash(const struct snapraid_block* block) { unsigned state = block_state_get(block); return state == BLOCK_STATE_BLK || state == BLOCK_STATE_INV; @@ -344,7 +350,7 @@ static inline int block_is_valid(struct snapraid_block* block) */ unsigned block_file_size(struct snapraid_block* block, unsigned block_size); -static inline int file_flag_has(struct snapraid_file* file, unsigned mask) +static inline int file_flag_has(const struct snapraid_file* file, unsigned mask) { return (file->flag & mask) == mask; } @@ -400,7 +406,7 @@ static inline tommy_uint32_t file_path_hash(const char* sub) return tommy_hash_u32(0, sub, strlen(sub)); } -static inline int link_flag_has(struct snapraid_link* link, unsigned mask) +static inline int link_flag_has(const struct snapraid_link* link, unsigned mask) { return (link->flag & mask) == mask; } @@ -446,7 +452,7 @@ static inline tommy_uint32_t link_name_hash(const char* name) return tommy_hash_u32(0, name, strlen(name)); } -static inline int dir_flag_has(struct snapraid_dir* dir, unsigned mask) +static inline int dir_flag_has(const struct snapraid_dir* dir, unsigned mask) { return (dir->flag & mask) == mask; } diff --git a/snapraid.1 b/snapraid.1 index ae67a457..f47688ab 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -6,7 +6,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] .PD 0 .PP .PD - [\(hyf, \(hy\(hyfilter PATTERN] [\(hyd, \(hy\(hyfilter\(hydisk NAME] + [\(hyf, \(hy\(hyfilter PATTERN] [\(hyd, \(hy\(hyfilter\(hydisk NAME] [\(hym, \(hy\(hyfilter\(hymissing] .PD 0 .PP .PD @@ -307,10 +307,11 @@ in \(a"sync\(a". .PP In this regard SnapRAID is more like a backup program than a RAID system. For example, you can use it to recover from an accidentally -deleted directory, simply running the fix command like. +deleted directory, simply running the fix command using the +\(hym, \(hy\(hyfilter\(hymissing filter, like: .PP .RS 4 -snapraid fix \(hyf DIR/ +snapraid fix \(hym \(hyf DIR/ .PD 0 .PP .PD @@ -319,7 +320,16 @@ snapraid fix \(hyf DIR/ Or to simply recover one file you can use: .PP .RS 4 -snapraid fix \(hyf FILE +snapraid fix \(hym \(hyf FILE +.PD 0 +.PP +.PD +.RE +.PP +Or to recover all the delete files with: +.PP +.RS 4 +snapraid fix \(hym .PD 0 .PP .PD @@ -425,13 +435,24 @@ You must specify a disk name as named in the configuration file. In \(a"check\(a", you can make it faster, specifing also \(hya, \(hy\(hyaudit\(hyonly option, to avoid to access other disks to check parity data. -If you combine \(hy\(hyfilter and \(hy\(hyfilter\(hydisk options, only files -matching both the two groups of rules are selected. +If you combine more \(hy\(hyfilter, \(hy\(hyfilter\(hydisk and \(hy\(hyfilter\(hymissing options, +only files matching all the set of filters are selected. This option can be used many times. This option can be used only with the \(a"check\(a" and \(a"fix\(a" commands. Note that it cannot be used with \(a"sync\(a", because \(a"sync\(a" always process the whole array. .TP +.B \(hyn, \(hy\(hyfileter\(hymissing +Filters the files to process in the \(a"check\(a" and \(a"fix\(a" +commands. +Only the files missing/deleted from the array are processed. +When used with \(a"fix\(a", this is a kind of \(a"undelete\(a" command. +If you combine more \(hy\(hyfilter, \(hy\(hyfilter\(hydisk and \(hy\(hyfilter\(hymissing options, +only files matching all the set of filters are selected. +This option can be used only with the \(a"check\(a" and \(a"fix\(a" commands. +Note that it cannot be used with \(a"sync\(a", because \(a"sync\(a" always +process the whole array. +.TP .B \(hya, \(hy\(hyaudit\(hyonly When checking, only verify the hash of the files, without doing any kind of check on the redundancy data. diff --git a/snapraid.c b/snapraid.c index 3ccc9223..54b023c2 100644 --- a/snapraid.c +++ b/snapraid.c @@ -45,10 +45,11 @@ void usage(void) { version(); - printf("Usage: " PACKAGE " sync|diff|check|fix [options]\n"); + printf("Usage: " PACKAGE " sync|pool|diff|dup|check|fix [options]\n"); printf("\n"); printf("Commands:\n"); printf(" sync Syncronize the state of the array of disks\n"); + printf(" pool Create or update the virtual view of the array of disks\n"); printf(" diff Show the changes that needs to be syncronized\n"); printf(" dup Find duplicate files\n"); printf(" check Check the array of disks\n"); @@ -56,8 +57,9 @@ void usage(void) printf("\n"); printf("Options:\n"); printf(" " SWITCH_GETOPT_LONG("-c, --conf FILE ", "-c") " Configuration file (default " CONF ")\n"); - printf(" " SWITCH_GETOPT_LONG("-f, --filter PATTERN ", "-f") " Filter the files to process with a pattern\n"); - printf(" " SWITCH_GETOPT_LONG("-d, --filter-dist NAME", "-f") " Filter the files to process with a disk name\n"); + printf(" " SWITCH_GETOPT_LONG("-f, --filter PATTERN ", "-f") " Process only files matching the pattern\n"); + printf(" " SWITCH_GETOPT_LONG("-d, --filter-dist NAME", "-f") " Process only files in the disk\n"); + printf(" " SWITCH_GETOPT_LONG("-m, --filter-missing ", "-m") " Process only missing/delete files\n"); printf(" " SWITCH_GETOPT_LONG("-a, --audit-only ", "-A") " Check only file data and not parity\n"); printf(" " SWITCH_GETOPT_LONG("-N, --find-by-name ", "-N") " Find the files by name instead than by inode\n"); printf(" " SWITCH_GETOPT_LONG("-Z, --force-zero ", "-Z") " Force synching of files that get zero size\n"); @@ -81,6 +83,7 @@ struct option long_options[] = { { "conf", 1, 0, 'c' }, { "filter", 1, 0, 'f' }, { "filter-disk", 1, 0, 'd' }, + { "filter-missing", 0, 0, 'm' }, { "start", 1, 0, 's' }, { "count", 1, 0, 't' }, { "force-zero", 0, 0, 'Z' }, @@ -103,7 +106,7 @@ struct option long_options[] = { }; #endif -#define OPTIONS "c:f:d:s:t:ZESNaTvhVG" +#define OPTIONS "c:f:d:ms:t:ZESNaTvhVG" volatile int global_interrupt = 0; @@ -146,6 +149,7 @@ int main(int argc, char* argv[]) int ret; tommy_list filterlist_file; tommy_list filterlist_disk; + int filter_missing; char* e; const char* command; @@ -168,6 +172,7 @@ int main(int argc, char* argv[]) blockcount = 0; tommy_list_init(&filterlist_file); tommy_list_init(&filterlist_disk); + filter_missing = 0; opterr = 0; while ((c = @@ -197,6 +202,9 @@ int main(int argc, char* argv[]) } tommy_list_insert_tail(&filterlist_disk, &filter->node, filter); } break; + case 'm' : + filter_missing = 1; + break; case 's' : blockstart = strtoul(optarg, &e, 0); if (!e || *e) { @@ -312,7 +320,7 @@ int main(int argc, char* argv[]) case OPERATION_DRY : break; default: - if (!tommy_list_empty(&filterlist_file) || !tommy_list_empty(&filterlist_disk)) { + if (!tommy_list_empty(&filterlist_file) || !tommy_list_empty(&filterlist_disk) || filter_missing) { fprintf(stderr, "You cannot filter with the '%s' command\n", command); exit(EXIT_FAILURE); } @@ -379,7 +387,7 @@ int main(int argc, char* argv[]) state_read(&state); /* apply the command line filter */ - state_filter(&state, &filterlist_file, &filterlist_disk); + state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing); /* intercept Ctrl+C */ signal(SIGINT, &signal_handler); @@ -397,7 +405,7 @@ int main(int argc, char* argv[]) state_read(&state); /* apply the command line filter */ - state_filter(&state, &filterlist_file, &filterlist_disk); + state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing); /* intercept Ctrl+C */ signal(SIGINT, &signal_handler); diff --git a/snapraid.d b/snapraid.d index 3ef8f73e..ff82ad05 100644 --- a/snapraid.d +++ b/snapraid.d @@ -3,7 +3,7 @@ Name{number} Synopsis :snapraid [-c, --conf CONFIG] - : [-f, --filter PATTERN] [-d, --filter-disk NAME] + : [-f, --filter PATTERN] [-d, --filter-disk NAME] [-m, --filter-missing] : [-a, --audit-only] : [-N, --find-by-name] : [-Z, --force-zero] [-E, --force-empty] @@ -176,13 +176,18 @@ Getting Started In this regard SnapRAID is more like a backup program than a RAID system. For example, you can use it to recover from an accidentally - deleted directory, simply running the fix command like. + deleted directory, simply running the fix command using the + -m, --filter-missing filter, like: - :snapraid fix -f DIR/ + :snapraid fix -m -f DIR/ Or to simply recover one file you can use: - :snapraid fix -f FILE + :snapraid fix -m -f FILE + + Or to recover all the delete files with: + + :snapraid fix -m Commands SnapRAID provides four simple commands that allow to: @@ -287,13 +292,24 @@ Options file. In "check", you can make it faster, specifing also -a, --audit-only option, to avoid to access other disks to check parity data. - If you combine --filter and --filter-disk options, only files - matching both the two groups of rules are selected. + If you combine more --filter, --filter-disk and --filter-missing options, + only files matching all the set of filters are selected. This option can be used many times. This option can be used only with the "check" and "fix" commands. Note that it cannot be used with "sync", because "sync" always process the whole array. + -n, --fileter-missing + Filters the files to process in the "check" and "fix" + commands. + Only the files missing/deleted from the array are processed. + When used with "fix", this is a kind of "undelete" command. + If you combine more --filter, --filter-disk and --filter-missing options, + only files matching all the set of filters are selected. + This option can be used only with the "check" and "fix" commands. + Note that it cannot be used with "sync", because "sync" always + process the whole array. + -a, --audit-only When checking, only verify the hash of the files, without doing any kind of check on the redundancy data. diff --git a/state.c b/state.c index 308ea490..034b3d21 100644 --- a/state.c +++ b/state.c @@ -1805,12 +1805,12 @@ void state_write(struct snapraid_state* state) state->need_write = 0; /* no write needed anymore */ } -void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk) +void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing) { tommy_node* i; /* if no filter, include all */ - if (tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk)) + if (!filter_missing && tommy_list_empty(filterlist_file) && tommy_list_empty(filterlist_disk)) return; printf("Filtering...\n"); @@ -1831,6 +1831,8 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom printf("/"); printf("\n"); } + if (filter_missing) + printf("\t\n"); } /* for each disk */ @@ -1842,7 +1844,8 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) { struct snapraid_file* file = j->data; - if (filter_path(filterlist_disk, disk->name, file->sub) != 0 + if (filter_existence(filter_missing, file->flag) != 0 + || filter_path(filterlist_disk, disk->name, file->sub) != 0 || filter_path(filterlist_file, disk->name, file->sub) != 0 ) { file_flag_set(file, FILE_IS_EXCLUDED); @@ -1857,7 +1860,8 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom for(j=tommy_list_head(&disk->linklist);j!=0;j=j->next) { struct snapraid_link* link = j->data; - if (filter_path(filterlist_disk, disk->name, link->sub) != 0 + if (filter_existence(filter_missing, link->flag) != 0 + || filter_path(filterlist_disk, disk->name, link->sub) != 0 || filter_path(filterlist_file, disk->name, link->sub) != 0 ) { link_flag_set(link, FILE_IS_EXCLUDED); @@ -1872,7 +1876,8 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tom for(j=tommy_list_head(&disk->dirlist);j!=0;j=j->next) { struct snapraid_dir* dir = j->data; - if (filter_dir(filterlist_disk, disk->name, dir->sub) != 0 + if (filter_existence(filter_missing, dir->flag) != 0 + || filter_dir(filterlist_disk, disk->name, dir->sub) != 0 || filter_dir(filterlist_file, disk->name, dir->sub) != 0 ) { dir_flag_set(dir, FILE_IS_EXCLUDED); diff --git a/state.h b/state.h index 42a37653..d919f3a0 100644 --- a/state.h +++ b/state.h @@ -118,7 +118,7 @@ void state_pool(struct snapraid_state* state); * Filter files, symlinks and dirs. * Apply an additional filter to the list currently loaded. */ -void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk); +void state_filter(struct snapraid_state* state, tommy_list* filterlist_file, tommy_list* filterlist_disk, int filter_missing); /** * Begins the progress visualization.