From cb4a72afca1fa8320ea438a45f1c4e015df05a23 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 26 Apr 2011 18:51:42 +0200 Subject: [PATCH] Adds include logic in filtering. --- HISTORY | 4 +-- check.c | 6 ++-- elem.c | 66 ++++++++++++++++++++++++++++---------- elem.h | 18 +++++------ scan.c | 20 ++---------- snapraid.1 | 89 +++++++++++++++++++++++++++++++++++++++++++++++----- snapraid.c | 2 +- snapraid.d | 50 ++++++++++++++++++++++++----- snapraid.txt | 50 ++++++++++++++++++++++++----- state.c | 33 +++++++++---------- state.h | 2 +- 11 files changed, 248 insertions(+), 92 deletions(-) diff --git a/HISTORY b/HISTORY index 9081c73f..54568a42 100644 --- a/HISTORY +++ b/HISTORY @@ -3,8 +3,8 @@ SnapRAID HISTORY 1.1 2011/5 ========== -* Added RAID6 support. -* Allow to specify multiple content files. +* Added RAID6 support and allow to specify multiple content files. +* Added filtering include logic, where anything not explicitely included is excluded. * Changed the default checksum to Murmur3 hash. It's a lot faster than MD5. MD5 is still supported for backward compatibility. * The check command returns with an error code if any kind of error is present. diff --git a/check.c b/check.c index 30ad463a..29afcc81 100644 --- a/check.c +++ b/check.c @@ -200,7 +200,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity struct snapraid_block* block = disk_block_get(handle[j].disk, i); if (block && bit_has(block->flag, BLOCK_HAS_HASH) /* only if the block is hashed */ - && !block->file->is_filtered /* only if the file is not filtered out */ + && !block->file->is_excluded /* only if the file is not filtered out */ ) { one_tocheck = 1; break; @@ -230,7 +230,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity struct snapraid_block* block = disk_block_get(handle[j].disk, i); if (block && bit_has(block->flag, BLOCK_HAS_HASH) /* only if the block is hashed */ - && !block->file->is_filtered /* only if the file is not filtered out */ + && !block->file->is_excluded /* only if the file is not filtered out */ ) { one_tocheck = 1; break; @@ -415,7 +415,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity /* update the fixed files */ for(j=0;jfile->is_filtered) + if (failed[j].block->file->is_excluded) continue; ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size); diff --git a/elem.c b/elem.c index 5601a396..b3e3f4f1 100644 --- a/elem.c +++ b/elem.c @@ -38,7 +38,7 @@ void content_free(struct snapraid_content* content) free(content); } -struct snapraid_filter* filter_alloc(const char* pattern) +struct snapraid_filter* filter_alloc(int direction, const char* pattern) { struct snapraid_filter* filter; char* i; @@ -47,6 +47,7 @@ struct snapraid_filter* filter_alloc(const char* pattern) filter = malloc_nofail(sizeof(struct snapraid_filter)); pathcpy(filter->pattern, sizeof(filter->pattern), pattern); + filter->direction = direction; /* find first and last slash */ first = 0; @@ -93,35 +94,35 @@ void filter_free(struct snapraid_filter* filter) free(filter); } -int filter_path(struct snapraid_filter* filter, const char* path, const char* name, int is_dir) +static int filter_apply(struct snapraid_filter* filter, const char* path, const char* name, int is_dir) { /* matches dirs with dirs and files with files */ if (filter->is_dir && !is_dir) - return -1; + return 0; if (!filter->is_dir && is_dir) - return -1; + return 0; if (filter->is_path) { /* skip initial slash, as always missing from the path */ if (fnmatch(filter->pattern + 1, path, FNM_PATHNAME) == 0) - return 0; + return filter->direction; } else { if (fnmatch(filter->pattern, name, 0) == 0) - return 0; + return filter->direction; } - return -1; + return 0; } -int filter_file(struct snapraid_filter* filter, struct snapraid_file* file) +static int filter_recurse(struct snapraid_filter* filter, const char* const_path, int is_dir) { char path[PATH_MAX]; char* name; unsigned i; - pathcpy(path, sizeof(path), file->sub); + pathcpy(path, sizeof(path), const_path); - /* filter for all the directory */ + /* filter for all the directories */ name = path; for(i=0;path[i] != 0;++i) { if (path[i] == '/') { @@ -129,8 +130,8 @@ int filter_file(struct snapraid_filter* filter, struct snapraid_file* file) path[i] = 0; /* filter the directory */ - if (filter_path(filter, path, name, 1) == 0) - return 0; + if (filter_apply(filter, path, name, 1) != 0) + return filter->direction; /* restore the slash */ path[i] = '/'; @@ -141,10 +142,43 @@ int filter_file(struct snapraid_filter* filter, struct snapraid_file* file) } /* filter the final file */ - if (filter_path(filter, path, name, 0) == 0) - return 0; + if (filter_apply(filter, path, name, is_dir) != 0) + return filter->direction; - return -1; + return 0; +} + +int filter_path(tommy_list* filterlist, const char* path, int is_dir) +{ + tommy_node* i; + + int direction = 1; /* by default include all */ + + /* for each filter */ + for(i=tommy_list_head(filterlist);i!=0;i=i->next) { + int ret; + struct snapraid_filter* filter = i->data; + + ret = filter_recurse(filter, path, is_dir); + if (ret > 0) { + /* include the file */ + direction = 1; + break; + } else if (ret < 0) { + /* exclude the file */ + direction = -1; + break; + } else { + /* default is opposite of the last filter */ + direction = -filter->direction; + /* continue with the next one */ + } + } + + if (direction < 0) + return -1; + + return 0; } block_off_t block_file_pos(struct snapraid_block* block) @@ -189,7 +223,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t file->mtime = mtime; file->inode = inode; file->is_present = 0; - file->is_filtered = 0; + file->is_excluded = 0; file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block)); /* set the back pointer */ diff --git a/elem.h b/elem.h index 3182d661..327e9b78 100644 --- a/elem.h +++ b/elem.h @@ -59,6 +59,7 @@ struct snapraid_filter { char pattern[PATH_MAX]; /**< Filter pattern. */ int is_path; /**< If the pattern is only for the complete path. */ int is_dir; /**< If the pattern is only for dir. */ + int direction; /**< If it's an inclusion (=1) or an exclusion (=-1). */ tommy_node node; /**< Next node in the list. */ }; @@ -88,8 +89,7 @@ struct snapraid_file { int64_t mtime; /**< Modification time. */ uint64_t inode; /**< Inode. */ int is_present; /**< If it's seen as present. */ - int is_filtered; /**< If it's filtered out. */ - + int is_excluded; /**< If it's an excluded file from the processing. */ /* nodes for data structures */ tommy_node nodelist; tommy_hashdyn_node nodeset; @@ -120,24 +120,20 @@ void content_free(struct snapraid_content* content); /** * Allocates an exclusion. */ -struct snapraid_filter* filter_alloc(const char* pattern); +struct snapraid_filter* filter_alloc(int is_include, const char* pattern); /** * Deallocates an exclusion. */ void filter_free(struct snapraid_filter* filter); -/** - * Filters a path/name. - * Returns 0 if it matches. - */ -int filter_path(struct snapraid_filter* filter, const char* path, const char* name, int is_dir); /** - * Filters a file. - * Returns 0 if it matches. + * Filters a path. + * For each element of the path all the filters are applied, until the first one that matches. + * Returns 0 if the files has to be processed. */ -int filter_file(struct snapraid_filter* filter, struct snapraid_file* file); +int filter_path(tommy_list* filterlist, const char* path, int is_dir); /** * Gets the relative position of a block inside the file. diff --git a/scan.c b/scan.c index c91b3eaf..a9059abb 100644 --- a/scan.c +++ b/scan.c @@ -186,22 +186,6 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, scan_file_insert(state, disk, file); } -/** - * Checks if the specified path/file should be listet. - */ -static int scan_filter(struct snapraid_state* state, const char* path, const char* file, int is_dir) -{ - tommy_node* node = tommy_list_head(&state->excludelist); - while (node) { - struct snapraid_filter* filter = node->data; - if (filter_path(filter, path, file, is_dir) == 0) - return -1; - node = node->next; - } - - return 0; -} - /** * Processes a directory. */ @@ -241,7 +225,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s } if (S_ISREG(st.st_mode)) { - if (scan_filter(state, sub_next, name, 0) == 0) { + if (filter_path(&state->filterlist, sub_next, 0) == 0) { scan_file(scan, state, disk, sub_next, &st); } else { if (state->verbose) { @@ -249,7 +233,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s } } } else if (S_ISDIR(st.st_mode)) { - if (scan_filter(state, sub_next, name, 1) == 0) { + if (filter_path(&state->filterlist, sub_next, 1) == 0) { pathslash(path_next, sizeof(path_next)); pathslash(sub_next, sizeof(sub_next)); scan_dir(scan, state, disk, path_next, sub_next); diff --git a/snapraid.1 b/snapraid.1 index b426517a..f784f8d1 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -390,8 +390,14 @@ you will invalidate the q\(hyparity file. You should use one option for each disk of the array. .TP .B exclude PATTERN -Defines the file or directory patterns to exclude from the sync -process. +.TP +.B include PATTERN +Defines the file or directory patterns to exclude and include +in the sync process. +All the patterns are processed in the specified order, and +we get the result of the first pattern that matches. +If no pattern matches, the file is excluded if the last pattern +is an \(a"include\(a", or included if the last pattern is an \(a"exclude\(a". See the PATTERN section for more details in the pattern specifications. This option can be used many times. @@ -477,7 +483,8 @@ block_size 256 .PD .RE .SH PATTERN -Patterns are used to select a subset of files to exclude or operate on. +Patterns are used to select a subset of files to exclude or include in +the process. .PP There are four different types of patterns: .TP @@ -506,7 +513,11 @@ This pattern is applied only to directories and not to files. Note that when globbing char are used in the command line, you have to quote them in Unix. Otherwise the shell will try to expand them. .PP -For example, in the configuration file: +In the configuration file, you can use different strategies to define +the files to process. +.PP +The most simple one is to only use \(a"exclude\(a" rules to remove all the +files and directories you do not want to process. For example: .PP .RS 4 # Excludes any file named \(a"*.bak\(a" @@ -525,7 +536,7 @@ exclude /lost+found/ .PD 0 .PP .PD -# Excludes any directory named \(a"tmp\(a" +# Excludes any sub\(hydirectory named \(a"tmp\(a" .PD 0 .PP .PD @@ -535,14 +546,78 @@ exclude tmp/ .PD .RE .PP -For example, in the command line: +The opposite way is to define only the file you want to process, using +only \(a"include\(a" rules. For example: +.PP +.RS 4 +# Includes only some directories +.PD 0 +.PP +.PD +include /movies/ +.PD 0 +.PP +.PD +include /musics/ +.PD 0 +.PP +.PD +include /pictures/ +.PD 0 +.PP +.PD +.RE +.PP +The final way, is to mix \(a"exclude\(a" and \(a"include\(a" rules. In this case take +care that the order of rules is important. Previous rules have the +precedence over the later ones. +To get things simpler you can first have all the \(a"exclude\(a" rules and then +all the \(a"include\(a" ones. For example: +.PP +.RS 4 +# Excludes any file named \(a"*.bak\(a" +.PD 0 +.PP +.PD +exclude *.bak +.PD 0 +.PP +.PD +# Excludes any sub\(hydirectory named \(a"tmp\(a" +.PD 0 +.PP +.PD +exclude tmp/ +.PD 0 +.PP +.PD +# Includes only some directories +.PD 0 +.PP +.PD +include /movies/ +.PD 0 +.PP +.PD +include /musics/ +.PD 0 +.PP +.PD +include /pictures/ +.PD 0 +.PP +.PD +.RE +.PP +On the command line, using the \(hyf option, you can only use \(a"include\(a" +patterns. For example: .PP .RS 4 # Checks only the .mp3 files. .PD 0 .PP .PD -# Note the \(a"\(a" use to avoid globbing expansion by the shell. +# Note the \(a"\(a" use to avoid globbing expansion by the shell in Unix. .PD 0 .PP .PD diff --git a/snapraid.c b/snapraid.c index 8aeb7708..205ec947 100644 --- a/snapraid.c +++ b/snapraid.c @@ -134,7 +134,7 @@ int main(int argc, char* argv[]) conf = optarg; break; case 'f' : { - struct snapraid_filter* filter = filter_alloc(optarg); + struct snapraid_filter* filter = filter_alloc(1, optarg); if (!filter) { fprintf(stderr, "Invalid filter specification '%s'\n", optarg); exit(EXIT_FAILURE); diff --git a/snapraid.d b/snapraid.d index 3b2d27af..12189f9c 100644 --- a/snapraid.d +++ b/snapraid.d @@ -285,8 +285,13 @@ Configuration You should use one option for each disk of the array. =exclude PATTERN - Defines the file or directory patterns to exclude from the sync - process. + =include PATTERN + Defines the file or directory patterns to exclude and include + in the sync process. + All the patterns are processed in the specified order, and + we get the result of the first pattern that matches. + If no pattern matches, the file is excluded if the last pattern + is an "include", or included if the last pattern is an "exclude". See the PATTERN section for more details in the pattern specifications. This option can be used many times. @@ -330,7 +335,8 @@ Configuration :block_size 256 Pattern - Patterns are used to select a subset of files to exclude or operate on. + Patterns are used to select a subset of files to exclude or include in + the process. There are four different types of patterns: @@ -359,17 +365,47 @@ Pattern Note that when globbing char are used in the command line, you have to quote them in Unix. Otherwise the shell will try to expand them. - For example, in the configuration file: + In the configuration file, you can use different strategies to define + the files to process. + + The most simple one is to only use "exclude" rules to remove all the + files and directories you do not want to process. For example: + :# Excludes any file named "*.bak" :exclude *.bak :# Excludes the root directory "/lost+found" :exclude /lost+found/ - :# Excludes any directory named "tmp" + :# Excludes any sub-directory named "tmp" :exclude tmp/ - For example, in the command line: + The opposite way is to define only the file you want to process, using + only "include" rules. For example: + + :# Includes only some directories + :include /movies/ + :include /musics/ + :include /pictures/ + + The final way, is to mix "exclude" and "include" rules. In this case take + care that the order of rules is important. Previous rules have the + precedence over the later ones. + To get things simpler you can first have all the "exclude" rules and then + all the "include" ones. For example: + + :# Excludes any file named "*.bak" + :exclude *.bak + :# Excludes any sub-directory named "tmp" + :exclude tmp/ + :# Includes only some directories + :include /movies/ + :include /musics/ + :include /pictures/ + + On the command line, using the -f option, you can only use "include" + patterns. For example: + :# Checks only the .mp3 files. - :# Note the "" use to avoid globbing expansion by the shell. + :# Note the "" use to avoid globbing expansion by the shell in Unix. :snapraid -f "*.mp3" check Content diff --git a/snapraid.txt b/snapraid.txt index 110bcc07..21db6a87 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -307,8 +307,13 @@ it should contains the following options: You should use one option for each disk of the array. exclude PATTERN - Defines the file or directory patterns to exclude from the sync - process. + include PATTERN + Defines the file or directory patterns to exclude and include + in the sync process. + All the patterns are processed in the specified order, and + we get the result of the first pattern that matches. + If no pattern matches, the file is excluded if the last pattern + is an "include", or included if the last pattern is an "exclude". See the PATTERN section for more details in the pattern specifications. This option can be used many times. @@ -355,7 +360,8 @@ An example of a typical configuration is: 8 PATTERN ========= -Patterns are used to select a subset of files to exclude or operate on. +Patterns are used to select a subset of files to exclude or include in +the process. There are four different types of patterns: @@ -384,17 +390,47 @@ There are four different types of patterns: Note that when globbing char are used in the command line, you have to quote them in Unix. Otherwise the shell will try to expand them. -For example, in the configuration file: +In the configuration file, you can use different strategies to define +the files to process. + +The most simple one is to only use "exclude" rules to remove all the +files and directories you do not want to process. For example: + # Excludes any file named "*.bak" exclude *.bak # Excludes the root directory "/lost+found" exclude /lost+found/ - # Excludes any directory named "tmp" + # Excludes any sub-directory named "tmp" exclude tmp/ -For example, in the command line: +The opposite way is to define only the file you want to process, using +only "include" rules. For example: + + # Includes only some directories + include /movies/ + include /musics/ + include /pictures/ + +The final way, is to mix "exclude" and "include" rules. In this case take +care that the order of rules is important. Previous rules have the +precedence over the later ones. +To get things simpler you can first have all the "exclude" rules and then +all the "include" ones. For example: + + # Excludes any file named "*.bak" + exclude *.bak + # Excludes any sub-directory named "tmp" + exclude tmp/ + # Includes only some directories + include /movies/ + include /musics/ + include /pictures/ + +On the command line, using the -f option, you can only use "include" +patterns. For example: + # Checks only the .mp3 files. - # Note the "" use to avoid globbing expansion by the shell. + # Note the "" use to avoid globbing expansion by the shell in Unix. snapraid -f "*.mp3" check diff --git a/state.c b/state.c index 6a4a2b7a..1877e6a2 100644 --- a/state.c +++ b/state.c @@ -40,8 +40,8 @@ void state_init(struct snapraid_state* state) state->level = 1; /* default is the lowest protection */ state->hash = HASH_MURMUR3; /* default is the fastest */ tommy_array_init(&state->diskarr); - tommy_list_init(&state->excludelist); tommy_list_init(&state->contentlist); + tommy_list_init(&state->filterlist); } void state_done(struct snapraid_state* state) @@ -52,7 +52,7 @@ void state_done(struct snapraid_state* state) disk_free(tommy_array_get(&state->diskarr, i)); tommy_array_done(&state->diskarr); tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free); - tommy_list_foreach(&state->excludelist, (tommy_foreach_func*)filter_free); + tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free); } void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable) @@ -150,12 +150,19 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i s = strtoken(s); tommy_array_insert(&state->diskarr, disk_alloc(name, s)); } else if (strcmp(tag, "exclude") == 0) { - struct snapraid_filter* filter = filter_alloc(s); + struct snapraid_filter* filter = filter_alloc(-1, s); if (!filter) { fprintf(stderr, "Invalid 'exclude' specification '%s' in '%s' at line %u\n", s, path, line); exit(EXIT_FAILURE); } - tommy_list_insert_tail(&state->excludelist, &filter->node, filter); + tommy_list_insert_tail(&state->filterlist, &filter->node, filter); + } else if (strcmp(tag, "include") == 0) { + struct snapraid_filter* filter = filter_alloc(1, s); + if (!filter) { + fprintf(stderr, "Invalid 'include' specification '%s' in '%s' at line %u\n", s, path, line); + exit(EXIT_FAILURE); + } + tommy_list_insert_tail(&state->filterlist, &filter->node, filter); } else { fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path); exit(EXIT_FAILURE); @@ -573,6 +580,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist) { unsigned i; + /* if no filter, include all */ if (tommy_list_empty(filterlist)) return; @@ -596,24 +604,11 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist) /* for each file */ for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) { - tommy_node* k; struct snapraid_file* file = j->data; - /* filter it out as default */ - file->is_filtered = 1; - - /* for each filter */ - for(k=tommy_list_head(filterlist);k!=0;k=k->next) { - struct snapraid_filter* filter = k->data; + file->is_excluded = filter_path(filterlist, file->sub, 0) != 0; - if (filter_file(filter, file) == 0) { - /* mark it as filtered */ - file->is_filtered = 0; - break; - } - } - - if (state->verbose && !file->is_filtered) { + if (state->verbose && !file->is_excluded) { printf("Processing file '%s'\n", file->sub); } } diff --git a/state.h b/state.h index 0837bde1..6f3d3c94 100644 --- a/state.h +++ b/state.h @@ -43,7 +43,7 @@ struct snapraid_state { unsigned hash; /**< Hash kind used. */ tommy_list contentlist; /**< List of content files. */ tommy_array diskarr; /**< Disk array. */ - tommy_list excludelist; /**< List of exclusion. */ + tommy_list filterlist; /**< List of inclusion/exclusion. */ }; /**