rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Adds include logic in filtering.

Browse source
This commit is contained in:
Andrea Mazzoleni 2011-04-26 18:51:42 +02:00
parent 99420cffab
commit cb4a72afca
11 changed files with 248 additions and 92 deletions
Download patch file Download diff file Expand all files Collapse all files

4
HISTORY
View file

@ -3,8 +3,8 @@ SnapRAID HISTORY
1.1 2011/5
==========
* Added RAID6 support.
* Allow to specify multiple content files.
* Added RAID6 support and allow to specify multiple content files.
* Added filtering include logic, where anything not explicitely included is excluded.
* Changed the default checksum to Murmur3 hash. It's a lot faster than MD5.
MD5 is still supported for backward compatibility.
* The check command returns with an error code if any kind of error is present.

6
check.c
View file

@ -200,7 +200,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block
&& bit_has(block->flag, BLOCK_HAS_HASH) /* only if the block is hashed */
&& !block->file->is_filtered /* only if the file is not filtered out */
&& !block->file->is_excluded /* only if the file is not filtered out */
) {
one_tocheck = 1;
break;
@ -230,7 +230,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
struct snapraid_block* block = disk_block_get(handle[j].disk, i);
if (block
&& bit_has(block->flag, BLOCK_HAS_HASH) /* only if the block is hashed */
&& !block->file->is_filtered /* only if the file is not filtered out */
&& !block->file->is_excluded /* only if the file is not filtered out */
) {
one_tocheck = 1;
break;
@ -415,7 +415,7 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
/* update the fixed files */
for(j=0;j<failed_count;++j) {
/* do not fix if the file filtered out */
if (failed[j].block->file->is_filtered)
if (failed[j].block->file->is_excluded)
continue;
ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size);

66
elem.c
View file

@ -38,7 +38,7 @@ void content_free(struct snapraid_content* content)
free(content);
}
struct snapraid_filter* filter_alloc(const char* pattern)
struct snapraid_filter* filter_alloc(int direction, const char* pattern)
{
struct snapraid_filter* filter;
char* i;
@ -47,6 +47,7 @@ struct snapraid_filter* filter_alloc(const char* pattern)
filter = malloc_nofail(sizeof(struct snapraid_filter));
pathcpy(filter->pattern, sizeof(filter->pattern), pattern);
filter->direction = direction;
/* find first and last slash */
first = 0;
@ -93,35 +94,35 @@ void filter_free(struct snapraid_filter* filter)
free(filter);
}
int filter_path(struct snapraid_filter* filter, const char* path, const char* name, int is_dir)
static int filter_apply(struct snapraid_filter* filter, const char* path, const char* name, int is_dir)
{
/* matches dirs with dirs and files with files */
if (filter->is_dir && !is_dir)
return -1;
return 0;
if (!filter->is_dir && is_dir)
return -1;
return 0;
if (filter->is_path) {
/* skip initial slash, as always missing from the path */
if (fnmatch(filter->pattern + 1, path, FNM_PATHNAME) == 0)
return 0;
return filter->direction;
} else {
if (fnmatch(filter->pattern, name, 0) == 0)
return 0;
return filter->direction;
}
return -1;
return 0;
}
int filter_file(struct snapraid_filter* filter, struct snapraid_file* file)
static int filter_recurse(struct snapraid_filter* filter, const char* const_path, int is_dir)
{
char path[PATH_MAX];
char* name;
unsigned i;
pathcpy(path, sizeof(path), file->sub);
pathcpy(path, sizeof(path), const_path);
/* filter for all the directory */
/* filter for all the directories */
name = path;
for(i=0;path[i] != 0;++i) {
if (path[i] == '/') {
@ -129,8 +130,8 @@ int filter_file(struct snapraid_filter* filter, struct snapraid_file* file)
path[i] = 0;
/* filter the directory */
if (filter_path(filter, path, name, 1) == 0)
return 0;
if (filter_apply(filter, path, name, 1) != 0)
return filter->direction;
/* restore the slash */
path[i] = '/';
@ -141,10 +142,43 @@ int filter_file(struct snapraid_filter* filter, struct snapraid_file* file)
}
/* filter the final file */
if (filter_path(filter, path, name, 0) == 0)
return 0;
if (filter_apply(filter, path, name, is_dir) != 0)
return filter->direction;
return -1;
return 0;
}
int filter_path(tommy_list* filterlist, const char* path, int is_dir)
{
tommy_node* i;
int direction = 1; /* by default include all */
/* for each filter */
for(i=tommy_list_head(filterlist);i!=0;i=i->next) {
int ret;
struct snapraid_filter* filter = i->data;
ret = filter_recurse(filter, path, is_dir);
if (ret > 0) {
/* include the file */
direction = 1;
break;
} else if (ret < 0) {
/* exclude the file */
direction = -1;
break;
} else {
/* default is opposite of the last filter */
direction = -filter->direction;
/* continue with the next one */
}
}
if (direction < 0)
return -1;
return 0;
}
block_off_t block_file_pos(struct snapraid_block* block)
@ -189,7 +223,7 @@ struct snapraid_file* file_alloc(unsigned block_size, const char* sub, uint64_t
file->mtime = mtime;
file->inode = inode;
file->is_present = 0;
file->is_filtered = 0;
file->is_excluded = 0;
file->blockvec = malloc_nofail(file->blockmax * sizeof(struct snapraid_block));
/* set the back pointer */

18
elem.h
View file

@ -59,6 +59,7 @@ struct snapraid_filter {
char pattern[PATH_MAX]; /**< Filter pattern. */
int is_path; /**< If the pattern is only for the complete path. */
int is_dir; /**< If the pattern is only for dir. */
int direction; /**< If it's an inclusion (=1) or an exclusion (=-1). */
tommy_node node; /**< Next node in the list. */
};
@ -88,8 +89,7 @@ struct snapraid_file {
int64_t mtime; /**< Modification time. */
uint64_t inode; /**< Inode. */
int is_present; /**< If it's seen as present. */
int is_filtered; /**< If it's filtered out. */
int is_excluded; /**< If it's an excluded file from the processing. */
/* nodes for data structures */
tommy_node nodelist;
tommy_hashdyn_node nodeset;
@ -120,24 +120,20 @@ void content_free(struct snapraid_content* content);
/**
* Allocates an exclusion.
*/
struct snapraid_filter* filter_alloc(const char* pattern);
struct snapraid_filter* filter_alloc(int is_include, const char* pattern);
/**
* Deallocates an exclusion.
*/
void filter_free(struct snapraid_filter* filter);
/**
* Filters a path/name.
* Returns 0 if it matches.
*/
int filter_path(struct snapraid_filter* filter, const char* path, const char* name, int is_dir);
/**
* Filters a file.
* Returns 0 if it matches.
* Filters a path.
* For each element of the path all the filters are applied, until the first one that matches.
* Returns 0 if the files has to be processed.
*/
int filter_file(struct snapraid_filter* filter, struct snapraid_file* file);
int filter_path(tommy_list* filterlist, const char* path, int is_dir);
/**
* Gets the relative position of a block inside the file.

20
scan.c
View file

@ -186,22 +186,6 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
scan_file_insert(state, disk, file);
}
/**
* Checks if the specified path/file should be listet.
*/
static int scan_filter(struct snapraid_state* state, const char* path, const char* file, int is_dir)
{
tommy_node* node = tommy_list_head(&state->excludelist);
while (node) {
struct snapraid_filter* filter = node->data;
if (filter_path(filter, path, file, is_dir) == 0)
return -1;
node = node->next;
}
return 0;
}
/**
* Processes a directory.
*/
@ -241,7 +225,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s
}
if (S_ISREG(st.st_mode)) {
if (scan_filter(state, sub_next, name, 0) == 0) {
if (filter_path(&state->filterlist, sub_next, 0) == 0) {
scan_file(scan, state, disk, sub_next, &st);
} else {
if (state->verbose) {
@ -249,7 +233,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, s
}
}
} else if (S_ISDIR(st.st_mode)) {
if (scan_filter(state, sub_next, name, 1) == 0) {
if (filter_path(&state->filterlist, sub_next, 1) == 0) {
pathslash(path_next, sizeof(path_next));
pathslash(sub_next, sizeof(sub_next));
scan_dir(scan, state, disk, path_next, sub_next);

89
snapraid.1
View file

@ -390,8 +390,14 @@ you will invalidate the q\(hyparity file.
You should use one option for each disk of the array.
.TP
.B exclude PATTERN
Defines the file or directory patterns to exclude from the sync
process.
.TP
.B include PATTERN
Defines the file or directory patterns to exclude and include
in the sync process.
All the patterns are processed in the specified order, and
we get the result of the first pattern that matches.
If no pattern matches, the file is excluded if the last pattern
is an \(a"include\(a", or included if the last pattern is an \(a"exclude\(a".
See the PATTERN section for more details in the
pattern specifications.
This option can be used many times.
@ -477,7 +483,8 @@ block_size 256
.PD
.RE
.SH PATTERN
Patterns are used to select a subset of files to exclude or operate on.
Patterns are used to select a subset of files to exclude or include in
the process.
.PP
There are four different types of patterns:
.TP
@ -506,7 +513,11 @@ This pattern is applied only to directories and not to files.
Note that when globbing char are used in the command line, you have to
quote them in Unix. Otherwise the shell will try to expand them.
.PP
For example, in the configuration file:
In the configuration file, you can use different strategies to define
the files to process.
.PP
The most simple one is to only use \(a"exclude\(a" rules to remove all the
files and directories you do not want to process. For example:
.PP
.RS 4
# Excludes any file named \(a"*.bak\(a"
@ -525,7 +536,7 @@ exclude /lost+found/
.PD 0
.PP
.PD
# Excludes any directory named \(a"tmp\(a"
# Excludes any sub\(hydirectory named \(a"tmp\(a"
.PD 0
.PP
.PD
@ -535,14 +546,78 @@ exclude tmp/
.PD
.RE
.PP
For example, in the command line:
The opposite way is to define only the file you want to process, using
only \(a"include\(a" rules. For example:
.PP
.RS 4
# Includes only some directories
.PD 0
.PP
.PD
include /movies/
.PD 0
.PP
.PD
include /musics/
.PD 0
.PP
.PD
include /pictures/
.PD 0
.PP
.PD
.RE
.PP
The final way, is to mix \(a"exclude\(a" and \(a"include\(a" rules. In this case take
care that the order of rules is important. Previous rules have the
precedence over the later ones.
To get things simpler you can first have all the \(a"exclude\(a" rules and then
all the \(a"include\(a" ones. For example:
.PP
.RS 4
# Excludes any file named \(a"*.bak\(a"
.PD 0
.PP
.PD
exclude *.bak
.PD 0
.PP
.PD
# Excludes any sub\(hydirectory named \(a"tmp\(a"
.PD 0
.PP
.PD
exclude tmp/
.PD 0
.PP
.PD
# Includes only some directories
.PD 0
.PP
.PD
include /movies/
.PD 0
.PP
.PD
include /musics/
.PD 0
.PP
.PD
include /pictures/
.PD 0
.PP
.PD
.RE
.PP
On the command line, using the \(hyf option, you can only use \(a"include\(a"
patterns. For example:
.PP
.RS 4
# Checks only the .mp3 files.
.PD 0
.PP
.PD
# Note the \(a"\(a" use to avoid globbing expansion by the shell.
# Note the \(a"\(a" use to avoid globbing expansion by the shell in Unix.
.PD 0
.PP
.PD

2
snapraid.c
View file

@ -134,7 +134,7 @@ int main(int argc, char* argv[])
conf = optarg;
break;
case 'f' : {
struct snapraid_filter* filter = filter_alloc(optarg);
struct snapraid_filter* filter = filter_alloc(1, optarg);
if (!filter) {
fprintf(stderr, "Invalid filter specification '%s'\n", optarg);
exit(EXIT_FAILURE);

50
snapraid.d
View file

@ -285,8 +285,13 @@ Configuration
You should use one option for each disk of the array.
=exclude PATTERN
Defines the file or directory patterns to exclude from the sync
process.
=include PATTERN
Defines the file or directory patterns to exclude and include
in the sync process.
All the patterns are processed in the specified order, and
we get the result of the first pattern that matches.
If no pattern matches, the file is excluded if the last pattern
is an "include", or included if the last pattern is an "exclude".
See the PATTERN section for more details in the
pattern specifications.
This option can be used many times.
@ -330,7 +335,8 @@ Configuration
:block_size 256
Pattern
Patterns are used to select a subset of files to exclude or operate on.
Patterns are used to select a subset of files to exclude or include in
the process.
There are four different types of patterns:
@ -359,17 +365,47 @@ Pattern
Note that when globbing char are used in the command line, you have to
quote them in Unix. Otherwise the shell will try to expand them.
For example, in the configuration file:
In the configuration file, you can use different strategies to define
the files to process.
The most simple one is to only use "exclude" rules to remove all the
files and directories you do not want to process. For example:
:# Excludes any file named "*.bak"
:exclude *.bak
:# Excludes the root directory "/lost+found"
:exclude /lost+found/
:# Excludes any directory named "tmp"
:# Excludes any sub-directory named "tmp"
:exclude tmp/
For example, in the command line:
The opposite way is to define only the file you want to process, using
only "include" rules. For example:
:# Includes only some directories
:include /movies/
:include /musics/
:include /pictures/
The final way, is to mix "exclude" and "include" rules. In this case take
care that the order of rules is important. Previous rules have the
precedence over the later ones.
To get things simpler you can first have all the "exclude" rules and then
all the "include" ones. For example:
:# Excludes any file named "*.bak"
:exclude *.bak
:# Excludes any sub-directory named "tmp"
:exclude tmp/
:# Includes only some directories
:include /movies/
:include /musics/
:include /pictures/
On the command line, using the -f option, you can only use "include"
patterns. For example:
:# Checks only the .mp3 files.
:# Note the "" use to avoid globbing expansion by the shell.
:# Note the "" use to avoid globbing expansion by the shell in Unix.
:snapraid -f "*.mp3" check
Content

50
snapraid.txt
View file

@ -307,8 +307,13 @@ it should contains the following options:
You should use one option for each disk of the array.
exclude PATTERN
Defines the file or directory patterns to exclude from the sync
process.
include PATTERN
Defines the file or directory patterns to exclude and include
in the sync process.
All the patterns are processed in the specified order, and
we get the result of the first pattern that matches.
If no pattern matches, the file is excluded if the last pattern
is an "include", or included if the last pattern is an "exclude".
See the PATTERN section for more details in the
pattern specifications.
This option can be used many times.
@ -355,7 +360,8 @@ An example of a typical configuration is:
8 PATTERN
=========
Patterns are used to select a subset of files to exclude or operate on.
Patterns are used to select a subset of files to exclude or include in
the process.
There are four different types of patterns:
@ -384,17 +390,47 @@ There are four different types of patterns:
Note that when globbing char are used in the command line, you have to
quote them in Unix. Otherwise the shell will try to expand them.
For example, in the configuration file:
In the configuration file, you can use different strategies to define
the files to process.
The most simple one is to only use "exclude" rules to remove all the
files and directories you do not want to process. For example:
# Excludes any file named "*.bak"
exclude *.bak
# Excludes the root directory "/lost+found"
exclude /lost+found/
# Excludes any directory named "tmp"
# Excludes any sub-directory named "tmp"
exclude tmp/
For example, in the command line:
The opposite way is to define only the file you want to process, using
only "include" rules. For example:
# Includes only some directories
include /movies/
include /musics/
include /pictures/
The final way, is to mix "exclude" and "include" rules. In this case take
care that the order of rules is important. Previous rules have the
precedence over the later ones.
To get things simpler you can first have all the "exclude" rules and then
all the "include" ones. For example:
# Excludes any file named "*.bak"
exclude *.bak
# Excludes any sub-directory named "tmp"
exclude tmp/
# Includes only some directories
include /movies/
include /musics/
include /pictures/
On the command line, using the -f option, you can only use "include"
patterns. For example:
# Checks only the .mp3 files.
# Note the "" use to avoid globbing expansion by the shell.
# Note the "" use to avoid globbing expansion by the shell in Unix.
snapraid -f "*.mp3" check

33
state.c
View file

@ -40,8 +40,8 @@ void state_init(struct snapraid_state* state)
state->level = 1; /* default is the lowest protection */
state->hash = HASH_MURMUR3; /* default is the fastest */
tommy_array_init(&state->diskarr);
tommy_list_init(&state->excludelist);
tommy_list_init(&state->contentlist);
tommy_list_init(&state->filterlist);
}
void state_done(struct snapraid_state* state)
@ -52,7 +52,7 @@ void state_done(struct snapraid_state* state)
disk_free(tommy_array_get(&state->diskarr, i));
tommy_array_done(&state->diskarr);
tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
tommy_list_foreach(&state->excludelist, (tommy_foreach_func*)filter_free);
tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
}
void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable)
@ -150,12 +150,19 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
s = strtoken(s);
tommy_array_insert(&state->diskarr, disk_alloc(name, s));
} else if (strcmp(tag, "exclude") == 0) {
struct snapraid_filter* filter = filter_alloc(s);
struct snapraid_filter* filter = filter_alloc(-1, s);
if (!filter) {
fprintf(stderr, "Invalid 'exclude' specification '%s' in '%s' at line %u\n", s, path, line);
exit(EXIT_FAILURE);
}
tommy_list_insert_tail(&state->excludelist, &filter->node, filter);
tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
} else if (strcmp(tag, "include") == 0) {
struct snapraid_filter* filter = filter_alloc(1, s);
if (!filter) {
fprintf(stderr, "Invalid 'include' specification '%s' in '%s' at line %u\n", s, path, line);
exit(EXIT_FAILURE);
}
tommy_list_insert_tail(&state->filterlist, &filter->node, filter);
} else {
fprintf(stderr, "Unknown tag '%s' in '%s'\n", tag, path);
exit(EXIT_FAILURE);
@ -573,6 +580,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist)
{
unsigned i;
/* if no filter, include all */
if (tommy_list_empty(filterlist))
return;
@ -596,24 +604,11 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist)
/* for each file */
for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) {
tommy_node* k;
struct snapraid_file* file = j->data;
/* filter it out as default */
file->is_filtered = 1;
/* for each filter */
for(k=tommy_list_head(filterlist);k!=0;k=k->next) {
struct snapraid_filter* filter = k->data;
file->is_excluded = filter_path(filterlist, file->sub, 0) != 0;
if (filter_file(filter, file) == 0) {
/* mark it as filtered */
file->is_filtered = 0;
break;
}
}
if (state->verbose && !file->is_filtered) {
if (state->verbose && !file->is_excluded) {
printf("Processing file '%s'\n", file->sub);
}
}

2
state.h
View file

@ -43,7 +43,7 @@ struct snapraid_state {
unsigned hash; /**< Hash kind used. */
tommy_list contentlist; /**< List of content files. */
tommy_array diskarr; /**< Disk array. */
tommy_list excludelist; /**< List of exclusion. */
tommy_list filterlist; /**< List of inclusion/exclusion. */
};
/**