diff --git a/HISTORY b/HISTORY index d1afaaa5..aa007574 100644 --- a/HISTORY +++ b/HISTORY @@ -6,6 +6,8 @@ SnapRAID HISTORY * If a file is modified or removed during a sync, the sync process doesn't stop anymore, but it will simply skip the file, resulting in an incomplete sync. Note that the sync will terminate with an error. +* If the content file is placed in a data disk, it's automatically excluded + from the sync process. 1.6 2011/9 - THIS UPDATE IS HIGHLY RECOMMENDED! =============================================== diff --git a/TODO b/TODO index 79dbadac..31aa6655 100644 --- a/TODO +++ b/TODO @@ -6,8 +6,7 @@ This is the list of TODO items for SnapRAID. - v1.7 * Put the content files in the data disks in the default configuration file. -Also, automatically exclude them from the sync process without the need to add -an exclusion rule. +But this requires to have at leas N+1 or them. * Remove the dependency of the 'disk' order specification in the conf file. The used order can be defined by SnapRAID and saved in the content file to avoid diff --git a/elem.c b/elem.c index 75768c3a..c7b88df5 100644 --- a/elem.c +++ b/elem.c @@ -148,7 +148,7 @@ static int filter_recurse(struct snapraid_filter* filter, const char* const_path return 0; } -int filter_path(tommy_list* filterlist, const char* path, int is_dir) +static int filter_element(tommy_list* filterlist, const char* sub, int is_dir) { tommy_node* i; @@ -159,7 +159,7 @@ int filter_path(tommy_list* filterlist, const char* path, int is_dir) int ret; struct snapraid_filter* filter = i->data; - ret = filter_recurse(filter, path, is_dir); + ret = filter_recurse(filter, sub, is_dir); if (ret > 0) { /* include the file */ direction = 1; @@ -181,6 +181,33 @@ int filter_path(tommy_list* filterlist, const char* path, int is_dir) return 0; } +int filter_path(tommy_list* filterlist, const char* sub) +{ + return filter_element(filterlist, sub, 0); +} + +int filter_dir(tommy_list* filterlist, const char* sub) +{ + return filter_element(filterlist, sub, 1); +} + +int filter_content(tommy_list* contentlist, const char* path) +{ + tommy_node* i; + + for(i=tommy_list_head(contentlist);i!=0;i=i->next) { + struct snapraid_content* content = i->data; + + /* FNM_PATHNAME because we are comparing full path */ + /* FNM_NOESCAPE because we are comparing paths */ + /* using fnmatch allows to ignore case in windows */ + if (fnmatch(content->content, path, FNM_PATHNAME | FNM_NOESCAPE) == 0) + return -1; + } + + return 0; +} + block_off_t block_file_pos(struct snapraid_block* block) { struct snapraid_file* file = block_file_get(block); diff --git a/elem.h b/elem.h index c5002009..ec3aa5d0 100644 --- a/elem.h +++ b/elem.h @@ -149,11 +149,24 @@ struct snapraid_filter* filter_alloc(int is_include, const char* pattern); void filter_free(struct snapraid_filter* filter); /** - * Filters a path. + * Filters a path using a list of filters. * For each element of the path all the filters are applied, until the first one that matches. - * Returns 0 if the files has to be processed. + * Return !=0 if it matches. */ -int filter_path(tommy_list* filterlist, const char* path, int is_dir); +int filter_path(tommy_list* filterlist, const char* sub); + +/** + * Filters a dir using a list of filters. + * For each element of the path all the filters are applied, until the first one that matches. + * Return !=0 if it matches. + */ +int filter_dir(tommy_list* filterlist, const char* sub); + +/** + * Filters a path if it's a content file. + * Return !=0 if it matches. + */ +int filter_content(tommy_list* contentlist, const char* path); /** * Gets the file containing the block. diff --git a/scan.c b/scan.c index 327add1e..3cf1fdc5 100644 --- a/scan.c +++ b/scan.c @@ -376,8 +376,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } if (S_ISREG(st.st_mode)) { - if (filter_path(&state->filterlist, sub_next, 0) == 0) { - + if (filter_content(&state->contentlist, path_next) == 0 + && filter_path(&state->filterlist, sub_next) == 0 + ) { /* check for read permission */ if (access(path_next, R_OK) != 0) { fprintf(stderr, "warning: Ignoring, for missing read permission, file '%s'\n", path_next); @@ -399,7 +400,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } } } else if (S_ISLNK(st.st_mode)) { - if (filter_path(&state->filterlist, sub_next, 0) == 0) { + if (filter_path(&state->filterlist, sub_next) == 0) { char subnew[PATH_MAX]; int ret; @@ -423,7 +424,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } } } else if (S_ISDIR(st.st_mode)) { - if (filter_path(&state->filterlist, sub_next, 1) == 0) { + if (filter_dir(&state->filterlist, sub_next) == 0) { pathslash(path_next, sizeof(path_next)); pathslash(sub_next, sizeof(sub_next)); scan_dir(scan, state, output, disk, path_next, sub_next); @@ -433,7 +434,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i } } } else { - if (filter_path(&state->filterlist, sub_next, 0) == 0) { + if (filter_content(&state->contentlist, path_next) == 0 + && filter_path(&state->filterlist, sub_next) == 0 + ) { fprintf(stderr, "warning: Ignoring special file '%s'\n", path_next); } else { if (state->verbose) { diff --git a/snapraid.1 b/snapraid.1 index c3540dd1..4197cfab 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -117,10 +117,13 @@ as the redundancy information may grow in size as the biggest data disk in the array. .PP These disks will be dedicated to store the \(a"parity\(a" and \(a"q\(hyparity\(a" -files. -You should not store other data on them, with the exception of -a copy the \(a"content\(a" file, that contains the list of all the files -stored in your array with all the checksums to verify their integrity. +files. You should not store other data in them, with the only +possible exception of the \(a"content\(a" files. +.PP +The list of files is stored in the \(a"content\(a" files, usually +stored in the parity disks or in other locations. +These files contain the details of you backup with all the +checksums to verify its integrity. .PP For example, suppose that you are interested only at one parity level of protection, and that your disks are present in: @@ -389,15 +392,17 @@ This option is optional and it can be used only one time. Defines the file to use to store the list and checksums of all the files present in your disk array. .PP -It can be placed in the same disk of the parity and q\(hyparity files, -but NOT in a data disk of the array. +It can be placed in the disk used to store data or the parity. +If you use a data disk, this file is automatically excluded +from the \(a"sync\(a" process. .PP This option is mandatory and it can be used more time to save more copies of the same files. .PP -It\(cqs suggested to store at least one copy for each parity disk. -One more doesn\(cqt hurt, just in case you lose all the parity disks, -and you want to be still able to check the data integrity. +You have to store at least one copy for each parity disk used. +It\(cqs suggested to use one more to ensure to always have +one copy in case of multiple disks damage, to be able to +check the data integrity of the remaining. .SS disk NAME DIR Defines the name and the mount point of the disks of the array. NAME is used to identify the disk, and it must be unique. diff --git a/snapraid.conf.example b/snapraid.conf.example index 2b937bc3..83056a55 100644 --- a/snapraid.conf.example +++ b/snapraid.conf.example @@ -14,8 +14,7 @@ parity /mnt/diskpar/parity # Defines the file to use as content list # You can use multiple specification to store more copies of the file # It's suggested to have at least N+1 copies of the file, where N is the number of parity files. -# It must NOT be in a data disk -# It can be in the disks used for parity storage +# It can be in the disks used for data or parity storage # Format: "content FILE_PATH" content /mnt/diskpar/content #content /mnt/diskqar/content diff --git a/snapraid.conf.example.windows b/snapraid.conf.example.windows index 5d5430d9..8385f477 100644 --- a/snapraid.conf.example.windows +++ b/snapraid.conf.example.windows @@ -14,8 +14,7 @@ parity E:\par\parity # Defines the file to use as content list # You can use multiple specification to store more copies of the file # It's suggested to have at least N+1 copies of the file, where N is the number of parity files. -# It must NOT be in a data disk -# It can be in the disks used for parity storage +# It can be in the disks used for data or parity storage # Format: "content FILE_PATH" content E:\par\content #content F:\qar\content diff --git a/snapraid.d b/snapraid.d index 59dc5558..f175a3dc 100644 --- a/snapraid.d +++ b/snapraid.d @@ -83,10 +83,13 @@ Getting Started disk in the array. These disks will be dedicated to store the "parity" and "q-parity" - files. - You should not store other data on them, with the exception of - a copy the "content" file, that contains the list of all the files - stored in your array with all the checksums to verify their integrity. + files. You should not store other data in them, with the only + possible exception of the "content" files. + + The list of files is stored in the "content" files, usually + stored in the parity disks or in other locations. + These files contain the details of you backup with all the + checksums to verify its integrity. For example, suppose that you are interested only at one parity level of protection, and that your disks are present in: @@ -292,15 +295,17 @@ Configuration Defines the file to use to store the list and checksums of all the files present in your disk array. - It can be placed in the same disk of the parity and q-parity files, - but NOT in a data disk of the array. + It can be placed in the disk used to store data or the parity. + If you use a data disk, this file is automatically excluded + from the "sync" process. This option is mandatory and it can be used more time to save more copies of the same files. - It's suggested to store at least one copy for each parity disk. - One more doesn't hurt, just in case you lose all the parity disks, - and you want to be still able to check the data integrity. + You have to store at least one copy for each parity disk used. + It's suggested to use one more to ensure to always have + one copy in case of multiple disks damage, to be able to + check the data integrity of the remaining. disk NAME DIR Defines the name and the mount point of the disks of the array. diff --git a/snapraid.txt b/snapraid.txt index 6c2dc18a..dce44b71 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -96,10 +96,13 @@ as the redundancy information may grow in size as the biggest data disk in the array. These disks will be dedicated to store the "parity" and "q-parity" -files. -You should not store other data on them, with the exception of -a copy the "content" file, that contains the list of all the files -stored in your array with all the checksums to verify their integrity. +files. You should not store other data in them, with the only +possible exception of the "content" files. + +The list of files is stored in the "content" files, usually +stored in the parity disks or in other locations. +These files contain the details of you backup with all the +checksums to verify its integrity. For example, suppose that you are interested only at one parity level of protection, and that your disks are present in: @@ -328,15 +331,17 @@ This option is optional and it can be used only one time. Defines the file to use to store the list and checksums of all the files present in your disk array. -It can be placed in the same disk of the parity and q-parity files, -but NOT in a data disk of the array. +It can be placed in the disk used to store data or the parity. +If you use a data disk, this file is automatically excluded +from the "sync" process. This option is mandatory and it can be used more time to save more copies of the same files. -It's suggested to store at least one copy for each parity disk. -One more doesn't hurt, just in case you lose all the parity disks, -and you want to be still able to check the data integrity. +You have to store at least one copy for each parity disk used. +It's suggested to use one more to ensure to always have +one copy in case of multiple disks damage, to be able to +check the data integrity of the remaining. 7.4 disk NAME DIR ----------------- diff --git a/state.c b/state.c index 0f96087b..ce3f27d3 100644 --- a/state.c +++ b/state.c @@ -56,6 +56,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i STREAM* f; unsigned line; unsigned content_count; + unsigned content_required; + tommy_node* i; state->verbose = verbose; state->gui = gui; @@ -83,7 +85,6 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i } line = 1; - content_count = 0; while (1) { char tag[PATH_MAX]; char buffer[PATH_MAX]; @@ -162,7 +163,6 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i content = content_alloc(buffer); tommy_list_insert_tail(&state->contentlist, &content->node, content); - ++content_count; } else if (strcmp(tag, "disk") == 0) { char dir[PATH_MAX]; @@ -250,12 +250,23 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i fprintf(stderr, "No 'parity' specification in '%s'\n", path); exit(EXIT_FAILURE); } + + /* count the content files */ + content_count = 0; + for(i=state->contentlist;i!=0;i=i->next) { + ++content_count; + } if (content_count == 0) { fprintf(stderr, "No 'content' specification in '%s'\n", path); exit(EXIT_FAILURE); } - if (state->qarity[0] != 0 && content_count < 2) { - fprintf(stderr, "With 'q-parity' you must have at least two 'content' specifications in '%s'\n", path); + if (state->qarity[0] != 0) { + content_required = 2; + } else { + content_required = 1; + } + if (content_count < content_required) { + fprintf(stderr, "You must have one 'content' file for each parity disk.\n"); exit(EXIT_FAILURE); } @@ -706,8 +717,8 @@ static void state_write_one(struct snapraid_state* state, const char* path, unsi printf("Saving state to %s...\n", path); - /* ignore some special files */ - if (strcmp(path, "/dev/null") == 0 || strcmp(path, "NUL") == 0) + /* ignore some special files, this is an undocument feature */ + if (strcmp(path, "/dev/null") == 0 || strcmp(path, "NUL") == 0 || strcmp(path, "nul") == 0) return; pathprint(tmp, sizeof(tmp), "%s.tmp", path); @@ -886,7 +897,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist) for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) { struct snapraid_file* file = j->data; - if (filter_path(filterlist, file->sub, 0) != 0) { + if (filter_path(filterlist, file->sub) != 0) { file_flag_set(file, FILE_IS_EXCLUDED); } @@ -899,7 +910,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist) for(j=tommy_list_head(&disk->linklist);j!=0;j=j->next) { struct snapraid_link* link = j->data; - if (filter_path(filterlist, link->sub, 0) != 0) { + if (filter_path(filterlist, link->sub) != 0) { link_flag_set(link, FILE_IS_EXCLUDED); }