Automatically excludes content files if stored in data disks.

HISTORY

@@ -6,6 +6,8 @@ SnapRAID HISTORY
 * If a file is modified or removed during a sync, the sync process doesn't
   stop anymore, but it will simply skip the file, resulting in an incomplete
   sync. Note that the sync will terminate with an error.
+* If the content file is placed in a data disk, it's automatically excluded
+  from the sync process.
 
 1.6 2011/9 - THIS UPDATE IS HIGHLY RECOMMENDED!
 ===============================================

TODO

@@ -6,8 +6,7 @@ This is the list of TODO items for SnapRAID.
 - v1.7
 
 * Put the content files in the data disks in the default configuration file.
-Also, automatically exclude them from the sync process without the need to add
-an exclusion rule.
+But this requires to have at leas N+1 or them.
 
 * Remove the dependency of the 'disk' order specification in the conf file.
 The used order can be defined by SnapRAID and saved in the content file to avoid

elem.c

@@ -148,7 +148,7 @@ static int filter_recurse(struct snapraid_filter* filter, const char* const_path
 	return 0;
 }
 
-int filter_path(tommy_list* filterlist, const char* path, int is_dir)
+static int filter_element(tommy_list* filterlist, const char* sub, int is_dir)
 {
 	tommy_node* i;
 
@@ -159,7 +159,7 @@ int filter_path(tommy_list* filterlist, const char* path, int is_dir)
 		int ret;
 		struct snapraid_filter* filter = i->data;
 
-		ret = filter_recurse(filter, path, is_dir);
+		ret = filter_recurse(filter, sub, is_dir);
 		if (ret > 0) {
 			/* include the file */
 			direction = 1;
@@ -181,6 +181,33 @@ int filter_path(tommy_list* filterlist, const char* path, int is_dir)
 	return 0;
 }
 
+int filter_path(tommy_list* filterlist, const char* sub)
+{
+	return filter_element(filterlist, sub, 0);
+}
+
+int filter_dir(tommy_list* filterlist, const char* sub)
+{
+	return filter_element(filterlist, sub, 1);
+}
+
+int filter_content(tommy_list* contentlist, const char* path)
+{
+	tommy_node* i;
+
+	for(i=tommy_list_head(contentlist);i!=0;i=i->next) {
+		struct snapraid_content* content = i->data;
+
+		/* FNM_PATHNAME because we are comparing full path */
+		/* FNM_NOESCAPE because we are comparing paths */
+		/* using fnmatch allows to ignore case in windows */
+		if (fnmatch(content->content, path, FNM_PATHNAME | FNM_NOESCAPE) == 0)
+			return -1;
+	}
+
+	return 0;
+}
+
 block_off_t block_file_pos(struct snapraid_block* block)
 {
 	struct snapraid_file* file = block_file_get(block);

elem.h

@@ -149,11 +149,24 @@ struct snapraid_filter* filter_alloc(int is_include, const char* pattern);
 void filter_free(struct snapraid_filter* filter);
 
 /**
- * Filters a path.
+ * Filters a path using a list of filters.
  * For each element of the path all the filters are applied, until the first one that matches.
- * Returns 0 if the files has to be processed.
+ * Return !=0 if it matches.
  */
-int filter_path(tommy_list* filterlist, const char* path, int is_dir);
+int filter_path(tommy_list* filterlist, const char* sub);
+
+/**
+ * Filters a dir using a list of filters.
+ * For each element of the path all the filters are applied, until the first one that matches.
+ * Return !=0 if it matches.
+ */
+int filter_dir(tommy_list* filterlist, const char* sub);
+
+/**
+ * Filters a path if it's a content file.
+ * Return !=0 if it matches.
+ */
+int filter_content(tommy_list* contentlist, const char* path);
 
 /**
  * Gets the file containing the block.

scan.c

@@ -376,8 +376,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i
 		}
 
 		if (S_ISREG(st.st_mode)) {
-			if (filter_path(&state->filterlist, sub_next, 0) == 0) {
-
+			if (filter_content(&state->contentlist, path_next) == 0
+				&& filter_path(&state->filterlist, sub_next) == 0
+			) {
 				/* check for read permission */
 				if (access(path_next, R_OK) != 0) {
 					fprintf(stderr, "warning: Ignoring, for missing read permission, file '%s'\n", path_next);
@@ -399,7 +400,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i
 				}
 			}
 		} else if (S_ISLNK(st.st_mode)) {
-			if (filter_path(&state->filterlist, sub_next, 0) == 0) {
+			if (filter_path(&state->filterlist, sub_next) == 0) {
 				char subnew[PATH_MAX];
 				int ret;
 
@@ -423,7 +424,7 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i
 				}
 			}
 		} else if (S_ISDIR(st.st_mode)) {
-			if (filter_path(&state->filterlist, sub_next, 1) == 0) {
+			if (filter_dir(&state->filterlist, sub_next) == 0) {
 				pathslash(path_next, sizeof(path_next));
 				pathslash(sub_next, sizeof(sub_next));
 				scan_dir(scan, state, output, disk, path_next, sub_next);
@@ -433,7 +434,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i
 				}
 			}
 		} else {
-			if (filter_path(&state->filterlist, sub_next, 0) == 0) {
+			if (filter_content(&state->contentlist, path_next) == 0
+				&& filter_path(&state->filterlist, sub_next) == 0
+			) {
 				fprintf(stderr, "warning: Ignoring special file '%s'\n", path_next);
 			} else {
 				if (state->verbose) {

snapraid.1

@@ -117,10 +117,13 @@ as the redundancy information may grow in size as the biggest data
 disk in the array.
 .PP
 These disks will be dedicated to store the \(a"parity\(a" and \(a"q\(hyparity\(a"
-files.
-You should not store other data on them, with the exception of
-a copy the \(a"content\(a" file, that contains the list of all the files
-stored in your array with all the checksums to verify their integrity.
+files. You should not store other data in them, with the only
+possible exception of the \(a"content\(a" files.
+.PP
+The list of files is stored in the \(a"content\(a" files, usually
+stored in the parity disks or in other locations.
+These files contain the details of you backup with all the
+checksums to verify its integrity.
 .PP
 For example, suppose that you are interested only at one parity level
 of protection, and that your disks are present in:
@@ -389,15 +392,17 @@ This option is optional and it can be used only one time.
 Defines the file to use to store the list and checksums of all the
 files present in your disk array.
 .PP
-It can be placed in the same disk of the parity and q\(hyparity files,
-but NOT in a data disk of the array.
+It can be placed in the disk used to store data or the parity.
+If you use a data disk, this file is automatically excluded
+from the \(a"sync\(a" process.
 .PP
 This option is mandatory and it can be used more time to save
 more copies of the same files.
 .PP
-It\(cqs suggested to store at least one copy for each parity disk.
-One more doesn\(cqt hurt, just in case you lose all the parity disks,
-and you want to be still able to check the data integrity.
+You have to store at least one copy for each parity disk used.
+It\(cqs suggested to use one more to ensure to always have
+one copy in case of multiple disks damage, to be able to
+check the data integrity of the remaining.
 .SS disk NAME DIR 
 Defines the name and the mount point of the disks of the array.
 NAME is used to identify the disk, and it must be unique.

snapraid.conf.example

@@ -14,8 +14,7 @@ parity /mnt/diskpar/parity
 # Defines the file to use as content list
 # You can use multiple specification to store more copies of the file
 # It's suggested to have at least N+1 copies of the file, where N is the number of parity files.
-# It must NOT be in a data disk
-# It can be in the disks used for parity storage
+# It can be in the disks used for data or parity storage
 # Format: "content FILE_PATH"
 content /mnt/diskpar/content
 #content /mnt/diskqar/content

snapraid.conf.example.windows

@@ -14,8 +14,7 @@ parity E:\par\parity
 # Defines the file to use as content list
 # You can use multiple specification to store more copies of the file
 # It's suggested to have at least N+1 copies of the file, where N is the number of parity files.
-# It must NOT be in a data disk
-# It can be in the disks used for parity storage
+# It can be in the disks used for data or parity storage
 # Format: "content FILE_PATH"
 content E:\par\content
 #content F:\qar\content

snapraid.d

@@ -83,10 +83,13 @@ Getting Started
 	disk in the array.
 
 	These disks will be dedicated to store the "parity" and "q-parity"
-	files.
-	You should not store other data on them, with the exception of
-	a copy the "content" file, that contains the list of all the files
-	stored in your array with all the checksums to verify their integrity.
+	files. You should not store other data in them, with the only
+	possible exception of the "content" files.
+
+	The list of files is stored in the "content" files, usually
+	stored in the parity disks or in other locations.
+	These files contain the details of you backup with all the
+	checksums to verify its integrity.
 
 	For example, suppose that you are interested only at one parity level
 	of protection, and that your disks are present in:
@@ -292,15 +295,17 @@ Configuration
 	Defines the file to use to store the list and checksums of all the
 	files present in your disk array.
 
-	It can be placed in the same disk of the parity and q-parity files,
-	but NOT in a data disk of the array.
+	It can be placed in the disk used to store data or the parity.
+	If you use a data disk, this file is automatically excluded
+	from the "sync" process.
 
 	This option is mandatory and it can be used more time to save
 	more copies of the same files.
 
-	It's suggested to store at least one copy for each parity disk.
-	One more doesn't hurt, just in case you lose all the parity disks,
-	and you want to be still able to check the data integrity.
+	You have to store at least one copy for each parity disk used.
+	It's suggested to use one more to ensure to always have
+	one copy in case of multiple disks damage, to be able to
+	check the data integrity of the remaining.
 
   disk NAME DIR
 	Defines the name and the mount point of the disks of the array.

snapraid.txt

@@ -96,10 +96,13 @@ as the redundancy information may grow in size as the biggest data
 disk in the array.
 
 These disks will be dedicated to store the "parity" and "q-parity"
-files.
-You should not store other data on them, with the exception of
-a copy the "content" file, that contains the list of all the files
-stored in your array with all the checksums to verify their integrity.
+files. You should not store other data in them, with the only
+possible exception of the "content" files.
+
+The list of files is stored in the "content" files, usually
+stored in the parity disks or in other locations.
+These files contain the details of you backup with all the
+checksums to verify its integrity.
 
 For example, suppose that you are interested only at one parity level
 of protection, and that your disks are present in:
@@ -328,15 +331,17 @@ This option is optional and it can be used only one time.
 Defines the file to use to store the list and checksums of all the
 files present in your disk array.
 
-It can be placed in the same disk of the parity and q-parity files,
-but NOT in a data disk of the array.
+It can be placed in the disk used to store data or the parity.
+If you use a data disk, this file is automatically excluded
+from the "sync" process.
 
 This option is mandatory and it can be used more time to save
 more copies of the same files.
 
-It's suggested to store at least one copy for each parity disk.
-One more doesn't hurt, just in case you lose all the parity disks,
-and you want to be still able to check the data integrity.
+You have to store at least one copy for each parity disk used.
+It's suggested to use one more to ensure to always have
+one copy in case of multiple disks damage, to be able to
+check the data integrity of the remaining.
 
 7.4 disk NAME DIR
 -----------------

state.c

@@ -56,6 +56,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 	STREAM* f;
 	unsigned line;
 	unsigned content_count;
+	unsigned content_required;
+	tommy_node* i;
 
 	state->verbose = verbose;
 	state->gui = gui;
@@ -83,7 +85,6 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 	}
 
 	line = 1;
-	content_count = 0;
 	while (1) {
 		char tag[PATH_MAX];
 		char buffer[PATH_MAX];
@@ -162,7 +163,6 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 			content = content_alloc(buffer);
 
 			tommy_list_insert_tail(&state->contentlist, &content->node, content);
-			++content_count;
 		} else if (strcmp(tag, "disk") == 0) {
 			char dir[PATH_MAX];
 
@@ -250,12 +250,23 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 		fprintf(stderr, "No 'parity' specification in '%s'\n", path);
 		exit(EXIT_FAILURE);
 	}
+
+	/* count the content files */
+	content_count = 0;
+	for(i=state->contentlist;i!=0;i=i->next) {
+		++content_count;
+	}
 	if (content_count == 0) {
 		fprintf(stderr, "No 'content' specification in '%s'\n", path);
 		exit(EXIT_FAILURE);
 	}
-	if (state->qarity[0] != 0 && content_count < 2) {
-		fprintf(stderr, "With 'q-parity' you must have at least two 'content' specifications in '%s'\n", path);
+	if (state->qarity[0] != 0) {
+		content_required = 2;
+	} else {
+		content_required = 1;
+	}
+	if (content_count < content_required) {
+		fprintf(stderr, "You must have one 'content' file for each parity disk.\n");
 		exit(EXIT_FAILURE);
 	}
 
@@ -706,8 +717,8 @@ static void state_write_one(struct snapraid_state* state, const char* path, unsi
 
 	printf("Saving state to %s...\n", path);
 
-	/* ignore some special files */
-	if (strcmp(path, "/dev/null") == 0 || strcmp(path, "NUL") == 0)
+	/* ignore some special files, this is an undocument feature */
+	if (strcmp(path, "/dev/null") == 0 || strcmp(path, "NUL") == 0 || strcmp(path, "nul") == 0)
 		return;
 
 	pathprint(tmp, sizeof(tmp), "%s.tmp", path);
@@ -886,7 +897,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist)
 		for(j=tommy_list_head(&disk->filelist);j!=0;j=j->next) {
 			struct snapraid_file* file = j->data;
 
-			if (filter_path(filterlist, file->sub, 0) != 0) {
+			if (filter_path(filterlist, file->sub) != 0) {
 				file_flag_set(file, FILE_IS_EXCLUDED);
 			}
 
@@ -899,7 +910,7 @@ void state_filter(struct snapraid_state* state, tommy_list* filterlist)
 		for(j=tommy_list_head(&disk->linklist);j!=0;j=j->next) {
 			struct snapraid_link* link = j->data;
 
-			if (filter_path(filterlist, link->sub, 0) != 0) {
+			if (filter_path(filterlist, link->sub) != 0) {
 				link_flag_set(link, FILE_IS_EXCLUDED);
 			}