Use the st_dev field to ensure to use different disks.

HISTORY

@@ -10,6 +10,8 @@ SnapRAID HISTORY
   from the sync process.
 * Increased by one the minimum number of content files. Before it was only
   a suggestion, but now it's a requirement in case you put them in data disks.
+* Added checks to ensure that data and parity disks are diffent, and to
+  correctly count the number of copies of "content" files in different disks.
 
 1.6 2011/9 - THIS UPDATE IS HIGHLY RECOMMENDED!
 ===============================================

Makefile.am

@@ -77,7 +77,7 @@ maintainer-clean-local:
 
 # Automatic testing
 
-CHECKFLAGS =
+CHECKFLAGS = --test-skip-device
 CHECKCOUNT = 2000
 CHECKSIZE = 4000
 

TODO

@@ -5,9 +5,6 @@ This is the list of TODO items for SnapRAID.
 
 - v1.7
 
-* Use st_dev to effectively check if data/parity disks are different and
-content files are in different disks.
-
 * Put the content files in the data disks in the default configuration file.
 
 * Remove the dependency of the 'disk' order specification in the conf file.
@@ -36,6 +33,8 @@ in Unix files starting with '.'.
 
 - v2.x
 
+* Remove the old MD5 support.
+
 * Implement an "undelete" command to recover missing files, without the
 integrity check of the existing ones.
 It should restore only missing files, without recovering modified ones

elem.c

@@ -23,12 +23,13 @@
 /****************************************************************************/
 /* snapraid */
 
-struct snapraid_content* content_alloc(const char* path)
+struct snapraid_content* content_alloc(const char* path, uint64_t dev)
 {
 	struct snapraid_content* content;
 
 	content = malloc_nofail(sizeof(struct snapraid_content));
 	pathcpy(content->content, sizeof(content->content), path);
+	content->device = dev;
 
 	return content;
 }
@@ -320,7 +321,7 @@ int link_name_compare(const void* void_arg, const void* void_data)
 	return strcmp(arg, link->sub);
 }
 
-struct snapraid_disk* disk_alloc(const char* name, const char* dir)
+struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev)
 {
 	struct snapraid_disk* disk;
 
@@ -331,6 +332,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir)
 	/* ensure that the dir terminate with "/" if it isn't empty */
 	pathslash(disk->dir, sizeof(disk->dir));
 
+	disk->device = dev;
 	disk->first_free_block = 0;
 	tommy_list_init(&disk->filelist);
 	tommy_hashdyn_init(&disk->inodeset);

elem.h

@@ -49,6 +49,7 @@ typedef int64_t data_off_t;
  */
 struct snapraid_content {
 	char content[PATH_MAX]; /**< Path of the content file. */
+	uint64_t device; /**< Device identifier. */
 	tommy_node node; /**< Next node in the list. */
 };
 
@@ -120,6 +121,7 @@ struct snapraid_link {
 struct snapraid_disk {
 	char name[PATH_MAX]; /**< Name of the disk. */
 	char dir[PATH_MAX]; /**< Mount point of the disk. It always terminates with /. */
+	uint64_t device; /**< Device identifier. */
 	block_off_t first_free_block; /**< First free searching block. */
 	tommy_list filelist; /**< List of all the files. */
 	tommy_hashdyn inodeset; /**< Hashtable by inode of all the files. */
@@ -131,7 +133,7 @@ struct snapraid_disk {
 /**
  * Allocates a content.
  */
-struct snapraid_content* content_alloc(const char* path);
+struct snapraid_content* content_alloc(const char* path, uint64_t dev);
 
 /**
  * Deallocates a content.
@@ -299,7 +301,7 @@ static inline tommy_uint32_t link_name_hash(const char* name)
 /**
  * Allocates a disk.
  */
-struct snapraid_disk* disk_alloc(const char* name, const char* dir);
+struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev);
 
 /**
  * Deallocates a disk.

mingw.c

@@ -121,6 +121,8 @@ static void windows_info2stat(const BY_HANDLE_FILE_INFORMATION* info, struct win
 	st->st_ino |= info->nFileIndexLow;
 
 	st->st_nlink = info->nNumberOfLinks;
+
+	st->st_dev = info->dwVolumeSerialNumber;
 }
 
 /**
@@ -167,6 +169,9 @@ static void windows_finddata2stat(const WIN32_FIND_DATAW* info, struct windows_s
 
 	/* No link information available */
 	st->st_nlink = 0;
+
+	/* No device information available */
+	st->st_dev = 0;
 }
 
 static void windows_finddata2dirent(const WIN32_FIND_DATAW* info, struct windows_dirent* dirent)
@@ -259,7 +264,7 @@ int windows_mkdir(const char* file)
 	return _wmkdir(u8tou16(file));
 }
 
-int lstat_inode(const char* file, struct windows_stat* st)
+int lstat_ex(const char* file, struct windows_stat* st)
 {
 	BY_HANDLE_FILE_INFORMATION info;
 	HANDLE h;
@@ -293,6 +298,39 @@ int lstat_inode(const char* file, struct windows_stat* st)
 	return 0;
 }
 
+int windows_stat(const char* file, struct windows_stat* st)
+{
+	BY_HANDLE_FILE_INFORMATION info;
+	HANDLE h;
+
+	/*
+	 * Open the handle of the file.
+	 *
+	 * Use FILE_FLAG_BACKUP_SEMANTICS to open directories and to override the file security checks.
+	 */
+	h = CreateFileW(u8tou16(file), 0, FILE_SHARE_READ | FILE_SHARE_WRITE, 0, OPEN_EXISTING, FILE_FLAG_BACKUP_SEMANTICS, 0);
+	if (h == INVALID_HANDLE_VALUE) {
+		windows_errno(GetLastError());
+		return -1;
+	}
+
+	if (!GetFileInformationByHandle(h, &info))  {
+		DWORD error = GetLastError();
+		CloseHandle(h);
+		windows_errno(error);
+		return -1;
+	}
+
+	if (!CloseHandle(h)) {
+		windows_errno(GetLastError());
+		return -1;
+	}
+
+	windows_info2stat(&info, st);
+
+	return 0;
+}
+
 int windows_ftruncate(int fd, off64_t off)
 {
 	HANDLE h;

mingw.h

@@ -64,18 +64,24 @@ struct windows_stat {
 	int64_t st_mtime;
 	uint32_t st_mode;
 	uint32_t st_nlink;
+	uint32_t st_dev;
 };
 
 /**
- * Like the C fstat() including the inode information.
+ * Like the C fstat() including the inode/device information.
  */
 int windows_fstat(int fd, struct windows_stat* st);
 
 /**
- * Like the C stat() but without the inode information.
+ * Like the C lstat() but without the inode information.
  */
 int windows_lstat(const char* file, struct windows_stat* st);
 
+/**
+ * Like the C stat() including the inode/device information.
+ */
+int windows_stat(const char* file, struct windows_stat* st);
+
 /**
  * Like the C access().
  */
@@ -87,12 +93,12 @@ int windows_access(const char* file, int mode);
 int windows_mkdir(const char* file);
 
 /**
- * Like the C lstat() including the inode information.
+ * Like the C lstat() including the inode/device information.
  * It doesn't work for all kind of files and directories. For example "\System Volume Information" cannot be opened.
  * Note that instead lstat() works for all the files.
  */
-#define HAVE_LSTAT_INODE 1
+#define HAVE_LSTAT_EX 1
-int lstat_inode(const char* file, struct windows_stat* st);
+int lstat_ex(const char* file, struct windows_stat* st);
 
 /**
  * Like the C ftruncate().

scan.c

@@ -385,9 +385,9 @@ static void scan_dir(struct snapraid_scan* scan, struct snapraid_state* state, i
 					continue;
 				}
 
-#if HAVE_LSTAT_INODE
+#if HAVE_LSTAT_EX
 				/* get inode info about the file, Windows needs an additional step */
-				if (lstat_inode(path_next, &st) != 0) {
+				if (lstat_ex(path_next, &st) != 0) {
 					fprintf(stderr, "Error in stat_inode file '%s'. %s.\n", path_next, strerror(errno));
 					exit(EXIT_FAILURE);
 				}

snapraid.1

@@ -403,7 +403,8 @@ This option is optional and it can be used only one time.
 Defines the file to use to store the list and checksums of all the
 files present in your disk array.
 .PP
-It can be placed in the disk used to store data or the parity.
+It can be placed in the disk used to store data, parity, or
+any other disk available.
 If you use a data disk, this file is automatically excluded
 from the \(a"sync\(a" process.
 .PP

snapraid.c

@@ -68,6 +68,7 @@ void usage(void)
 #define OPT_TEST_KILL_AFTER_SYNC 257
 #define OPT_TEST_EXPECT_UNRECOVERABLE 258
 #define OPT_TEST_EXPECT_RECOVERABLE 259
+#define OPT_TEST_SKIP_DEVICE 260
 
 #if HAVE_GETOPT_LONG
 struct option long_options[] = {
@@ -83,10 +84,11 @@ struct option long_options[] = {
 	{ "help", 0, 0, 'h' },
 	{ "version", 0, 0, 'V' },
 	/* test specific options, DO NOT USE! */
-	{ "test-skip-self", 0, 0, OPT_TEST_SKIP_SELF },
 	{ "test-kill-after-sync", 0, 0, OPT_TEST_KILL_AFTER_SYNC },
 	{ "test-expect-unrecoverable", 0, 0, OPT_TEST_EXPECT_UNRECOVERABLE },
 	{ "test-expect-recoverable", 0, 0, OPT_TEST_EXPECT_RECOVERABLE },
+	{ "test-skip-self", 0, 0, OPT_TEST_SKIP_SELF },
+	{ "test-skip-device", 0, 0, OPT_TEST_SKIP_DEVICE },
 	{ 0, 0, 0, 0 }
 };
 #endif
@@ -121,6 +123,7 @@ int main(int argc, char* argv[])
 	int test_expect_recoverable;
 	int test_kill_after_sync;
 	int test_skip_self;
+	int test_skip_device;
 	const char* conf;
 	struct snapraid_state state;
 	int operation;
@@ -140,6 +143,7 @@ int main(int argc, char* argv[])
 	test_expect_recoverable = 0;
 	test_kill_after_sync = 0;
 	test_skip_self = 0;
+	test_skip_device = 0;
 	blockstart = 0;
 	blockcount = 0;
 	tommy_list_init(&filterlist);
@@ -199,9 +203,6 @@ int main(int argc, char* argv[])
 		case 'T' :
 			speed();
 			exit(EXIT_SUCCESS);
-		case OPT_TEST_SKIP_SELF :
-			test_skip_self = 1;
-			break;
 		case OPT_TEST_KILL_AFTER_SYNC :
 			test_kill_after_sync = 1;
 			break;
@@ -211,6 +212,12 @@ int main(int argc, char* argv[])
 		case OPT_TEST_EXPECT_RECOVERABLE :
 			test_expect_recoverable = 1;
 			break;
+		case OPT_TEST_SKIP_SELF :
+			test_skip_self = 1;
+			break;
+		case OPT_TEST_SKIP_DEVICE :
+			test_skip_device = 1;
+			break;
 		default:
 			fprintf(stderr, "Unknown option '%c'\n", (char)c);
 			exit(EXIT_FAILURE);
@@ -244,7 +251,7 @@ int main(int argc, char* argv[])
 
 	state_init(&state);
 
-	state_config(&state, conf, verbose, gui, force_zero, force_empty, test_expect_unrecoverable, test_expect_recoverable);
+	state_config(&state, conf, verbose, gui, force_zero, force_empty, test_expect_unrecoverable, test_expect_recoverable, test_skip_device);
 
 	if (operation == OPERATION_DIFF) {
 		state_read(&state);

snapraid.conf.example

@@ -11,8 +11,8 @@ parity /mnt/diskpar/parity
 # Format: "q-parity FILE_PATH"
 #q-parity /mnt/diskqar/q-parity
 
-# Defines the file to use as content list
+# Defines the files to use as content list
-# You can use multiple specification to store more copies of the file
+# You can use multiple specification to store more copies
 # You must have least one copy for each parity file plus one. Some more don't hurt
 # They can be in the disks used for data, parity or boot,
 # but each file must be in a different disk

snapraid.conf.example.windows

@@ -11,8 +11,8 @@ parity E:\par\parity
 # Format: "q-parity FILE_PATH"
 #q-parity F:\qar\q-parity
 
-# Defines the file to use as content list
+# Defines the files to use as content list
-# You can use multiple specification to store more copies of the file
+# You can use multiple specification to store more copies
 # You must have least one copy for each parity file plus one. Some more don't hurt
 # They can be in the disks used for data, parity or boot,
 # but each file must be in a different disk

snapraid.d

@@ -300,7 +300,8 @@ Configuration
 	Defines the file to use to store the list and checksums of all the
 	files present in your disk array.
 
-	It can be placed in the disk used to store data or the parity.
+	It can be placed in the disk used to store data, parity, or
+	any other disk available.
 	If you use a data disk, this file is automatically excluded
 	from the "sync" process.
 

snapraid.txt

@@ -336,7 +336,8 @@ This option is optional and it can be used only one time.
 Defines the file to use to store the list and checksums of all the
 files present in your disk array.
 
-It can be placed in the disk used to store data or the parity.
+It can be placed in the disk used to store data, parity, or
+any other disk available.
 If you use a data disk, this file is automatically excluded
 from the "sync" process.
 

state.c

@@ -32,7 +32,9 @@ void state_init(struct snapraid_state* state)
 	state->need_write = 0;
 	state->block_size = 256 * 1024; /* default 256 KiB */
 	state->parity[0] = 0;
+	state->parity_device = 0;
 	state->qarity[0] = 0;
+	state->qarity_device = 0;
 	state->level = 1; /* default is the lowest protection */
 	state->hash = HASH_MURMUR3; /* default is the fastest */
 	tommy_array_init(&state->diskarr);
@@ -51,13 +53,10 @@ void state_done(struct snapraid_state* state)
 	tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
 }
 
-void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable)
+void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable, int skip_device)
 {
 	STREAM* f;
 	unsigned line;
-	unsigned content_count;
-	unsigned content_required;
-	tommy_node* i;
 
 	state->verbose = verbose;
 	state->gui = gui;
@@ -125,6 +124,10 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 			}
 			state->block_size *= 1024;
 		} else if (strcmp(tag, "parity") == 0) {
+			char device[PATH_MAX];
+			char* slash;
+			struct stat st;
+
 			if (*state->parity) {
 				fprintf(stderr, "Multiple 'parity' specification in '%s' at line %u\n", path, line);
 				exit(EXIT_FAILURE);
@@ -136,8 +139,26 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 				exit(EXIT_FAILURE);
 			}
 
-			pathcpy(state->parity, sizeof(state->parity), buffer);
+			pathimport(state->parity, sizeof(state->parity), buffer);
+
+			/* get the device of the directory containing the content file */
+			pathimport(device, sizeof(device), buffer);
+			slash = strrchr(device, '/');
+			if (slash)
+				*slash = 0;
+			else
+				pathcpy(device, sizeof(device), ".");
+			if (stat(device, &st) != 0) {
+				fprintf(stderr, "Error accessing 'parity' dir '%s' specification in '%s' at line %u\n", device, path, line);
+				exit(EXIT_FAILURE);
+			}
+
+			state->parity_device = st.st_dev;
 		} else if (strcmp(tag, "q-parity") == 0) {
+			char device[PATH_MAX];
+			char* slash;
+			struct stat st;
+
 			if (*state->qarity) {
 				fprintf(stderr, "Multiple 'q-parity' specification in '%s' at line %u\n", path, line);
 				exit(EXIT_FAILURE);
@@ -149,10 +170,29 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 				exit(EXIT_FAILURE);
 			}
 
-			pathcpy(state->qarity, sizeof(state->qarity), buffer);
+			pathimport(state->qarity, sizeof(state->qarity), buffer);
+
+			/* get the device of the directory containing the content file */
+			pathimport(device, sizeof(device), buffer);
+			slash = strrchr(device, '/');
+			if (slash)
+				*slash = 0;
+			else
+				pathcpy(device, sizeof(device), ".");
+			if (stat(device, &st) != 0) {
+				fprintf(stderr, "Error accessing 'qarity' dir '%s' specification in '%s' at line %u\n", device, path, line);
+				exit(EXIT_FAILURE);
+			}
+
+			state->qarity_device = st.st_dev;
+
+			/* we have two level of parity */
 			state->level = 2;
 		} else if (strcmp(tag, "content") == 0) {
 			struct snapraid_content* content;
+			char device[PATH_MAX];
+			char* slash;
+			struct stat st;
 
 			ret = sgetlasttok(f, buffer, sizeof(buffer));
 			if (ret < 0) {
@@ -160,11 +200,25 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 				exit(EXIT_FAILURE);
 			}
 
-			content = content_alloc(buffer);
+			/* get the device of the directory containing the content file */
+			pathimport(device, sizeof(device), buffer);
+			slash = strrchr(device, '/');
+			if (slash)
+				*slash = 0;
+			else
+				pathcpy(device, sizeof(device), ".");
+			if (stat(device, &st) != 0) {
+				fprintf(stderr, "Error accessing 'content' dir '%s' specification in '%s' at line %u\n", device, path, line);
+				exit(EXIT_FAILURE);
+			}
+
+			content = content_alloc(buffer, st.st_dev);
 
 			tommy_list_insert_tail(&state->contentlist, &content->node, content);
 		} else if (strcmp(tag, "disk") == 0) {
 			char dir[PATH_MAX];
+			char device[PATH_MAX];
+			struct stat st;
 
 			ret = sgettok(f, buffer, sizeof(buffer));
 			if (ret < 0) {
@@ -180,7 +234,14 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 				exit(EXIT_FAILURE);
 			}
 
-			tommy_array_insert(&state->diskarr, disk_alloc(buffer, dir));
+			/* get the device of the dir */
+			pathimport(device, sizeof(device), dir);
+			if (stat(device, &st) != 0) {
+				fprintf(stderr, "Error accessing 'disk' '%s' specification in '%s' at line %u\n", dir, device, line);
+				exit(EXIT_FAILURE);
+			}
+
+			tommy_array_insert(&state->diskarr, disk_alloc(buffer, dir, st.st_dev));
 		} else if (strcmp(tag, "exclude") == 0) {
 			struct snapraid_filter* filter;
 
@@ -251,23 +312,73 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
 		exit(EXIT_FAILURE);
 	}
 
-	/* count the content files */
+	if (tommy_list_empty(&state->contentlist)) {
-	content_count = 0;
-	for(i=state->contentlist;i!=0;i=i->next) {
-		++content_count;
-	}
-	if (content_count == 0) {
 		fprintf(stderr, "No 'content' specification in '%s'\n", path);
 		exit(EXIT_FAILURE);
 	}
-	if (state->qarity[0] != 0) {
+
-		content_required = 3;
+	/* count the content files */
-	} else {
+	if (!skip_device) {
-		content_required = 2;
+		unsigned content_count;
+		tommy_node* i;
+
+		content_count = 0;
+		for(i=state->contentlist;i!=0;i=i->next) {
+			tommy_node* j;
+			struct snapraid_content* content = i->data;
+
+			/* check if there are others in the same disk */
+			for(j=i->next;j!=0;j=j->next) {
+				struct snapraid_content* other = j->data;
+				if (content->device == other->device) {
+					break;
+				}
+			}
+			if (j != 0) {
+				/* skip it */
+				continue;
+			}
+
+			++content_count;
+		}
+
+		if (content_count < state->level+1) {
+			fprintf(stderr, "You must have at least %d 'content' files in different disks.\n", state->level+1);
+			exit(EXIT_FAILURE);
+		}
 	}
-	if (content_count < content_required) {
+
-		fprintf(stderr, "You must have at least %d 'content' files. One for each parity disk plus one.\n", content_required);
+	/* check if all the data and parity disks are different */
-		exit(EXIT_FAILURE);
+	if (!skip_device) {
+		unsigned j;
+		unsigned diskmax;
+		diskmax = tommy_array_size(&state->diskarr);
+		for(j=0;j<diskmax;++j) {
+			unsigned k;
+			struct snapraid_disk* disk = tommy_array_get(&state->diskarr, j);
+
+			for(k=j+1;k<diskmax;++k) {
+				struct snapraid_disk* other = tommy_array_get(&state->diskarr, k);
+				if (disk->device == other->device) {
+					fprintf(stderr, "Disks '%s' and '%s' are on the same device.\n", disk->dir, other->dir);
+					exit(EXIT_FAILURE);
+				}
+			}
+
+			if (disk->device == state->parity_device) {
+				fprintf(stderr, "Disk '%s' and parity '%s' are on the same device.\n", disk->dir, state->parity);
+				exit(EXIT_FAILURE);
+			}
+
+			if (disk->device == state->qarity_device) {
+				fprintf(stderr, "Disk '%s' and parity '%s' are on the same device.\n", disk->dir, state->qarity);
+				exit(EXIT_FAILURE);
+			}
+		}
+		if (state->parity_device == state->qarity_device) {
+			fprintf(stderr, "Parity '%s' and '%s' are on the same device.\n", state->parity, state->qarity);
+			exit(EXIT_FAILURE);
+		}
 	}
 
 	if (state->gui) {

state.h

@@ -39,7 +39,9 @@ struct snapraid_state {
 	int need_write; /**< If the state is changed. */
 	uint32_t block_size; /**< Block size in bytes. */
 	char parity[PATH_MAX]; /**< Path of the parity file. */
+	uint64_t parity_device; /**< Device identifier of the parity. */
 	char qarity[PATH_MAX]; /**< Path of the qarity file. */
+	uint64_t qarity_device; /**< Device identifier of the qarity. */
 	unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */
 	unsigned hash; /**< Hash kind used. */
 	tommy_list contentlist; /**< List of content files. */
@@ -62,7 +64,7 @@ void state_done(struct snapraid_state* state);
 /**
  * Read the configuration file.
  */
-void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable);
+void state_config(struct snapraid_state* state, const char* path, int verbose, int gui, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable, int skip_device);
 
 /**
  * Read the state.