Uses a smarter copy detection to avoid false positive.

Now a if the file has a zero nanosecond timestamp, it must
match the full pathname.
Also add the new -N option and an new undocumented test-nano
command to set randomly the nanosecond timestamp of files
that have it at zero.

HISTORY

@@ -4,6 +4,14 @@ SnapRAID HISTORY
 6.2 2014/5
 ==========
  * Fixed the regression test when run as root.
+ * Added a new euristic to detect file copies. Now a file is assumed
+   to be a copy if name, size and nanosecond timestamp are matching,
+   but if the nanosecond part of the timestamp is 0, we require
+   the full path matching and not only the name.
+ * Added the -N, --force-nocopy option to disable completely
+   the copy detection. SnapRAID also suggest to use this option
+   in the error message of a data mismatch if caused by the
+   copy detection.
 
 6.1 2014/4
 ==========

Makefile.am

@@ -30,6 +30,7 @@ snapraid_SOURCES = \
 	cmdline/pool.c \
 	cmdline/parity.c \
 	cmdline/handle.c \
+	cmdline/nano.c \
 	cmdline/fnmatch.c \
 	cmdline/selftest.c \
 	cmdline/speed.c \
@@ -184,6 +185,12 @@ endif
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) check
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -a check
+	echo --- Dry
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-dry
+	echo --- Nano
+	touch -d "2001-02-01 12:34:56" bench/disk1/a/a*
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-nano
 	echo --- Check the --gen-conf command
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --gen-conf bench/content
 	echo --- Filter
@@ -237,8 +244,6 @@ endif
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) fix -l test.log
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) check
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check
-	echo --- Dry
-	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-dry
 	echo --- Scrub some times
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-force-scrub 100 scrub
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) status

cmdline/elem.c

@@ -428,6 +428,20 @@ void file_copy(struct snapraid_file* src_file, struct snapraid_file* dst_file)
 		/* LCOV_EXCL_STOP */
 	}
 
+	if (src_file->mtime_sec != dst_file->mtime_sec) {
+		/* LCOV_EXCL_START */
+		fprintf(stderr, "Internal inconsistency in copy file with different mtime_sec\n");
+		exit(EXIT_FAILURE);
+		/* LCOV_EXCL_STOP */
+	}
+
+	if (src_file->mtime_nsec != dst_file->mtime_nsec) {
+		/* LCOV_EXCL_START */
+		fprintf(stderr, "Internal inconsistency in copy file with different mtime_nsec\n");
+		exit(EXIT_FAILURE);
+		/* LCOV_EXCL_STOP */
+	}
+
 	for(i=0;i<dst_file->blockmax;++i) {
 		/* set a block with hash computed but without parity */
 		block_state_set(&dst_file->blockvec[i], BLOCK_STATE_REP);
@@ -435,6 +449,8 @@ void file_copy(struct snapraid_file* src_file, struct snapraid_file* dst_file)
 		/* copy the hash */
 		memcpy(dst_file->blockvec[i].hash, src_file->blockvec[i].hash, HASH_SIZE);
 	}
+
+	file_flag_set(dst_file, FILE_IS_COPY);
 }
 
 const char* file_name(const struct snapraid_file* file)
@@ -538,6 +554,17 @@ int file_namestamp_compare(const void* void_a, const void* void_b)
 	return file_stamp_compare(void_a, void_b);
 }
 
+int file_pathstamp_compare(const void* void_a, const void* void_b)
+{
+	int ret;
+
+	ret = file_path_compare(void_a, void_b);
+	if (ret != 0)
+		return ret;
+
+	return file_stamp_compare(void_a, void_b);
+}
+
 struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag)
 {
 	struct snapraid_link* link;

cmdline/elem.h

@@ -203,6 +203,12 @@ struct snapraid_block {
  */
 #define FILE_IS_FINISHED 0x40
 
+/**
+ * If the file hash was obtained from a file copy
+ * identified by the same name, size and stamp.
+ */
+#define FILE_IS_COPY 0x80
+
 #define FILE_IS_HARDLINK 0x100 /**< If it's an hardlink. */
 #define FILE_IS_SYMLINK 0x200 /**< If it's a file symlink. */
 #define FILE_IS_SYMDIR 0x400 /**< If it's a dir symlink for Windows. Not yet supported. */
@@ -696,6 +702,7 @@ int file_path_compare_to_arg(const void* void_arg, const void* void_data);
 int file_name_compare(const void* void_a, const void* void_b);
 int file_stamp_compare(const void* void_a, const void* void_b);
 int file_namestamp_compare(const void* void_a, const void* void_b);
+int file_pathstamp_compare(const void* void_a, const void* void_b);
 
 /**
  * Computes the hash of a file path.

cmdline/handle.c

@@ -393,7 +393,7 @@ int file_utime(struct snapraid_file* file, int f)
 
 	ret = futimesat(f, 0, tv);
 #else
-#error No function available to set file timestamps
+#error No function available to set file timestamps with sub-second precision
 #endif
 
 	if (ret != 0) {

cmdline/scan.c

@@ -519,10 +519,10 @@ static void scan_file(struct snapraid_scan* scan, int output, const char* sub, c
 			/* mark as present */
 			file_flag_set(file, FILE_IS_PRESENT);
 
-			/* update the nano seconds mtime only if different */
+			/* update the nanoseconds mtime only if different */
 			/* to avoid unneeded updates */
 			if (file->mtime_nsec == STAT_NSEC_INVALID
-				&& STAT_NSEC(st) != STAT_NSEC_INVALID
+				&& STAT_NSEC(st) != file->mtime_nsec
 			) {
 				file->mtime_nsec = STAT_NSEC(st);
 
@@ -746,32 +746,40 @@ static void scan_file(struct snapraid_scan* scan, int output, const char* sub, c
 	/* mark it as present */
 	file_flag_set(file, FILE_IS_PRESENT);
 
-	/* now search for a file with the same name and timestamp in all the disks */
-	for(i=state->disklist;i!=0;i=i->next) {
-		struct snapraid_disk* other_disk = i->data;
-		struct snapraid_file* other_file;
+	/* if copy detection is enabled */
+	if (!state->opt.force_nocopy) {
 		tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec);
 
-		/* search for a file with the same name and stamp */
-		other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
+		/* search for a file with the same name and stamp in all the disks */
+		for(i=state->disklist;i!=0;i=i->next) {
+			struct snapraid_disk* other_disk = i->data;
+			struct snapraid_file* other_file;
 
-		/* if found, and it's a fully hashed file */
-		if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
-			/* assume that the file is a copy, and reuse the hash */
-			file_copy(other_file, file);
+			/* if the nanosecond part of the time stamp is valid, search */
+			/* for name and stamp, otherwise for path and stamp */
+			if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID)
+				other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash);
+			else
+				other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash);
 
-			/* revert old counter and use the copy one */
-			add_insert = 0;
-			add_change = 0;
-			++scan->count_copy;
+			/* if found, and it's a fully hashed file */
+			if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) {
+				/* assume that the file is a copy, and reuse the hash */
+				file_copy(other_file, file);
 
-			fprintf(stdlog, "scan:copy:%s:%s:%s:%s\n", other_disk->name, other_file->sub, disk->name, file->sub);
-			if (output) {
-				printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub);
+				/* revert old counter and use the copy one */
+				add_insert = 0;
+				add_change = 0;
+				++scan->count_copy;
+
+				fprintf(stdlog, "scan:copy:%s:%s:%s:%s\n", other_disk->name, other_file->sub, disk->name, file->sub);
+				if (output) {
+					printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub);
+				}
+
+				/* no need to continue the search */
+				break;
 			}
-
-			/* no need to continue the search */
-			break;
 		}
 	}
 

cmdline/snapraid.c

@@ -64,6 +64,7 @@ void usage(void)
 	printf("  " SWITCH_GETOPT_LONG("-E, --force-empty     ", "-E") "  Force synching of disks that get empty\n");
 	printf("  " SWITCH_GETOPT_LONG("-U, --force-uuid      ", "-U") "  Force commands on disks with uuid changed\n");
 	printf("  " SWITCH_GETOPT_LONG("-D, --force-device    ", "-D") "  Force commands on disks with same device id\n");
+	printf("  " SWITCH_GETOPT_LONG("-N, --force-nocopy    ", "-N") "  Force commands disabling the copy detection\n");
 	printf("  " SWITCH_GETOPT_LONG("-s, --start BLKSTART  ", "-s") "  Start from the specified block number\n");
 	printf("  " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT  ", "-t") "  Count of block to process\n");
 	printf("  " SWITCH_GETOPT_LONG("-v, --verbose         ", "-v") "  Verbose\n");
@@ -249,7 +250,7 @@ struct option long_options[] = {
 	{ "force-empty", 0, 0, 'E' },
 	{ "force-uuid", 0, 0, 'U' },
 	{ "force-device", 0, 0, 'D' },
-	{ "find-by-name", 0, 0, 'N' }, /* deprecated in SnapRAID 4.0 */
+	{ "force-nocopy", 0, 0, 'N' },
 	{ "audit-only", 0, 0, 'a' },
 	{ "pre-hash", 0, 0, 'h' },
 	{ "speed-test", 0, 0, 'T' },
@@ -356,6 +357,7 @@ void signal_handler(int signal)
 #define OPERATION_SCRUB 9
 #define OPERATION_STATUS 10
 #define OPERATION_REWRITE 11
+#define OPERATION_NANO 12
 
 int main(int argc, char* argv[])
 {
@@ -386,6 +388,9 @@ int main(int argc, char* argv[])
 	raid_init();
 	crc32c_init();
 
+	/* always different random numbers */
+	srand(time(0));
+
 	/* defaults */
 	config(conf, sizeof(conf), argv[0]);
 	memset(&opt, 0, sizeof(opt));
@@ -509,10 +514,8 @@ int main(int argc, char* argv[])
 			opt.force_device = 1;
 			break;
 		case 'N' :
-			/* LCOV_EXCL_START */
-			fprintf(stderr, "WARNING! Option --find-by-name, -N is deprecated and does nothing!\n");
+			opt.force_nocopy = 1;
 			break;
-			/* LCOV_EXCL_STOP */
 		case 'a' :
 			opt.auditonly = 1;
 			break;
@@ -658,6 +661,8 @@ int main(int argc, char* argv[])
 		operation = OPERATION_STATUS;
 	} else if (strcmp(argv[optind], "test-rewrite") == 0) {
 		operation = OPERATION_REWRITE;
+	} else if (strcmp(argv[optind], "test-nano") == 0) {
+		operation = OPERATION_NANO;
 	} else {
 		/* LCOV_EXCL_START */
 		fprintf(stderr, "Unknown command '%s'\n", argv[optind]);
@@ -749,6 +754,7 @@ int main(int argc, char* argv[])
 	case OPERATION_POOL :
 	case OPERATION_STATUS :
 	case OPERATION_REWRITE :
+	case OPERATION_NANO :
 		/* avoid to check and access parity disks if not needed */
 		opt.skip_parity_access = 1;
 		break;
@@ -909,6 +915,14 @@ int main(int argc, char* argv[])
 
 		state_write(&state);
 
+		memory();
+	} else if (operation == OPERATION_NANO) {
+		state_read(&state);
+
+		state_nano(&state);
+
+		state_write(&state);
+
 		memory();
 	} else if (operation == OPERATION_STATUS) {
 		state_read(&state);

cmdline/state.h

@@ -65,6 +65,7 @@ struct snapraid_option {
 	int force_empty; /**< Forced dangerous operations of synching disks now empty. */
 	int force_uuid; /**< Forced dangerous operations of synching disks with uuid changed. */
 	int force_device; /**< Forced dangerous operations of using disks with save device id. */
+	int force_nocopy; /**< Force dangerous operations of synching files without using copy detection. */
 	int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */
 	int expect_recoverable; /**< Expect presence of recoverable error in checking. */
 	int skip_device; /**< Skip devices matching checks. */
@@ -166,6 +167,11 @@ void state_write(struct snapraid_state* state);
  */
 void state_scan(struct snapraid_state* state, int output);
 
+/**
+ * Sets the nanosecond timestamp of all files that have a zero value.
+ */
+void state_nano(struct snapraid_state* state);
+
 /**
  * Syncs the parity data.
  */

cmdline/sync.c

@@ -654,7 +654,15 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari
 						fprintf(stdlog, "error:%u:%s:%s: Unexpected data change\n", i, disk->name, file->sub);
 						fprintf(stderr, "Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block));
 						fprintf(stderr, "WARNING! Unexpected data modification of a file without parity!\n");
-						fprintf(stderr, "Try removing the file from the array and rerun the 'sync' command!\n");
+
+						if (file_flag_has(file, FILE_IS_COPY)) {
+							fprintf(stderr, "This file was detected as a copy of another file with the same name, size,\n");
+							fprintf(stderr, "and timestamp, but the file data isn't matching the assumed copy.\n");
+							fprintf(stderr, "If this is a false positive, and the files are expected to be different,\n");
+							fprintf(stderr, "you can 'sync' anyway using 'snapraid --force-nocopy sync'\n");
+						} else {
+							fprintf(stderr, "Try removing the file from the array and rerun the 'sync' command!\n");
+						}
 
 						++error;
 

cmdline/util.c

@@ -1641,8 +1641,6 @@ void randomize(void* void_ptr, unsigned size)
 	unsigned char* ptr = void_ptr;
 	unsigned i;
 
-	srand(time(0));
-
 	for(i=0;i<size;++i)
 		ptr[i] = rand();
 }

snapraid.1

@@ -29,6 +29,10 @@ snapraid [\-c, \-\-conf CONFIG]
 	[\-U, \-\-force\-uuid] [\-D, \-\-force\-device]
 .PD 0
 .PP
+.PD
+	[\-N, \-\-force\-nocopy]
+.PD 0
+.PP
 .PD
 	[\-v, \-\-verbose] [\-l, \-\-log FILE]
 .PD 0
@@ -816,6 +820,17 @@ But it could happen that you want to temporarily restore a lost
 disk in the free space left in an already used disk. and this
 option allows you to continue anyway.
 .TP
+.B \-N, \-\-force\-nocopy
+In the \[dq]sync\[dq] command, disables the copy detection optimization.
+Withouth this option SnapRAID assumes that files with the
+same name, size, and nanosecond timestamp are copies with the
+same data.
+This allows to identify copied files from one dir to another,
+and to reuse the already computed hash information to detect
+silent errors in the copy.
+This behaviour, in some rare cases, may result in false positives,
+and this option allow to resolve them.
+.TP
 .B \-l, \-\-log FILE
 Write a detailed log of errors found in check and fix.
 This log contains the exact specification of which block of

snapraid.d

@@ -9,6 +9,7 @@ Synopsis
 	:	[-p, --percentage PERC] [-o, --older-than DAYS]
 	:	[-Z, --force-zero] [-E, --force-empty]
 	:	[-U, --force-uuid] [-D, --force-device]
+	:	[-N, --force-nocopy]
 	:	[-v, --verbose] [-l, --log FILE]
 	:	[-s, --start BLKSTART] [-t, --count BLKCOUNT]
 	:	status|sync|scrub|fix|check|list|diff|dup|pool|rehash
@@ -582,6 +583,17 @@ Options
 		disk in the free space left in an already used disk. and this
 		option allows you to continue anyway.
 
+	-N, --force-nocopy
+		In the "sync" command, disables the copy detection optimization.
+		Withouth this option SnapRAID assumes that files with the
+		same name, size, and nanosecond timestamp are copies with the
+		same data.
+		This allows to identify copied files from one dir to another,
+		and to reuse the already computed hash information to detect
+		silent errors in the copy.
+		This behaviour, in some rare cases, may result in false positives,
+		and this option allow to resolve them.
+
 	-l, --log FILE
 		Write a detailed log of errors found in check and fix.
 		This log contains the exact specification of which block of

snapraid.txt

@@ -13,6 +13,7 @@ snapraid [-c, --conf CONFIG]
 	[-p, --percentage PERC] [-o, --older-than DAYS]
 	[-Z, --force-zero] [-E, --force-empty]
 	[-U, --force-uuid] [-D, --force-device]
+	[-N, --force-nocopy]
 	[-v, --verbose] [-l, --log FILE]
 	[-s, --start BLKSTART] [-t, --count BLKCOUNT]
 	status|sync|scrub|fix|check|list|diff|dup|pool|rehash
@@ -627,6 +628,17 @@ SnapRAID provides the following options:
         disk in the free space left in an already used disk. and this
         option allows you to continue anyway.
 
+    -N, --force-nocopy
+        In the "sync" command, disables the copy detection optimization.
+        Withouth this option SnapRAID assumes that files with the
+        same name, size, and nanosecond timestamp are copies with the
+        same data.
+        This allows to identify copied files from one dir to another,
+        and to reuse the already computed hash information to detect
+        silent errors in the copy.
+        This behaviour, in some rare cases, may result in false positives,
+        and this option allow to resolve them.
+
     -l, --log FILE
         Write a detailed log of errors found in check and fix.
         This log contains the exact specification of which block of