diff --git a/HISTORY b/HISTORY index 16247850..ccefee41 100644 --- a/HISTORY +++ b/HISTORY @@ -4,6 +4,14 @@ SnapRAID HISTORY 6.2 2014/5 ========== * Fixed the regression test when run as root. + * Added a new euristic to detect file copies. Now a file is assumed + to be a copy if name, size and nanosecond timestamp are matching, + but if the nanosecond part of the timestamp is 0, we require + the full path matching and not only the name. + * Added the -N, --force-nocopy option to disable completely + the copy detection. SnapRAID also suggest to use this option + in the error message of a data mismatch if caused by the + copy detection. 6.1 2014/4 ========== diff --git a/Makefile.am b/Makefile.am index a49b86d3..ab868a48 100644 --- a/Makefile.am +++ b/Makefile.am @@ -30,6 +30,7 @@ snapraid_SOURCES = \ cmdline/pool.c \ cmdline/parity.c \ cmdline/handle.c \ + cmdline/nano.c \ cmdline/fnmatch.c \ cmdline/selftest.c \ cmdline/speed.c \ @@ -184,6 +185,12 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) -a check + echo --- Dry + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-dry + echo --- Nano + touch -d "2001-02-01 12:34:56" bench/disk1/a/a* + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-nano echo --- Check the --gen-conf command $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --gen-conf bench/content echo --- Filter @@ -237,8 +244,6 @@ endif $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) fix -l test.log $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(PAR1) check $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check - echo --- Dry - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) test-dry echo --- Scrub some times $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-force-scrub 100 scrub $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) status diff --git a/cmdline/elem.c b/cmdline/elem.c index 52a455ec..42f1f5de 100644 --- a/cmdline/elem.c +++ b/cmdline/elem.c @@ -428,6 +428,20 @@ void file_copy(struct snapraid_file* src_file, struct snapraid_file* dst_file) /* LCOV_EXCL_STOP */ } + if (src_file->mtime_sec != dst_file->mtime_sec) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Internal inconsistency in copy file with different mtime_sec\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + + if (src_file->mtime_nsec != dst_file->mtime_nsec) { + /* LCOV_EXCL_START */ + fprintf(stderr, "Internal inconsistency in copy file with different mtime_nsec\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } + for(i=0;iblockmax;++i) { /* set a block with hash computed but without parity */ block_state_set(&dst_file->blockvec[i], BLOCK_STATE_REP); @@ -435,6 +449,8 @@ void file_copy(struct snapraid_file* src_file, struct snapraid_file* dst_file) /* copy the hash */ memcpy(dst_file->blockvec[i].hash, src_file->blockvec[i].hash, HASH_SIZE); } + + file_flag_set(dst_file, FILE_IS_COPY); } const char* file_name(const struct snapraid_file* file) @@ -538,6 +554,17 @@ int file_namestamp_compare(const void* void_a, const void* void_b) return file_stamp_compare(void_a, void_b); } +int file_pathstamp_compare(const void* void_a, const void* void_b) +{ + int ret; + + ret = file_path_compare(void_a, void_b); + if (ret != 0) + return ret; + + return file_stamp_compare(void_a, void_b); +} + struct snapraid_link* link_alloc(const char* sub, const char* linkto, unsigned link_flag) { struct snapraid_link* link; diff --git a/cmdline/elem.h b/cmdline/elem.h index 8827bcbd..073d3de9 100644 --- a/cmdline/elem.h +++ b/cmdline/elem.h @@ -203,6 +203,12 @@ struct snapraid_block { */ #define FILE_IS_FINISHED 0x40 +/** + * If the file hash was obtained from a file copy + * identified by the same name, size and stamp. + */ +#define FILE_IS_COPY 0x80 + #define FILE_IS_HARDLINK 0x100 /**< If it's an hardlink. */ #define FILE_IS_SYMLINK 0x200 /**< If it's a file symlink. */ #define FILE_IS_SYMDIR 0x400 /**< If it's a dir symlink for Windows. Not yet supported. */ @@ -696,6 +702,7 @@ int file_path_compare_to_arg(const void* void_arg, const void* void_data); int file_name_compare(const void* void_a, const void* void_b); int file_stamp_compare(const void* void_a, const void* void_b); int file_namestamp_compare(const void* void_a, const void* void_b); +int file_pathstamp_compare(const void* void_a, const void* void_b); /** * Computes the hash of a file path. diff --git a/cmdline/handle.c b/cmdline/handle.c index 029fa3a2..7dba07be 100644 --- a/cmdline/handle.c +++ b/cmdline/handle.c @@ -393,7 +393,7 @@ int file_utime(struct snapraid_file* file, int f) ret = futimesat(f, 0, tv); #else -#error No function available to set file timestamps +#error No function available to set file timestamps with sub-second precision #endif if (ret != 0) { diff --git a/cmdline/scan.c b/cmdline/scan.c index 57e6c2fc..23b94bb7 100644 --- a/cmdline/scan.c +++ b/cmdline/scan.c @@ -519,10 +519,10 @@ static void scan_file(struct snapraid_scan* scan, int output, const char* sub, c /* mark as present */ file_flag_set(file, FILE_IS_PRESENT); - /* update the nano seconds mtime only if different */ + /* update the nanoseconds mtime only if different */ /* to avoid unneeded updates */ if (file->mtime_nsec == STAT_NSEC_INVALID - && STAT_NSEC(st) != STAT_NSEC_INVALID + && STAT_NSEC(st) != file->mtime_nsec ) { file->mtime_nsec = STAT_NSEC(st); @@ -746,32 +746,40 @@ static void scan_file(struct snapraid_scan* scan, int output, const char* sub, c /* mark it as present */ file_flag_set(file, FILE_IS_PRESENT); - /* now search for a file with the same name and timestamp in all the disks */ - for(i=state->disklist;i!=0;i=i->next) { - struct snapraid_disk* other_disk = i->data; - struct snapraid_file* other_file; + /* if copy detection is enabled */ + if (!state->opt.force_nocopy) { tommy_uint32_t hash = file_stamp_hash(file->size, file->mtime_sec, file->mtime_nsec); - /* search for a file with the same name and stamp */ - other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash); + /* search for a file with the same name and stamp in all the disks */ + for(i=state->disklist;i!=0;i=i->next) { + struct snapraid_disk* other_disk = i->data; + struct snapraid_file* other_file; - /* if found, and it's a fully hashed file */ - if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) { - /* assume that the file is a copy, and reuse the hash */ - file_copy(other_file, file); + /* if the nanosecond part of the time stamp is valid, search */ + /* for name and stamp, otherwise for path and stamp */ + if (file->mtime_nsec != 0 && file->mtime_nsec != STAT_NSEC_INVALID) + other_file = tommy_hashdyn_search(&other_disk->stampset, file_namestamp_compare, file, hash); + else + other_file = tommy_hashdyn_search(&other_disk->stampset, file_pathstamp_compare, file, hash); - /* revert old counter and use the copy one */ - add_insert = 0; - add_change = 0; - ++scan->count_copy; + /* if found, and it's a fully hashed file */ + if (other_file && file_is_full_hashed_and_stable(scan->state, other_file)) { + /* assume that the file is a copy, and reuse the hash */ + file_copy(other_file, file); - fprintf(stdlog, "scan:copy:%s:%s:%s:%s\n", other_disk->name, other_file->sub, disk->name, file->sub); - if (output) { - printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub); + /* revert old counter and use the copy one */ + add_insert = 0; + add_change = 0; + ++scan->count_copy; + + fprintf(stdlog, "scan:copy:%s:%s:%s:%s\n", other_disk->name, other_file->sub, disk->name, file->sub); + if (output) { + printf("copy %s%s -> %s%s\n", other_disk->dir, other_file->sub, disk->dir, file->sub); + } + + /* no need to continue the search */ + break; } - - /* no need to continue the search */ - break; } } diff --git a/cmdline/snapraid.c b/cmdline/snapraid.c index 8adf7768..1496e94c 100644 --- a/cmdline/snapraid.c +++ b/cmdline/snapraid.c @@ -64,6 +64,7 @@ void usage(void) printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force synching of disks that get empty\n"); printf(" " SWITCH_GETOPT_LONG("-U, --force-uuid ", "-U") " Force commands on disks with uuid changed\n"); printf(" " SWITCH_GETOPT_LONG("-D, --force-device ", "-D") " Force commands on disks with same device id\n"); + printf(" " SWITCH_GETOPT_LONG("-N, --force-nocopy ", "-N") " Force commands disabling the copy detection\n"); printf(" " SWITCH_GETOPT_LONG("-s, --start BLKSTART ", "-s") " Start from the specified block number\n"); printf(" " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT ", "-t") " Count of block to process\n"); printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n"); @@ -249,7 +250,7 @@ struct option long_options[] = { { "force-empty", 0, 0, 'E' }, { "force-uuid", 0, 0, 'U' }, { "force-device", 0, 0, 'D' }, - { "find-by-name", 0, 0, 'N' }, /* deprecated in SnapRAID 4.0 */ + { "force-nocopy", 0, 0, 'N' }, { "audit-only", 0, 0, 'a' }, { "pre-hash", 0, 0, 'h' }, { "speed-test", 0, 0, 'T' }, @@ -356,6 +357,7 @@ void signal_handler(int signal) #define OPERATION_SCRUB 9 #define OPERATION_STATUS 10 #define OPERATION_REWRITE 11 +#define OPERATION_NANO 12 int main(int argc, char* argv[]) { @@ -386,6 +388,9 @@ int main(int argc, char* argv[]) raid_init(); crc32c_init(); + /* always different random numbers */ + srand(time(0)); + /* defaults */ config(conf, sizeof(conf), argv[0]); memset(&opt, 0, sizeof(opt)); @@ -509,10 +514,8 @@ int main(int argc, char* argv[]) opt.force_device = 1; break; case 'N' : - /* LCOV_EXCL_START */ - fprintf(stderr, "WARNING! Option --find-by-name, -N is deprecated and does nothing!\n"); + opt.force_nocopy = 1; break; - /* LCOV_EXCL_STOP */ case 'a' : opt.auditonly = 1; break; @@ -658,6 +661,8 @@ int main(int argc, char* argv[]) operation = OPERATION_STATUS; } else if (strcmp(argv[optind], "test-rewrite") == 0) { operation = OPERATION_REWRITE; + } else if (strcmp(argv[optind], "test-nano") == 0) { + operation = OPERATION_NANO; } else { /* LCOV_EXCL_START */ fprintf(stderr, "Unknown command '%s'\n", argv[optind]); @@ -749,6 +754,7 @@ int main(int argc, char* argv[]) case OPERATION_POOL : case OPERATION_STATUS : case OPERATION_REWRITE : + case OPERATION_NANO : /* avoid to check and access parity disks if not needed */ opt.skip_parity_access = 1; break; @@ -909,6 +915,14 @@ int main(int argc, char* argv[]) state_write(&state); + memory(); + } else if (operation == OPERATION_NANO) { + state_read(&state); + + state_nano(&state); + + state_write(&state); + memory(); } else if (operation == OPERATION_STATUS) { state_read(&state); diff --git a/cmdline/state.h b/cmdline/state.h index 8117a9ab..853e8ea7 100644 --- a/cmdline/state.h +++ b/cmdline/state.h @@ -65,6 +65,7 @@ struct snapraid_option { int force_empty; /**< Forced dangerous operations of synching disks now empty. */ int force_uuid; /**< Forced dangerous operations of synching disks with uuid changed. */ int force_device; /**< Forced dangerous operations of using disks with save device id. */ + int force_nocopy; /**< Force dangerous operations of synching files without using copy detection. */ int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */ int expect_recoverable; /**< Expect presence of recoverable error in checking. */ int skip_device; /**< Skip devices matching checks. */ @@ -166,6 +167,11 @@ void state_write(struct snapraid_state* state); */ void state_scan(struct snapraid_state* state, int output); +/** + * Sets the nanosecond timestamp of all files that have a zero value. + */ +void state_nano(struct snapraid_state* state); + /** * Syncs the parity data. */ diff --git a/cmdline/sync.c b/cmdline/sync.c index 7c960dc2..ef2748a0 100644 --- a/cmdline/sync.c +++ b/cmdline/sync.c @@ -654,7 +654,15 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari fprintf(stdlog, "error:%u:%s:%s: Unexpected data change\n", i, disk->name, file->sub); fprintf(stderr, "Data change at file '%s' at position '%u'\n", handle[j].path, block_file_pos(block)); fprintf(stderr, "WARNING! Unexpected data modification of a file without parity!\n"); - fprintf(stderr, "Try removing the file from the array and rerun the 'sync' command!\n"); + + if (file_flag_has(file, FILE_IS_COPY)) { + fprintf(stderr, "This file was detected as a copy of another file with the same name, size,\n"); + fprintf(stderr, "and timestamp, but the file data isn't matching the assumed copy.\n"); + fprintf(stderr, "If this is a false positive, and the files are expected to be different,\n"); + fprintf(stderr, "you can 'sync' anyway using 'snapraid --force-nocopy sync'\n"); + } else { + fprintf(stderr, "Try removing the file from the array and rerun the 'sync' command!\n"); + } ++error; diff --git a/cmdline/util.c b/cmdline/util.c index f3ee821d..cde900a3 100644 --- a/cmdline/util.c +++ b/cmdline/util.c @@ -1641,8 +1641,6 @@ void randomize(void* void_ptr, unsigned size) unsigned char* ptr = void_ptr; unsigned i; - srand(time(0)); - for(i=0;i