Changes the import command to use a new timestamp recognition.

This avoid the need of prehashing all the data, that could
be very slow.

Makefile.am

@@ -35,6 +35,7 @@ snapraid_SOURCES = \
 	cmdline/selftest.c \
 	cmdline/speed.c \
 	cmdline/import.c \
+	cmdline/search.c \
 	cmdline/mingw.c \
 	cmdline/unix.c
 
@@ -74,6 +75,7 @@ noinst_HEADERS = \
 	cmdline/spooky2test.c \
 	cmdline/fnmatch.h \
 	cmdline/import.h \
+	cmdline/search.h \
 	cmdline/mingw.h \
 	cmdline/unix.h
 
@@ -508,7 +510,14 @@ endif
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) --test-force-scrub-even scrub
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) status
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) check
-	echo --- Delete files from three disks and check/fix with import in PAR2
+	echo --- Delete files from three disks and check/fix with import by data in PAR2
+	rm -r bench/disk1/a
+	rm -r bench/disk2/a
+	mv bench/disk3/a bench/a
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-import-content bench/a -c $(PAR2) fix -l test.log
+	rm -r bench/a
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -c $(CONF) sync
+	echo --- Delete files from three disks and check/fix with import by timestamp in PAR2
 	rm -r bench/disk1/a
 	rm -r bench/disk2/a
 	mv bench/disk3/a bench/a

cmdline/check.c

@@ -20,6 +20,7 @@
 #include "util.h"
 #include "elem.h"
 #include "import.h"
+#include "search.h"
 #include "state.h"
 #include "parity.h"
 #include "handle.h"
@@ -369,7 +370,8 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 			/* if we have the hash for it */
 			if ((block_state == BLOCK_STATE_BLK || block_state == BLOCK_STATE_REP)
 			        /* try to fetch the block using the known hash */
-				&& state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0
+				&& (state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0
+					|| state_search_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0)
 			) {
 				/* we already have corrected it! */
 				fprintf(stdlog, "hash_import: Fixed entry %u\n", j);
@@ -498,7 +500,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 				/* try to fetch the old block using the old hash for CHG and DELETED blocks */
 			} else if ((block_state == BLOCK_STATE_CHG || block_state == BLOCK_STATE_DELETED)
 				&& hash_is_real(failed[j].block->hash)
-				&& state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
+				&& state_import_fetch(state, rehash, failed[j].block, buffer[failed[j].index]) == 0) {
 
 				/* note that from now the buffer is definitively lost */
 				/* we can do this only because it's the last retry of recovering */

cmdline/import.c

@@ -148,11 +148,12 @@ void import_file_free(struct snapraid_import_file* file)
 	free(file);
 }
 
-int state_import_fetch(struct snapraid_state* state, int rehash, const unsigned char* hash, unsigned char* buffer)
+int state_import_fetch(struct snapraid_state* state, int rehash, struct snapraid_block* missing_block, unsigned char* buffer)
 {
 	struct snapraid_import_block* block;
 	int ret;
 	int f;
+	const unsigned char* hash = missing_block->hash;
 	unsigned block_size = state->block_size;
 	unsigned read_size;
 	unsigned char buffer_hash[HASH_SIZE];

cmdline/import.h

@@ -63,7 +63,7 @@ void import_file_free(struct snapraid_import_file* file);
  * Fetch a block from the specified hash.
  * Return ==0 if the block is found, and copied into buffer.
  */
-int state_import_fetch(struct snapraid_state* state, int prevhash, const unsigned char* hash, unsigned char* buffer);
+int state_import_fetch(struct snapraid_state* state, int prevhash, struct snapraid_block* missing_block, unsigned char* buffer);
 
 /**
  * Import files from the specified directory.

cmdline/snapraid.c

@@ -21,6 +21,7 @@
 #include "util.h"
 #include "elem.h"
 #include "import.h"
+#include "search.h"
 #include "state.h"
 #include "raid/raid.h"
 
@@ -234,6 +235,7 @@ void config(char* conf, size_t conf_size, const char* argv0)
 #define OPT_TEST_EXPECT_FAILURE 277
 #define OPT_TEST_RUN 278
 #define OPT_TEST_FORCE_SCAN_WINFIND 279
+#define OPT_TEST_IMPORT_CONTENT 280
 
 #if HAVE_GETOPT_LONG
 struct option long_options[] = {
@@ -332,6 +334,9 @@ struct option long_options[] = {
 	/* Use the FindFirst/Next approach in Windows to list files */
 	{ "test-force-scan-winfind", 0, 0, OPT_TEST_FORCE_SCAN_WINFIND },
 
+	/* Alternative import working by data */
+	{ "test-import-content", 1, 0, OPT_TEST_IMPORT_CONTENT },
+
 	{ 0, 0, 0, 0 }
 };
 #endif
@@ -383,7 +388,8 @@ int main(int argc, char* argv[])
 	int olderthan;
 	char* e;
 	const char* command;
-	const char* import;
+	const char* import_timestamp;
+	const char* import_content;
 	const char* log;
 	int lock;
 	const char* gen_conf;
@@ -409,7 +415,8 @@ int main(int argc, char* argv[])
 	filter_error = 0;
 	percentage = -1;
 	olderthan = -1;
-	import = 0;
+	import_timestamp = 0;
+	import_content = 0;
 	log = 0;
 	lock = 0;
 	gen_conf = 0;
@@ -491,13 +498,22 @@ int main(int argc, char* argv[])
 			}
 			break;
 		case 'i' :
-			if (import) {
+			if (import_timestamp) {
 				/* LCOV_EXCL_START */
-				fprintf(stderr, "Import directory '%s' already specified as '%s'\n", optarg, import);
+				fprintf(stderr, "Import directory '%s' already specified as '%s'\n", optarg, import_timestamp);
 				exit(EXIT_FAILURE);
 				/* LCOV_EXCL_STOP */
 			}
-			import = optarg;
+			import_timestamp = optarg;
+			break;
+		case OPT_TEST_IMPORT_CONTENT :
+			if (import_content) {
+				/* LCOV_EXCL_START */
+				fprintf(stderr, "Import directory '%s' already specified as '%s'\n", optarg, import_content);
+				exit(EXIT_FAILURE);
+				/* LCOV_EXCL_STOP */
+			}
+			import_content = optarg;
 			break;
 		case 'l' :
 			if (log) {
@@ -790,7 +806,7 @@ int main(int argc, char* argv[])
 	case OPERATION_FIX :
 		break;
 	default :
-		if (import != 0) {
+		if (import_timestamp != 0 || import_content != 0) {
 			/* LCOV_EXCL_START */
 			fprintf(stderr, "You cannot import with the '%s' command\n", command);
 			exit(EXIT_FAILURE);
@@ -1011,8 +1027,10 @@ int main(int argc, char* argv[])
 	} else {
 		state_read(&state);
 
-		if (import != 0)
-			state_import(&state, import);
+		if (import_timestamp != 0)
+			state_search(&state, import_timestamp);
+		if (import_content != 0)
+			state_import(&state, import_content);
 
 		/* apply the command line filter */
 		state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing, filter_error);

cmdline/state.c

@@ -19,6 +19,7 @@
 
 #include "elem.h"
 #include "import.h"
+#include "search.h"
 #include "state.h"
 #include "util.h"
 #include "parity.h"
@@ -106,6 +107,7 @@ void state_init(struct snapraid_state* state)
 	tommy_list_init(&state->importlist);
 	tommy_hashdyn_init(&state->importset);
 	tommy_hashdyn_init(&state->previmportset);
+	tommy_hashdyn_init(&state->searchset);
 	tommy_arrayblkof_init(&state->infoarr, sizeof(snapraid_info));
 }
 
@@ -116,8 +118,10 @@ void state_done(struct snapraid_state* state)
 	tommy_list_foreach(&state->contentlist, (tommy_foreach_func*)content_free);
 	tommy_list_foreach(&state->filterlist, (tommy_foreach_func*)filter_free);
 	tommy_list_foreach(&state->importlist, (tommy_foreach_func*)import_file_free);
+	tommy_hashdyn_foreach(&state->searchset, (tommy_foreach_func*)search_file_free);
 	tommy_hashdyn_done(&state->importset);
 	tommy_hashdyn_done(&state->previmportset);
+	tommy_hashdyn_done(&state->searchset);
 	tommy_arrayblkof_done(&state->infoarr);
 }
 

cmdline/state.h

@@ -127,6 +127,7 @@ struct snapraid_state {
 	tommy_list importlist; /**< List of import file. */
 	tommy_hashdyn importset; /**< Hashtable by hash of all the import blocks. */
 	tommy_hashdyn previmportset; /**< Hashtable by prevhash of all the import blocks. Valid only if we are in a rehash state. */
+	tommy_hashdyn searchset; /**< Hashtable by timestamp of all the search files. */
 	tommy_arrayblkof infoarr; /**< Block information array. */
 
 	/**