Reviews HISTORY.

Conflicts:
	check.c

HISTORY

@@ -1,8 +1,15 @@
 SnapRAID HISTORY
 ================
 
-5.2 2013/12 (BETA)
+5.3 2014/3
-==================
+=================
+ * Don't warn about UUID changed if it's for an empty disk.
+ * Fixed the number of blocks that scrub has to process when
+   selecting a high percentage of the array.
+ * Removed duplicate recovery attempts in synched state.
+
+5.2 2013/12
+===========
  * If a disk changes UUID, automatically disable the inode
    recognition, because this is likely a new filesystem with
    all the inodes reassigned, and we don't want to risk a false
@@ -15,7 +22,8 @@ SnapRAID HISTORY
  * Doesn't count empty dirs as files in the diff counters.
  * Added a new "share" configuration option to allow to share
    in the network the pool directory also in Windows.
- * Fixed build problem in OpenBSD due the old assembler.
+ * Fixed build problems in OpenBSD due the old assembler.
+ * Fixed build problems in platforms different than x86.
 
 5.1 2013/12
 ===========

check.c

@@ -160,6 +160,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
 	/* if too many error, we don't have any strategy */
 	n = state->level;
 	if (failed_count > n) {
+		fprintf(stdlog, "strategy_error:%u: No strategy to recover from %u failures with %u parity\n", pos, failed_count, n);
 		return -1;
 	}
 
@@ -260,7 +261,9 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
 	/* return the number of failed attempts, or -1 if no strategy */
 	if (error)
 		return error;
-	else
+
+	fprintf(stdlog, "strategy_error:%u: No strategy to recover from %u failures with %u parity %s hash\n",
+		pos, failed_count, n, has_hash ? "with" : "without");
 	return -1;
 }
 
@@ -271,6 +274,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 	unsigned j;
 	int n;
 	int something_to_recover;
+	int something_unsynched;
 
 	error = 0;
 
@@ -386,6 +390,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 	/* and we are not interested to recover DELETED ones. */
 	n = 0;
 	something_to_recover = 0; /* keep track if there is at least one block to fix */
+	something_unsynched = 0; /* keep track if we have some unsynched info to process */
 	for(j=0;j<failed_count;++j) {
 		unsigned block_state = block_state_get(failed[j].block);
 
@@ -398,6 +403,7 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 			/* because the parity is computed with old content, and not with the new one. */
 			/* Note that this recovering is done just to make possible to recover any other BLK one, */
 			/* we are not really interested in DELETED and CHG (old version) ones. */
+			something_unsynched = 1;
 
 			/* try to fetch the old block using the old hash */
 			if (state_import_fetch(state, rehash, failed[j].block->hash, buffer[failed[j].index]) == 0) {
@@ -444,7 +450,8 @@ static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsign
 	}
 
 	/* if nothing to fix, we just don't try */
-	if (something_to_recover) {
+	/* if nothing unsynched we also don't retry, because it's the same try as before */
+	if (something_to_recover && something_unsynched) {
 		ret = repair_step(state, rehash, pos, diskmax, failed, failed_map, n, buffer, buffer_recov, buffer_zero);
 		if (ret == 0) {
 			/* we alreay marked as outdated NEW and CHG blocks, we don't need to do it again */
@@ -622,6 +629,8 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 				failed[failed_count].block = block;
 				failed[failed_count].handle = 0;
 				++failed_count;
+
+				fprintf(stdlog, "block_delete:%u:%s: Delete block\n", i, handle[j].disk->name);
 				continue;
 			}
 
@@ -737,6 +746,8 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 				failed[failed_count].block = block;
 				failed[failed_count].handle = &handle[j];
 				++failed_count;
+
+				fprintf(stdlog, "block_change:%u:%s:%s: Change block at position %u\n", i, handle[j].disk->name, block_file_get(block)->sub, block_file_pos(block));
 				continue;
 			}
 
@@ -806,7 +817,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 				/* print a list of all the errors in files */
 				for(j=0;j<failed_count;++j) {
 					if (failed[j].is_bad)
-						fprintf(stdlog, "unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
+						fprintf(stdlog, "unrecoverable:%u:%s:%s: Unrecoverable synched error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
 				}
 
 				/* keep track of damaged files */
@@ -824,7 +835,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix,
 				for(j=0;j<failed_count;++j) {
 					if (failed[j].is_bad && failed[j].is_outofdate) {
 						++partial_recover_error;
-						fprintf(stdlog, "unrecoverable:%u:%s:%s: Unrecoverable error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
+						fprintf(stdlog, "unrecoverable:%u:%s:%s: Unrecoverable unsynched error at position %u\n", i, failed[j].handle->disk->name, block_file_get(failed[j].block)->sub, block_file_pos(failed[j].block));
 					}
 				}
 				if (partial_recover_error != 0) {

configure.ac

@@ -1,6 +1,6 @@
 dnl Process this file with autoconf to produce a configure script.
 AC_PREREQ([2.65])
-AC_INIT([snapraid], [5.2], [], [], [http://snapraid.sourceforge.net])
+AC_INIT([snapraid], [5.3], [], [], [http://snapraid.sourceforge.net])
 AM_INIT_AUTOMAKE([foreign no-dependencies -Wall])
 AC_CONFIG_SRCDIR([snapraid.c])
 AC_CONFIG_HEADER([config.h])

elem.c

@@ -533,6 +533,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
 	disk->has_volatile_inodes = 0;
 	disk->has_unreliable_physical = 0;
 	disk->has_different_uuid = 0;
+	disk->had_empty_uuid = 0;
 	disk->mapping = -1;
 	tommy_list_init(&disk->filelist);
 	tommy_list_init(&disk->deletedlist);

elem.h

@@ -284,6 +284,7 @@ struct snapraid_disk {
 	int has_volatile_inodes; /**< If the underline filesystem has not persistent inodes. */
 	int has_unreliable_physical; /**< If the physical offset of files has duplicates. */
 	int has_different_uuid; /**< If the disk has a different UUID, meaning that it is not the same filesystem. */
+	int had_empty_uuid; /**< If the disk had an empty UUID, meaning that it's a new disk. */
 	int mapping; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
 
 	/**<

fnmatch.c

@@ -9,12 +9,7 @@
    This library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
+   Library General Public License for more details.  */
-
-   You should have received a copy of the GNU Library General Public
-   License along with this library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
 
 #if HAVE_CONFIG_H
 # include <config.h>

fnmatch.h

@@ -9,12 +9,7 @@
    The GNU C Library is distributed in the hope that it will be useful,
    but WITHOUT ANY WARRANTY; without even the implied warranty of
    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
-   Library General Public License for more details.
+   Library General Public License for more details.  */
-
-   You should have received a copy of the GNU Library General Public
-   License along with the GNU C Library; see the file COPYING.LIB.  If not,
-   write to the Free Software Foundation, Inc., 59 Temple Place - Suite 330,
-   Boston, MA 02111-1307, USA.  */
 
 #ifndef	_FNMATCH_H
 #define	_FNMATCH_H	1

scan.c

@@ -1297,7 +1297,11 @@ void state_scan(struct snapraid_state* state, int output)
 		for(i=state->disklist;i!=0;i=i->next) {
 			struct snapraid_disk* disk = i->data;
 
-			if (disk->has_different_uuid) {
+			/* don't print the message if the UUID changed because before */
+			/* if was no set. */
+			/* this is the normal condition for an empty disk because it */
+			/* isn't stored */
+			if (disk->has_different_uuid && !disk->had_empty_uuid) {
 				if (!done) {
 					done = 1;
 					fprintf(stderr, "WARNING! UUID is changed for disks: '%s'", disk->name);

scrub.c

@@ -471,7 +471,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
 		recentlimit = now;
 	} else if (percentage != -1 || olderthan != -1) {
 		if (percentage != -1)
-			countlimit = (blockmax * percentage + 99) / 100;
+			countlimit = (blockmax * (uint64_t)percentage + 99) / 100;
 		else
 			countlimit = blockmax;
 		if (olderthan != -1)

selftest.c

@@ -184,7 +184,7 @@ static void recovtest(unsigned mode, unsigned diskmax, unsigned block_size)
 					}
 
 					/* recover */
-					map[r-1][0](r, d, p, buffer, diskmax, zero, block_size);
+					map[r-1][j](r, d, p, buffer, diskmax, zero, block_size);
 
 					/* check */
 					for(i=0;i<r;++i) {
@@ -260,6 +260,7 @@ static void gentest(unsigned mode, unsigned diskmax, unsigned block_size)
 		map[mac++] = raid_par2_sse2ext;
 #endif
 	}
+#endif
 
 	if (mode == RAID_MODE_CAUCHY) {
 		map[mac++] = raid_par3_int8;
@@ -267,6 +268,7 @@ static void gentest(unsigned mode, unsigned diskmax, unsigned block_size)
 		map[mac++] = raid_par5_int8;
 		map[mac++] = raid_par6_int8;
 
+#if defined(__i386__) || defined(__x86_64__)
 		if (cpu_has_ssse3()) {
 			map[mac++] = raid_par3_ssse3;
 			map[mac++] = raid_par4_ssse3;
@@ -281,9 +283,9 @@ static void gentest(unsigned mode, unsigned diskmax, unsigned block_size)
 		}
 #endif
 	} else {
-#if defined(__i386__) || defined(__x86_64__)
 		map[mac++] = raid_parz_int32;
 		map[mac++] = raid_parz_int64;
+#if defined(__i386__) || defined(__x86_64__)
 		if (cpu_has_sse2()) {
 			map[mac++] = raid_parz_sse2;
 #if defined(__x86_64__)

snapraid.c

@@ -104,6 +104,7 @@ void memory(void)
 #define OPT_TEST_FORCE_SCRUB_EVEN 272
 #define OPT_TEST_FORCE_CONTENT_WRITE 273
 #define OPT_TEST_FORCE_CONTENT_TEXT 274
+#define OPT_TEST_SKIP_CONTENT_CHECK 275
 
 #if HAVE_GETOPT_LONG
 struct option long_options[] = {
@@ -185,6 +186,9 @@ struct option long_options[] = {
 	/* Force the use of text content file . */
 	{ "test-force-content-text", 0, 0, OPT_TEST_FORCE_CONTENT_TEXT },
 
+	/* Relax the checks done at the content file */
+	{ "test-skip-content-check", 0, 0, OPT_TEST_SKIP_CONTENT_CHECK },
+
 	{ 0, 0, 0, 0 }
 };
 #endif
@@ -399,6 +403,9 @@ int main(int argc, char* argv[])
 		case OPT_TEST_SKIP_DEVICE :
 			opt.skip_device = 1;
 			break;
+		case OPT_TEST_SKIP_CONTENT_CHECK :
+			opt.skip_content_check = 1;
+			break;
 		case OPT_TEST_FORCE_MURMUR3 :
 			opt.force_murmur3 = 1;
 			break;

speed.c

@@ -385,6 +385,7 @@ void speed(void)
 	printf("%8s", "");
 	printf("%8s", "");
 
+#if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_sse2()) {
 		printf("%8s", "");
 
@@ -392,6 +393,7 @@ void speed(void)
 		printf("%8s", "");
 #endif
 	}
+#endif
 
 #if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_ssse3()) {
@@ -429,6 +431,7 @@ void speed(void)
 	printf("%8s", "");
 	printf("%8s", "");
 
+#if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_sse2()) {
 		printf("%8s", "");
 
@@ -436,6 +439,7 @@ void speed(void)
 		printf("%8s", "");
 #endif
 	}
+#endif
 
 #if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_ssse3()) {
@@ -473,6 +477,7 @@ void speed(void)
 	printf("%8s", "");
 	printf("%8s", "");
 
+#if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_sse2()) {
 		printf("%8s", "");
 
@@ -480,6 +485,7 @@ void speed(void)
 		printf("%8s", "");
 #endif
 	}
+#endif
 
 #if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_ssse3()) {
@@ -517,6 +523,7 @@ void speed(void)
 	printf("%8s", "");
 	printf("%8s", "");
 
+#if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_sse2()) {
 		printf("%8s", "");
 
@@ -524,6 +531,7 @@ void speed(void)
 		printf("%8s", "");
 #endif
 	}
+#endif
 
 #if defined(__i386__) || defined(__x86_64__)
 	if (cpu_has_ssse3()) {
@@ -552,7 +560,9 @@ void speed(void)
 	printf("%8s", "");
 	printf("%8s", "best");
 	printf("%8s", "int8");
+#if defined(__i386__) || defined(__x86_64__)
 	printf("%8s", "ssse3");
+#endif
 	printf("\n");
 
 	printf("%8s", "rec1");

state.c

@@ -864,6 +864,8 @@ static void state_map(struct snapraid_state* state)
 				/* count the number of uuid change */
 				++uuid_mismatch;
 				fprintf(stderr, "UUID change for disk '%s' from '%s' to '%s'\n", disk->name, map->uuid, uuid);
+			} else {
+				disk->had_empty_uuid = 1;
 			}
 
 			/* update the uuid in the mapping, */
@@ -2155,7 +2157,7 @@ static void decoding_error(const char* path, STREAM* f)
 		fprintf(stderr, "This content file is damaged! Use an alternate copy.\n");
 		exit(EXIT_FAILURE);
 	} else {
-		fprintf(stderr, "The file CRC is correct! This seems a SnapRAID bug!\n");
+		fprintf(stderr, "The file CRC is correct!\n");
 	}
 }
 
@@ -2226,6 +2228,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			ret = sgetb32(f, &mapping);
 			if (ret < 0 || mapping >= mapping_max) {
 				decoding_error(path, f);
+				fprintf(stderr, "Internal inconsistency in mapping index!\n");
 				exit(EXIT_FAILURE);
 			}
 			disk = tommy_array_get(&disk_mapping, mapping);
@@ -2239,7 +2242,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			/* check for impossible file size to avoid to crash for a too big allocation */
 			if (v_size / state->block_size > blockmax) {
 				decoding_error(path, f);
-				fprintf(stderr, "Internal inconsistency in file size!\n");
+				fprintf(stderr, "Internal inconsistency in file size too big!\n");
 				exit(EXIT_FAILURE);
 			}
 
@@ -2313,10 +2316,16 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 				}
 
 				if (v_pos + v_count > blockmax) {
-					decoding_error(path, f);
+					if (state->opt.skip_content_check) {
 						fprintf(stderr, "Internal inconsistency in block size!\n");
+						fprintf(stderr, "Overriding from %u to %u...\n", blockmax, v_pos + v_count);
+						blockmax = v_pos + v_count;
+					} else {
+						fprintf(stderr, "Internal inconsistency in block size!\n");
+						decoding_error(path, f);
 						exit(EXIT_FAILURE);
 					}
+				}
 
 				/* grow the array */
 				tommy_arrayblk_grow(&disk->blockarr, v_pos + v_count);
@@ -2416,10 +2425,16 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 				}
 
 				if (v_pos + v_count > blockmax) {
+					if (state->opt.skip_content_check) {
+						fprintf(stderr, "Internal inconsistency in info size!\n");
+						fprintf(stderr, "Overriding from %u to %u...\n", blockmax, v_pos + v_count);
+						blockmax = v_pos + v_count;
+					} else {
 						decoding_error(path, f);
 						fprintf(stderr, "Internal inconsistency in info size!\n");
 						exit(EXIT_FAILURE);
 					}
+				}
 
 				ret = sgetb32(f, &flag);
 				if (ret < 0) {
@@ -2488,6 +2503,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			ret = sgetb32(f, &mapping);
 			if (ret < 0 || mapping >= mapping_max) {
 				decoding_error(path, f);
+				fprintf(stderr, "Internal inconsistency in mapping index!\n");
 				exit(EXIT_FAILURE);
 			}
 			disk = tommy_array_get(&disk_mapping, mapping);
@@ -2503,10 +2519,16 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 				}
 
 				if (v_pos + v_count > blockmax) {
+					if (state->opt.skip_content_check) {
+						fprintf(stderr, "Internal inconsistency in hole size!\n");
+						fprintf(stderr, "Overriding from %u to %u...\n", blockmax, v_pos + v_count);
+						blockmax = v_pos + v_count;
+					} else {
 						decoding_error(path, f);
 						fprintf(stderr, "Internal inconsistency in hole size!\n");
 						exit(EXIT_FAILURE);
 					}
+				}
 
 				/* get the subcommand */
 				c = sgetc(f);
@@ -2577,6 +2599,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			ret = sgetb32(f, &mapping);
 			if (ret < 0 || mapping >= mapping_max) {
 				decoding_error(path, f);
+				fprintf(stderr, "Internal inconsistency in mapping index!\n");
 				exit(EXIT_FAILURE);
 			}
 			disk = tommy_array_get(&disk_mapping, mapping);
@@ -2618,6 +2641,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			ret = sgetb32(f, &mapping);
 			if (ret < 0 || mapping >= mapping_max) {
 				decoding_error(path, f);
+				fprintf(stderr, "Internal inconsistency in mapping index!\n");
 				exit(EXIT_FAILURE);
 			}
 			disk = tommy_array_get(&disk_mapping, mapping);
@@ -2658,6 +2682,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			ret = sgetb32(f, &mapping);
 			if (ret < 0 || mapping >= mapping_max) {
 				decoding_error(path, f);
+				fprintf(stderr, "Internal inconsistency in mapping index!\n");
 				exit(EXIT_FAILURE);
 			}
 			disk = tommy_array_get(&disk_mapping, mapping);
@@ -2788,6 +2813,7 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 			if (!disk) {
 				decoding_error(path, f);
 				fprintf(stderr, "Disk named '%s' not present in the configuration file!\n", buffer);
+				fprintf(stderr, "If you have removed it from the configuration file, please restore it\n");
 				exit(EXIT_FAILURE);
 			}
 
@@ -2841,8 +2867,13 @@ static void state_read_binary(struct snapraid_state* state, const char* path, ST
 	/* check that the stored parity size matches the loaded state */
 	if (blockmax != parity_size(state)) {
 		fprintf(stderr, "Internal inconsistency in parity size in '%s' at offset %"PRIi64"\n", path, stell(f));
+		if (state->opt.skip_content_check) {
+			fprintf(stderr, "Overriding from %u to %u...\n", blockmax, parity_size(state));
+			blockmax = parity_size(state);
+		} else {
 			exit(EXIT_FAILURE);
 		}
+	}
 
 	/* set the required parity size */
 	state->loaded_paritymax = paritymax;

state.h

@@ -71,6 +71,7 @@ struct snapraid_option {
 	int skip_sequential; /**< Skip sequential hint. */
 	int skip_lock; /**< Skip the lock file protection. */
 	int skip_self; /**< Skip the self-test. */
+	int skip_content_check; /**< Relax some content file checks. */
 	int kill_after_sync; /**< Kill the process after sync without saving the final state. */
 	int force_murmur3; /**< Force Murmur3 choice. */
 	int force_spooky2; /**< Force Spooky2 choice. */