Merge branch 'b4'

HISTORY

@@ -14,8 +14,7 @@ SnapRAID HISTORY
 4.1 2013/9 (BETA)
 =================
  * If the underline filesystem doesn't support the FIEMAP command,
-   automatically fallback to use FIBMAP and then the inode number for sorting
-   files.
+   automatically fallback to use FIBMAP for sorting files.
  * Fixes the import of content files from previous version of SnapRAID that
    are the result of an incomplete sync.
  * Added a new -C, --gen-conf option to generate a dummy configuration

elem.c

@@ -524,6 +524,7 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
 	disk->device = dev;
 	disk->first_free_block = 0;
 	disk->has_not_persistent_inodes = 0;
+	disk->has_not_reliable_physical = 0;
 	disk->mapping = -1;
 	tommy_list_init(&disk->filelist);
 	tommy_list_init(&disk->deletedlist);

elem.h

@@ -274,6 +274,7 @@ struct snapraid_disk {
 	uint64_t device; /**< Device identifier. */
 	block_off_t first_free_block; /**< First free searching block. */
 	int has_not_persistent_inodes; /**< If the underline filesystem has dynamic inodes. */
+	int has_not_reliable_physical; /**< If the physical offset of files has duplicates. */
 	int mapping; /**< Index in the mapping vector. Used only as buffer when writing the content file. */
 
 	/**<

scan.c

@@ -277,14 +277,16 @@ static void scan_file_insert(struct snapraid_state* state, struct snapraid_disk*
 static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state, int output, struct snapraid_disk* disk, const char* sub, const struct stat* st, uint64_t physical)
 {
 	struct snapraid_file* file;
+	uint64_t inode;
 
 	file = 0;
 
 	/*
-	 * If the disk has persistent inodes, try a search by inode.
+	 * If the disk has persistent inodes, try a search by inode,
+	 * to detect moved files.
 	 *
 	 * For persistent inodes we mean inodes that keep their values when the filesystem
-	 * is unmounted and remounted. This don't always happen. 
+	 * is unmounted and remounted. This don't always happen.
 	 *
 	 * Cases found are:
 	 * - Linux FUSE with exFAT driver from https://code.google.com/p/exfat/.
@@ -293,15 +295,18 @@ static void scan_file(struct snapraid_scan* scan, struct snapraid_state* state,
 	 *   making inode collision more easy (exFAT by design supports 10ms precision).
 	 * - Linux VFAT kernel (3.2) driver. Inodes are fully reassigned at every mount.
 	 *
-	 * In such cases, to avoid possible random collisions, it's better to just avoid
-	 * to use inodes at all.
+	 * In such cases, to avoid possible random collisions, it's better to disable the moved
+	 * file recognizition.
+	 *
+	 * We do this implicitely removing all the inode before searching for files.
+	 * This ensure that no file is found with an old inode, but at the same time,
+	 * it allows to find new files with the same inode, and to identify them as hardlinks.
 	 */
-	if (!disk->has_not_persistent_inodes) {
-		/* first try finding by inode */
-		uint64_t inode = st->st_ino;
-		file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
-	}
 
+	inode = st->st_ino;
+	file = tommy_hashdyn_search(&disk->inodeset, file_inode_compare_to_arg, &inode, file_inode_hash(inode));
+
+	/* identify moved files searching by inode */
 	if (file) {
 		/* check if the file is not changed */
 		if (file->size == st->st_size
@@ -816,6 +821,9 @@ void state_scan(struct snapraid_state* state, int output)
 		tommy_node* node;
 		int ret;
 		int has_persistent_inode;
+		unsigned phy_count;
+		unsigned phy_dup;
+		uint64_t phy_last;
 
 		scan = malloc_nofail(sizeof(struct snapraid_scan));
 		scan->count_equal = 0;
@@ -838,10 +846,29 @@ void state_scan(struct snapraid_state* state, int output)
 			fprintf(stderr, "Error accessing disk '%s' to get filesystem info. %s.\n", disk->dir, strerror(errno));
 			exit(EXIT_FAILURE);
 		}
-		/* mark the disk if required */
 		if (!has_persistent_inode) {
-			fprintf(stderr, "WARNING! Inodes use disabled for disk '%s' because they are not persistent.\n", disk->dir);
+			/* mask the disk */
 			disk->has_not_persistent_inodes = 1;
+
+			/* removes all the inodes from the inode collection */
+			/* if they are not persistent, all of them could be changed now */
+			/* and we don't want to find false matching ones */
+			/* see scan_file() for more details */
+			node = disk->filelist;
+			while (node) {
+				struct snapraid_file* file = node->data;
+
+				node = node->next;
+
+				/* remove from the inode set */
+				tommy_hashdyn_remove_existing(&disk->inodeset, &file->nodeset);
+
+				/* clear the inode */
+				file->inode = 0;
+
+				/* mark as missing inode */
+				file_flag_set(file, FILE_IS_WITHOUT_INODE);
+			}
 		}
 
 		scan_dir(scan, state, output, disk, disk->dir, "");
@@ -939,9 +966,20 @@ void state_scan(struct snapraid_state* state, int output)
 		/* insert all the new files, we insert them only after the deletion */
 		/* to reuse the just freed space */
 		node = scan->file_insert_list;
+		phy_count = 0;
+		phy_dup = 0;
+		phy_last = -1;
 		while (node) {
 			struct snapraid_file* file = node->data;
 
+			/* if the file is not empty, count duplicate physical offsets */
+			if (state->opt.force_order == SORT_PHYSICAL && file->size != 0) {
+				if (phy_count > 0 && file->physical == phy_last)
+					++phy_dup;
+				phy_last = file->physical;
+				++phy_count;
+			}
+
 			/* next node */
 			node = node->next;
 
@@ -949,6 +987,12 @@ void state_scan(struct snapraid_state* state, int output)
 			scan_file_insert(state, disk, file);
 		}
 
+		/* mark the disk without reliable physical offset if it has duplicates */
+		/* here it should never happen because we already sorted out hardlinks */
+		if (state->opt.force_order == SORT_PHYSICAL && phy_dup > 0) {
+			disk->has_not_reliable_physical = 1;
+		}
+
 		/* insert all the new links */
 		node = scan->link_insert_list;
 		while (node) {
@@ -976,21 +1020,21 @@ void state_scan(struct snapraid_state* state, int output)
 
 	/* checks for disks where all the previously existing files where removed */
 	if (!state->opt.force_empty) {
-		int has_empty = 0;
+		int done = 0;
 		for(i=state->disklist,j=scanlist;i!=0;i=i->next,j=j->next) {
 			struct snapraid_disk* disk = i->data;
 			struct snapraid_scan* scan = j->data;
 
 			if (scan->count_equal == 0 && scan->count_move == 0 && scan->count_restore == 0 && (scan->count_remove != 0 || scan->count_change != 0)) {
-				if (!has_empty) {
-					has_empty = 1;
+				if (!done) {
+					done = 1;
 					fprintf(stderr, "All the files previously present in disk '%s' at dir '%s'", disk->name, disk->dir);
 				} else {
 					fprintf(stderr, ", disk '%s' at dir '%s'", disk->name, disk->dir);
 				}
 			}
 		}
-		if (has_empty) {
+		if (done) {
 			fprintf(stderr, " are now missing or rewritten!\n");
 			fprintf(stderr, "This could happen when deleting all the files from a disk,\n");
 			fprintf(stderr, "and restoring them with a program not setting correctly the timestamps.\n");
@@ -1001,6 +1045,46 @@ void state_scan(struct snapraid_state* state, int output)
 		}
 	}
 
+	/* checks for disks without the physical offset support */
+	if (state->opt.force_order == SORT_PHYSICAL) {
+		int done = 0;
+		for(i=state->disklist;i!=0;i=i->next) {
+			struct snapraid_disk* disk = i->data;
+
+			if (disk->has_not_reliable_physical) {
+				if (!done) {
+					done = 1;
+					fprintf(stderr, "WARNING! Physical offsets not supported for disk '%s'", disk->name);
+				} else {
+					fprintf(stderr, ", '%s", disk->name);
+				}
+			}
+		}
+		if (done) {
+			fprintf(stderr, ". Performance won't be optimal.\n");
+		}
+	}
+
+	/* Check for disks without persisten inodes */
+	{
+		int done = 0;
+		for(i=state->disklist;i!=0;i=i->next) {
+			struct snapraid_disk* disk = i->data;
+
+			if (disk->has_not_persistent_inodes) {
+				if (!done) {
+					done = 1;
+					fprintf(stderr, "WARNING! Inodes are not persistent for disk '%s'", disk->name);
+				} else {
+					fprintf(stderr, ", '%s", disk->name);
+				}
+			}
+		}
+		if (done) {
+			fprintf(stderr, ". Move operations won't be optimized.\n");
+		}
+	}
+
 	if (state->opt.verbose || output) {
 		struct snapraid_scan total;
 

scrub.c

@@ -101,10 +101,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 				continue;
 			}
 
-			/* if we reached the count limit */
-			if (countmax >= countlimit) {
-				/* skip it */
-				continue;
+			/* if the time is less than the limit, always include */
+			/* otherwise, check if we reaced the count max */
+			if (blocktime == timelimit) {
+				/* if we reached the count limit */
+				if (countmax >= countlimit) {
+					/* skip it */
+					continue;
+				}
 			}
 		}
 
@@ -154,10 +158,14 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par
 				continue;
 			}
 
-			/* if we reached the count limit */
-			if (recountmax >= countlimit) {
-				/* skip it */
-				continue;
+			/* if the time is less than the limit, always include */
+			/* otherwise, check if we reaced the count max */
+			if (blocktime == timelimit) {
+				/* if we reached the count limit */
+				if (recountmax >= countlimit) {
+					/* skip it */
+					continue;
+				}
 			}
 		}
 
@@ -439,7 +447,7 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
 		recentlimit = now;
 	} else if (percentage != -1 || olderthan != -1) {
 		if (percentage != -1)
-			countlimit = blockmax * percentage / 100;
+			countlimit = (blockmax * percentage + 99) / 100;
 		else
 			countlimit = blockmax;
 		if (olderthan != -1)
@@ -488,8 +496,9 @@ int state_scrub(struct snapraid_state* state, int percentage, int olderthan)
 		timelimit = info_get_time(infomap[countlimit - 1]);
 
 		/* don't scrub too recent blocks */
-		if (timelimit > recentlimit)
+		if (timelimit > recentlimit) {
 			timelimit = recentlimit;
+		}
 	} else {
 		/* we don't have a time limit based on quota */
 		timelimit = recentlimit;

snapraid.1

@@ -791,14 +791,14 @@ memory to run SnapRAID.
 As a rule of thumb, with 4 GiB or more memory use the default 256,
 with 2 GiB use 512, and with 1 GiB use 1024.
 .PP
-In more details SnapRAID requires about TS*24/BS bytes
+In more details SnapRAID requires about TS*28/BS bytes
 of RAM memory to run. Where TS is the total size in bytes of
 your disk array, and BS is the block size in bytes.
 .PP
-For example with 6 disk of 2 TiB and a block size of 256 KiB
+For example with 4 disk of 3 TiB and a block size of 256 KiB
 (1 KiB = 1024 Bytes) you have:
 .PP
-RAM = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB
+RAM = (4 * 3 * 2^40) * 28 / (256 * 2^10) = 1.4 GiB
 .PD 0
 .PP
 .PD

snapraid.d

@@ -623,14 +623,14 @@ Configuration
 	As a rule of thumb, with 4 GiB or more memory use the default 256,
 	with 2 GiB use 512, and with 1 GiB use 1024.
 
-	In more details SnapRAID requires about TS*24/BS bytes
+	In more details SnapRAID requires about TS*28/BS bytes
 	of RAM memory to run. Where TS is the total size in bytes of
 	your disk array, and BS is the block size in bytes.
 
-	For example with 6 disk of 2 TiB and a block size of 256 KiB
+	For example with 4 disk of 3 TiB and a block size of 256 KiB
 	(1 KiB = 1024 Bytes) you have:
 
-	:RAM = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB
+	:RAM = (4 * 3 * 2^40) * 28 / (256 * 2^10) = 1.4 GiB
 
 	You could instead decrease this value if you have a lot of
 	small files in the disk array. For each file, even if of few

snapraid.txt

@@ -683,14 +683,14 @@ memory to run SnapRAID.
 As a rule of thumb, with 4 GiB or more memory use the default 256,
 with 2 GiB use 512, and with 1 GiB use 1024.
 
-In more details SnapRAID requires about TS*24/BS bytes
+In more details SnapRAID requires about TS*28/BS bytes
 of RAM memory to run. Where TS is the total size in bytes of
 your disk array, and BS is the block size in bytes.
 
-For example with 6 disk of 2 TiB and a block size of 256 KiB
+For example with 4 disk of 3 TiB and a block size of 256 KiB
 (1 KiB = 1024 Bytes) you have:
 
-RAM = (6 * 2 * 2^40) * 24 / (256 * 2^10) = 1.1 GiB
+RAM = (4 * 3 * 2^40) * 28 / (256 * 2^10) = 1.4 GiB
 
 You could instead decrease this value if you have a lot of
 small files in the disk array. For each file, even if of few

state.c

@@ -640,7 +640,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
 	/* no previous hash by default */
 	state->prevhash = HASH_UNDEFINED;
 
-	/* intentionally not set the prevhashseed */
+	/* intentionally not set the prevhashseed, if used valgrind will warn about it */
 
 	if (state->opt.gui) {
 		tommy_node* i;
@@ -1718,7 +1718,7 @@ static void state_write_text(struct snapraid_state* state, STREAM* f)
 		if (info_has_rehash) {
 			if (state->prevhash == HASH_MURMUR3) {
 				sputsl("prevchecksum murmur3", f);
-			} else if (state->hash == HASH_SPOOKY2) {
+			} else if (state->prevhash == HASH_SPOOKY2) {
 				sputsl("prevchecksum spooky2", f);
 			} else {
 				fprintf(stderr, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
@@ -2783,7 +2783,7 @@ static void state_write_binary(struct snapraid_state* state, STREAM* f)
 			sputc('C', f);
 			if (state->prevhash == HASH_MURMUR3) {
 				sputc('u', f);
-			} else if (state->hash == HASH_SPOOKY2) {
+			} else if (state->prevhash == HASH_SPOOKY2) {
 				sputc('k', f);
 			} else {
 				fprintf(stderr, "Unexpected prevhash when writing the content file '%s'.\n", serrorfile(f));
@@ -3185,7 +3185,8 @@ void state_read(struct snapraid_state* state)
 
 	/* previous hash, start with an undefined value */
 	state->prevhash = HASH_UNDEFINED;
-	memset(state->prevhashseed, 0, HASH_SIZE);
+
+	/* intentionally not set the prevhashseed, if used valgrind will warn about it */
 
 	/* get the first char to detect the file type */
 	c = sgetc(f);

status.c

@@ -249,7 +249,11 @@ int state_status(struct snapraid_state* state)
 	if (rehash) {
 		printf("You have a rehash in progress at %u%%.\n", (count - rehash) * 100 / count);
 	} else {
-		printf("No rehash is in progress.\n");
+		if (state->besthash != state->hash) {
+			printf("No rehash is in progress, but for optimal performance one is recommended.\n");
+		} else {
+			printf("No rehash is in progress or needed.\n");
+		}
 	}
 
 	if (bad) {

unix.c

@@ -141,14 +141,14 @@ int filephy(const char* path, struct stat* st, uint64_t* physical)
 	}
 
 	/* first try with FIEMAP */
-	/* if works for ext2, ext3, ext4, xfs */
+	/* if works for ext2, ext3, ext4, xfs, btrfs */
 	memset(&fm, 0, sizeof(fm));
 	fm.fiemap.fm_start = 0;
 	fm.fiemap.fm_length = ~0ULL;
 	fm.fiemap.fm_flags = FIEMAP_FLAG_SYNC; /* required to ensure that just created files report a valid address and not 0 */
 	fm.fiemap.fm_extent_count = 1; /* we are interested only at the first block */
 
-	if (ioctl(f, FS_IOC_FIEMAP, &fm) == 0) {
+	if (ioctl(f, FS_IOC_FIEMAP, &fm) != -1) {
 		*physical = fm.fiemap.fm_extents[0].fe_physical;
 		if (close(f) == -1)
 			return -1;
@@ -167,7 +167,7 @@ int filephy(const char* path, struct stat* st, uint64_t* physical)
 	/* it works for jfs, reiserfs, ntfs-3g */
 	/* in exfat it always returns 0, that it's anyway better than the fake inodes */
 	blknum = 0; /* first block */
-	if (ioctl(f, FIBMAP, &blknum) == 0) {
+	if (ioctl(f, FIBMAP, &blknum) != -1) {
 		*physical = blknum;
 		if (close(f) == -1)
 			return -1;
@@ -181,10 +181,12 @@ int filephy(const char* path, struct stat* st, uint64_t* physical)
 	if (close(f) == -1)
 		return -1;
 #else
-	/* In a generic Unix uses the inode as fake physical address */
-	*physical = st->st_ino;
+	/* In a generic Unix use a dummy value for all the files */
+	/* We don't want to risk to use the inode without knowing if it really improves performance */
+	*physical = 0;
 
 	(void)path; /* not used here */
+	(void)st;
 #endif
 
 	return 0;