Adds a new "dup" command to find duplicates.

HISTORY

@@ -3,6 +3,7 @@ SnapRAID HISTORY
 
 1.8 2011/12
 ===========
+ * Added a new "dup" command to find all the duplicate files.
  * The example configuration files now put the content files in the data
    disks instead than in the parity disks.
 

Makefile.am

@@ -11,6 +11,7 @@ snapraid_SOURCES = \
 	sync.c \
 	check.c \
 	dry.c \
+	dup.c \
 	parity.c \
 	handle.c \
 	fnmatch.c \
@@ -91,17 +92,20 @@ check-local:
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -h
 	echo --- As empty, sync and check
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf diff
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf dup
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
 	echo --- Create some files, sync raid6 and check
 	$(TESTENV) ./mktest$(EXEEXT) generate 1 3 $(CHECKCOUNT) $(CHECKSIZE)
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
 	echo --- Create some more files, sync raid5 and check
 	$(TESTENV) ./mktest$(EXEEXT) generate 2 3 $(CHECKCOUNT) $(CHECKSIZE)
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf diff
+	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf dup
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf sync
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
 	echo --- Extend raid5 to raid6 with fix and check
@@ -112,7 +116,7 @@ check-local:
 	echo --- Dry
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf dry
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf dry
-	echo --- Make a hole
+	echo --- Make a hole in the disk array
 	mv test/disk2 test/disk2.old
 	mkdir test/disk2
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --force-empty -c $(srcdir)/test-raid6.conf sync
@@ -125,7 +129,7 @@ check-local:
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf fix 2>> test.log
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf check
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf dry
-	echo --- Fill the hole
+	echo --- Fill the hole with a new disk
 	rm -r test/disk2
 	mv test/disk2.old test/disk2
 	$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync

TODO

@@ -26,8 +26,6 @@ that they are files and not directories.
 
 * Put the content files in the data disks in the default configuration file.
 
-* Easy way to find duplicates using the computed hash.
-
 * Easy way to find files using a specified block.
 
 * Adds a checksum for the content file. When reading if the file is invalid,

dup.c

@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include "portable.h"
+
+#include "util.h"
+#include "elem.h"
+#include "state.h"
+#include "parity.h"
+#include "handle.h"
+
+/****************************************************************************/
+/* dup */
+
+struct snapraid_hash {
+	struct snapraid_disk* disk; /**< Disk. */
+	struct snapraid_file* file; /**< File. */
+	unsigned char hash[HASH_SIZE]; /**< Hash of the whole file. */
+
+	/* nodes for data structures */
+	tommy_node nodelist;
+	tommy_hashdyn_node nodeset;
+};
+
+struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_file* file)
+{
+	struct snapraid_hash* hash;
+	block_off_t i;
+	unsigned char* buf;
+
+	hash = malloc_nofail(sizeof(struct snapraid_hash));
+	hash->disk = disk;
+	hash->file = file;
+
+	buf = malloc_nofail(file->blockmax * HASH_SIZE);
+
+	/* set the back pointer */
+	for(i=0;i<file->blockmax;++i) {
+		memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE);
+
+		if (!block_flag_has(&file->blockvec[i], BLOCK_HAS_HASH)) {
+			free(buf);
+			return 0;
+		}
+	}
+
+	memhash(HASH_MURMUR3, hash->hash, buf, file->blockmax * HASH_SIZE);
+
+	free(buf);
+
+	return hash;
+}
+
+static inline tommy_uint32_t hash_hash(struct snapraid_hash* hash)
+{
+	return tommy_hash_u32(0, hash->hash, HASH_SIZE);
+}
+
+void hash_free(struct snapraid_hash* hash)
+{
+	free(hash);
+}
+
+int hash_compare(const void* void_arg, const void* void_data)
+{
+	const char* arg = void_arg;
+	const struct snapraid_hash* hash = void_data;
+	return memcmp(arg, hash->hash, HASH_SIZE);
+}
+
+void state_dup(struct snapraid_state* state)
+{
+	tommy_hashdyn hashset;
+	tommy_list hashlist;
+	tommy_node* i;
+	unsigned count;
+	data_off_t size;
+
+	tommy_hashdyn_init(&hashset);
+	tommy_list_init(&hashlist);
+
+	count = 0;
+	size = 0;
+	
+	/* for each disk */
+	for(i=state->disklist;i!=0;i=i->next) {
+		tommy_node* j;
+		struct snapraid_disk* disk = i->data;
+
+		/* for each file */
+		for(j=disk->filelist;j!=0;j=j->next) {
+			struct snapraid_file* file = j->data;
+			struct snapraid_hash* hash = hash_alloc(disk, file);
+			tommy_hash_t hash32;
+
+			/* if no hash, skip it */
+			if (!hash)
+				continue;
+			
+			hash32 = hash_hash(hash);
+
+			struct snapraid_hash* dup = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash32);
+			if (dup) {
+				++count;
+				size += dup->file->size;
+				if (state->gui) {
+					fprintf(stderr, "dup:%s:%s:%s:%s:%"PRIu64": dup\n", disk->name, file->sub, dup->disk->name, dup->file->sub, dup->file->size);
+				} else {
+					printf("Duplicate '%s%s'\n", disk->dir, file->sub);
+					printf("of '%s%s'\n", dup->disk->dir, dup->file->sub);
+				}
+				hash_free(hash);
+			} else {
+				tommy_hashdyn_insert(&hashset, &hash->nodeset, hash, hash32);
+				tommy_list_insert_tail(&hashlist, &hash->nodelist, hash);
+			}
+		}
+	}
+
+	tommy_list_foreach(&hashlist, (tommy_foreach_func*)hash_free);
+	tommy_hashdyn_done(&hashset);
+
+	if (!state->gui) {
+		if (count)
+			printf("%u duplicates, for %"PRIu64" MiB.\n", count, size / (1024*1024));
+		else
+			printf("No duplicate\n");
+	}
+}
+

elem.c

@@ -353,20 +353,10 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
 
 void disk_free(struct snapraid_disk* disk)
 {
-	tommy_node* node = disk->filelist;
-	while (node) {
-		struct snapraid_file* file = node->data;
-		node = node->next;
-		file_free(file);
-	}
+	tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free);
 	tommy_hashdyn_done(&disk->inodeset);
 	tommy_hashdyn_done(&disk->pathset);
-	node = disk->linklist;
-	while (node) {
-		struct snapraid_link* link = node->data;
-		node = node->next;
-		link_free(link);
-	}
+	tommy_list_foreach(&disk->linklist, (tommy_foreach_func*)link_free);
 	tommy_hashdyn_done(&disk->linkset);
 	tommy_array_done(&disk->blockarr);
 	free(disk);

snapraid.1

@@ -22,7 +22,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hyf, \(hy\(hyfilter PATTERN]
 .PD 0
 .PP
 .PD
-	sync|diff|check|fix
+	sync|diff|dup|check|fix
 .PD 0
 .PP
 .PD
@@ -310,6 +310,11 @@ Lists all the files modified from the last \(a"sync\(a" command that
 have to recompute their redundancy data.
 .PP
 Nothing is modified.
+.SS dup 
+Lists all the duplicate files. Two files are assumed equal if their hashes
+are matching. The effective data is not read.
+.PP
+Nothing is modified.
 .SS check 
 Checks all the files and the redundancy data.
 All the files are hashed and compared with the snapshot saved

snapraid.c

@@ -49,6 +49,7 @@ void usage(void)
 	printf("Commands:\n");
 	printf("  sync   Syncronize the state of the array of disks\n");
 	printf("  diff   Show the changes that needs to be syncronized\n");
+	printf("  dup    Find duplicate files\n");
 	printf("  check  Check the array of disks\n");
 	printf("  fix    Fix the array of disks\n");
 	printf("\n");
@@ -113,6 +114,7 @@ void signal_handler(int signal)
 #define OPERATION_CHECK 2
 #define OPERATION_FIX 3
 #define OPERATION_DRY 4
+#define OPERATION_DUP 5
 
 int main(int argc, char* argv[])
 {
@@ -246,6 +248,8 @@ int main(int argc, char* argv[])
 		operation = OPERATION_FIX;
 	} else  if (strcmp(argv[optind], "dry") == 0) {
 		operation = OPERATION_DRY;
+	} else  if (strcmp(argv[optind], "dup") == 0) {
+		operation = OPERATION_DUP;
 	} else {
 		fprintf(stderr, "Unknown command '%s'\n", argv[optind]);
 		exit(EXIT_FAILURE);
@@ -312,6 +316,12 @@ int main(int argc, char* argv[])
 		signal(SIGINT, &signal_handler);
 
 		state_dry(&state, blockstart, blockcount);
+	} else if (operation == OPERATION_DUP) {
+		state_read(&state);
+
+		state_filter(&state, &filterlist);
+
+		state_dup(&state);
 	} else {
 		state_read(&state);
 

snapraid.d

@@ -7,7 +7,7 @@ Synopsis
 	:	[-Z, --force-zero] [-E, --force-empty]
 	:	[-s, --start BLKSTART] [-t, --count BLKCOUNT]
 	:	[-v, --verbose]
-	:	sync|diff|check|fix
+	:	sync|diff|dup|check|fix
 
 	:snapraid [-V, --version] [-h, --help]
 
@@ -192,6 +192,12 @@ Commands
 
 	Nothing is modified.
 
+  dup
+	Lists all the duplicate files. Two files are assumed equal if their hashes
+	are matching. The effective data is not read.
+
+	Nothing is modified.
+
   check
 	Checks all the files and the redundancy data.
 	All the files are hashed and compared with the snapshot saved

snapraid.txt

@@ -11,7 +11,7 @@ snapraid [-c, --conf CONFIG] [-f, --filter PATTERN]
 	[-Z, --force-zero] [-E, --force-empty]
 	[-s, --start BLKSTART] [-t, --count BLKCOUNT]
 	[-v, --verbose]
-	sync|diff|check|fix
+	sync|diff|dup|check|fix
 
 snapraid [-V, --version] [-h, --help]
 
@@ -212,7 +212,15 @@ have to recompute their redundancy data.
 
 Nothing is modified.
 
-5.3 check
+5.3 dup
+-------
+
+Lists all the duplicate files. Two files are assumed equal if their hashes
+are matching. The effective data is not read.
+
+Nothing is modified.
+
+5.4 check
 ---------
 
 Checks all the files and the redundancy data.
@@ -222,7 +230,7 @@ in the previous "sync" command.
 Files are identified by path, and checked by content.
 Nothing is modified.
 
-5.4 fix
+5.5 fix
 -------
 
 Checks and fix all the files. It's like "check" but it

state.h

@@ -94,10 +94,15 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
 void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount);
 
 /**
- * Read the files.
+ * Dry the files.
  */
 void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount);
 
+/**
+ * Find duplicates.
+ */
+void state_dup(struct snapraid_state* state);
+
 /**
  * Filter files.
  */