diff --git a/HISTORY b/HISTORY
index 6cfb543b..6e175b37 100644
--- a/HISTORY
+++ b/HISTORY
@@ -3,6 +3,7 @@ SnapRAID HISTORY
1.8 2011/12
===========
+ * Added a new "dup" command to find all the duplicate files.
* The example configuration files now put the content files in the data
disks instead than in the parity disks.
diff --git a/Makefile.am b/Makefile.am
index bfafbfa5..a59916d6 100644
--- a/Makefile.am
+++ b/Makefile.am
@@ -11,6 +11,7 @@ snapraid_SOURCES = \
sync.c \
check.c \
dry.c \
+ dup.c \
parity.c \
handle.c \
fnmatch.c \
@@ -91,17 +92,20 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -h
echo --- As empty, sync and check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf diff
+ $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Create some files, sync raid6 and check
$(TESTENV) ./mktest$(EXEEXT) generate 1 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf diff
+ $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid6.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
echo --- Create some more files, sync raid5 and check
$(TESTENV) ./mktest$(EXEEXT) generate 2 3 $(CHECKCOUNT) $(CHECKSIZE)
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf diff
+ $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf dup
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -v -c $(srcdir)/test-raid5.conf sync
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf check
echo --- Extend raid5 to raid6 with fix and check
@@ -112,7 +116,7 @@ check-local:
echo --- Dry
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf dry
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf dry
- echo --- Make a hole
+ echo --- Make a hole in the disk array
mv test/disk2 test/disk2.old
mkdir test/disk2
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self --force-empty -c $(srcdir)/test-raid6.conf sync
@@ -125,7 +129,7 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf fix 2>> test.log
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf check
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6-hole.conf dry
- echo --- Fill the hole
+ echo --- Fill the hole with a new disk
rm -r test/disk2
mv test/disk2.old test/disk2
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
diff --git a/TODO b/TODO
index 85a59fbb..b1a8c352 100644
--- a/TODO
+++ b/TODO
@@ -26,8 +26,6 @@ that they are files and not directories.
* Put the content files in the data disks in the default configuration file.
-* Easy way to find duplicates using the computed hash.
-
* Easy way to find files using a specified block.
* Adds a checksum for the content file. When reading if the file is invalid,
diff --git a/dup.c b/dup.c
new file mode 100644
index 00000000..f7ee405c
--- /dev/null
+++ b/dup.c
@@ -0,0 +1,144 @@
+/*
+ * Copyright (C) 2011 Andrea Mazzoleni
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program. If not, see .
+ */
+
+#include "portable.h"
+
+#include "util.h"
+#include "elem.h"
+#include "state.h"
+#include "parity.h"
+#include "handle.h"
+
+/****************************************************************************/
+/* dup */
+
+struct snapraid_hash {
+ struct snapraid_disk* disk; /**< Disk. */
+ struct snapraid_file* file; /**< File. */
+ unsigned char hash[HASH_SIZE]; /**< Hash of the whole file. */
+
+ /* nodes for data structures */
+ tommy_node nodelist;
+ tommy_hashdyn_node nodeset;
+};
+
+struct snapraid_hash* hash_alloc(struct snapraid_disk* disk, struct snapraid_file* file)
+{
+ struct snapraid_hash* hash;
+ block_off_t i;
+ unsigned char* buf;
+
+ hash = malloc_nofail(sizeof(struct snapraid_hash));
+ hash->disk = disk;
+ hash->file = file;
+
+ buf = malloc_nofail(file->blockmax * HASH_SIZE);
+
+ /* set the back pointer */
+ for(i=0;iblockmax;++i) {
+ memcpy(buf + i * HASH_SIZE, file->blockvec[i].hash, HASH_SIZE);
+
+ if (!block_flag_has(&file->blockvec[i], BLOCK_HAS_HASH)) {
+ free(buf);
+ return 0;
+ }
+ }
+
+ memhash(HASH_MURMUR3, hash->hash, buf, file->blockmax * HASH_SIZE);
+
+ free(buf);
+
+ return hash;
+}
+
+static inline tommy_uint32_t hash_hash(struct snapraid_hash* hash)
+{
+ return tommy_hash_u32(0, hash->hash, HASH_SIZE);
+}
+
+void hash_free(struct snapraid_hash* hash)
+{
+ free(hash);
+}
+
+int hash_compare(const void* void_arg, const void* void_data)
+{
+ const char* arg = void_arg;
+ const struct snapraid_hash* hash = void_data;
+ return memcmp(arg, hash->hash, HASH_SIZE);
+}
+
+void state_dup(struct snapraid_state* state)
+{
+ tommy_hashdyn hashset;
+ tommy_list hashlist;
+ tommy_node* i;
+ unsigned count;
+ data_off_t size;
+
+ tommy_hashdyn_init(&hashset);
+ tommy_list_init(&hashlist);
+
+ count = 0;
+ size = 0;
+
+ /* for each disk */
+ for(i=state->disklist;i!=0;i=i->next) {
+ tommy_node* j;
+ struct snapraid_disk* disk = i->data;
+
+ /* for each file */
+ for(j=disk->filelist;j!=0;j=j->next) {
+ struct snapraid_file* file = j->data;
+ struct snapraid_hash* hash = hash_alloc(disk, file);
+ tommy_hash_t hash32;
+
+ /* if no hash, skip it */
+ if (!hash)
+ continue;
+
+ hash32 = hash_hash(hash);
+
+ struct snapraid_hash* dup = tommy_hashdyn_search(&hashset, hash_compare, hash->hash, hash32);
+ if (dup) {
+ ++count;
+ size += dup->file->size;
+ if (state->gui) {
+ fprintf(stderr, "dup:%s:%s:%s:%s:%"PRIu64": dup\n", disk->name, file->sub, dup->disk->name, dup->file->sub, dup->file->size);
+ } else {
+ printf("Duplicate '%s%s'\n", disk->dir, file->sub);
+ printf("of '%s%s'\n", dup->disk->dir, dup->file->sub);
+ }
+ hash_free(hash);
+ } else {
+ tommy_hashdyn_insert(&hashset, &hash->nodeset, hash, hash32);
+ tommy_list_insert_tail(&hashlist, &hash->nodelist, hash);
+ }
+ }
+ }
+
+ tommy_list_foreach(&hashlist, (tommy_foreach_func*)hash_free);
+ tommy_hashdyn_done(&hashset);
+
+ if (!state->gui) {
+ if (count)
+ printf("%u duplicates, for %"PRIu64" MiB.\n", count, size / (1024*1024));
+ else
+ printf("No duplicate\n");
+ }
+}
+
diff --git a/elem.c b/elem.c
index 5016f01c..b6240360 100644
--- a/elem.c
+++ b/elem.c
@@ -353,20 +353,10 @@ struct snapraid_disk* disk_alloc(const char* name, const char* dir, uint64_t dev
void disk_free(struct snapraid_disk* disk)
{
- tommy_node* node = disk->filelist;
- while (node) {
- struct snapraid_file* file = node->data;
- node = node->next;
- file_free(file);
- }
+ tommy_list_foreach(&disk->filelist, (tommy_foreach_func*)file_free);
tommy_hashdyn_done(&disk->inodeset);
tommy_hashdyn_done(&disk->pathset);
- node = disk->linklist;
- while (node) {
- struct snapraid_link* link = node->data;
- node = node->next;
- link_free(link);
- }
+ tommy_list_foreach(&disk->linklist, (tommy_foreach_func*)link_free);
tommy_hashdyn_done(&disk->linkset);
tommy_array_done(&disk->blockarr);
free(disk);
diff --git a/snapraid.1 b/snapraid.1
index 2615bceb..23d0caf6 100644
--- a/snapraid.1
+++ b/snapraid.1
@@ -22,7 +22,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hyf, \(hy\(hyfilter PATTERN]
.PD 0
.PP
.PD
- sync|diff|check|fix
+ sync|diff|dup|check|fix
.PD 0
.PP
.PD
@@ -310,6 +310,11 @@ Lists all the files modified from the last \(a"sync\(a" command that
have to recompute their redundancy data.
.PP
Nothing is modified.
+.SS dup
+Lists all the duplicate files. Two files are assumed equal if their hashes
+are matching. The effective data is not read.
+.PP
+Nothing is modified.
.SS check
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
diff --git a/snapraid.c b/snapraid.c
index 2f78fd59..60acb4ba 100644
--- a/snapraid.c
+++ b/snapraid.c
@@ -49,6 +49,7 @@ void usage(void)
printf("Commands:\n");
printf(" sync Syncronize the state of the array of disks\n");
printf(" diff Show the changes that needs to be syncronized\n");
+ printf(" dup Find duplicate files\n");
printf(" check Check the array of disks\n");
printf(" fix Fix the array of disks\n");
printf("\n");
@@ -113,6 +114,7 @@ void signal_handler(int signal)
#define OPERATION_CHECK 2
#define OPERATION_FIX 3
#define OPERATION_DRY 4
+#define OPERATION_DUP 5
int main(int argc, char* argv[])
{
@@ -246,6 +248,8 @@ int main(int argc, char* argv[])
operation = OPERATION_FIX;
} else if (strcmp(argv[optind], "dry") == 0) {
operation = OPERATION_DRY;
+ } else if (strcmp(argv[optind], "dup") == 0) {
+ operation = OPERATION_DUP;
} else {
fprintf(stderr, "Unknown command '%s'\n", argv[optind]);
exit(EXIT_FAILURE);
@@ -312,6 +316,12 @@ int main(int argc, char* argv[])
signal(SIGINT, &signal_handler);
state_dry(&state, blockstart, blockcount);
+ } else if (operation == OPERATION_DUP) {
+ state_read(&state);
+
+ state_filter(&state, &filterlist);
+
+ state_dup(&state);
} else {
state_read(&state);
diff --git a/snapraid.d b/snapraid.d
index 260dca77..5cc4aa5e 100644
--- a/snapraid.d
+++ b/snapraid.d
@@ -7,7 +7,7 @@ Synopsis
: [-Z, --force-zero] [-E, --force-empty]
: [-s, --start BLKSTART] [-t, --count BLKCOUNT]
: [-v, --verbose]
- : sync|diff|check|fix
+ : sync|diff|dup|check|fix
:snapraid [-V, --version] [-h, --help]
@@ -192,6 +192,12 @@ Commands
Nothing is modified.
+ dup
+ Lists all the duplicate files. Two files are assumed equal if their hashes
+ are matching. The effective data is not read.
+
+ Nothing is modified.
+
check
Checks all the files and the redundancy data.
All the files are hashed and compared with the snapshot saved
diff --git a/snapraid.txt b/snapraid.txt
index 8e8d2e54..e3d500a1 100644
--- a/snapraid.txt
+++ b/snapraid.txt
@@ -11,7 +11,7 @@ snapraid [-c, --conf CONFIG] [-f, --filter PATTERN]
[-Z, --force-zero] [-E, --force-empty]
[-s, --start BLKSTART] [-t, --count BLKCOUNT]
[-v, --verbose]
- sync|diff|check|fix
+ sync|diff|dup|check|fix
snapraid [-V, --version] [-h, --help]
@@ -212,7 +212,15 @@ have to recompute their redundancy data.
Nothing is modified.
-5.3 check
+5.3 dup
+-------
+
+Lists all the duplicate files. Two files are assumed equal if their hashes
+are matching. The effective data is not read.
+
+Nothing is modified.
+
+5.4 check
---------
Checks all the files and the redundancy data.
@@ -222,7 +230,7 @@ in the previous "sync" command.
Files are identified by path, and checked by content.
Nothing is modified.
-5.4 fix
+5.5 fix
-------
Checks and fix all the files. It's like "check" but it
diff --git a/state.h b/state.h
index 1d620469..69db8861 100644
--- a/state.h
+++ b/state.h
@@ -94,10 +94,15 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount);
/**
- * Read the files.
+ * Dry the files.
*/
void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount);
+/**
+ * Find duplicates.
+ */
+void state_dup(struct snapraid_state* state);
+
/**
* Filter files.
*/