diff --git a/HISTORY b/HISTORY index 155aaacb..acb71464 100644 --- a/HISTORY +++ b/HISTORY @@ -4,8 +4,9 @@ SnapRAID HISTORY 3.2 2013/6 ========== * Better selection of the RAID6 implementation for different CPUs. - * Adds the new Spooky hash. It's faster in 64 bit architectures. - * Adds RAID5/RAID6 mmx and sse2 implementations with unrolling by 4. + * Added the new Spooky hash. It's faster in 64 bit architectures. + To convert you can use the new 'rehash' command. + * Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4. They are a little faster than the previous unroll by 2. 3.1 2013/5 diff --git a/Makefile.am b/Makefile.am index 49fcbea6..33c216d1 100644 --- a/Makefile.am +++ b/Makefile.am @@ -11,6 +11,7 @@ snapraid_SOURCES = \ sync.c \ check.c \ dry.c \ + rehash.c \ dup.c \ pool.c \ parity.c \ @@ -95,10 +96,10 @@ check-local: mkdir test/pool echo --- Help $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -h - echo --- As empty, sync and check + echo --- As empty, sync and check forcing murmur3 $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf diff $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf dup - $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf --test-force-murmur3 sync if HAVE_SYMLINK $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool endif @@ -178,6 +179,9 @@ endif mv test/disk3/7* test/disk3/_a/ $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check + echo --- Rehash to spooky2 + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-force-spooky2 rehash + $(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check echo --- Delete one disk, fix and check with raid5 rm -r test/disk3 mkdir test/disk3 diff --git a/rehash.c b/rehash.c new file mode 100644 index 00000000..38397196 --- /dev/null +++ b/rehash.c @@ -0,0 +1,233 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "util.h" +#include "elem.h" +#include "import.h" +#include "state.h" +#include "parity.h" +#include "handle.h" +#include "raid.h" + +/****************************************************************************/ +/* rehash */ + +static int state_rehash_process(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax) +{ + struct snapraid_handle* handle; + unsigned diskmax; + block_off_t i; + unsigned j; + void* buffer_alloc; + unsigned char* buffer_aligned; + int ret; + data_off_t countsize; + block_off_t countpos; + block_off_t countmax; + unsigned error; + + handle = handle_map(state, &diskmax); + + buffer_aligned = malloc_nofail_align(state->block_size, &buffer_alloc); + + error = 0; + + /* check that we are in a sync complete state */ + for(i=blockstart;iskip_sequential); + if (ret == -1) { + fprintf(stderr, "DANGER! Unexpected open error in a data disk, it isn't possible to rehash.\n"); + printf("Stopping at block %u\n", i); + ++error; + goto bail; + } + } + + /* read from the file */ + read_size = handle_read(&handle[j], block, buffer_aligned, state->block_size, stdlog); + if (read_size == -1) { + fprintf(stderr, "DANGER! Unexpected read error in a data disk, it isn't possible to rehash.\n"); + printf("Stopping at block %u\n", i); + ++error; + goto bail; + } + + countsize += read_size; + + /* compute the new hash */ + memhash(state->besthash, hash_new, buffer_aligned, read_size); + + /* compute the old hash */ + /* note that we intentionally compute the new one before the old */ + /* to ensure to detect any random bit flip in the memory */ + memhash(state->hash, hash, buffer_aligned, read_size); + + /* compare the hash */ + if (memcmp(hash, block->hash, HASH_SIZE) != 0) { + fprintf(stderr, "DANGER! Unexpected data error in a data disk, it isn't possible to rehash.\n"); + printf("Stopping at block %u\n", i); + ++error; + } + + /* store the new hash */ + memcpy(block->hash, hash_new, HASH_SIZE); + } + + /* count the number of processed block */ + ++countpos; + + /* progress */ + if (state_progress(state, i, countpos, countmax, countsize)) { + break; + } + } + + state_progress_end(state, countpos, countmax, countsize); + + /* set the new hash */ + state->hash = state->besthash; + + /* mark the state as needing write */ + state->need_write = 1; + +bail: + /* close all the files left open */ + for(j=0;jhash == state->besthash) { + fprintf(stderr, "You don't need to rehash because you are already using the best hash.\n"); + exit(EXIT_FAILURE); + } + + printf("Initializing...\n"); + + blockmax = parity_size(state); + + printf("Rehashing...\n"); + + error = 0; + + ret = state_rehash_process(state, 0, blockmax); + if (ret == -1) { + ++error; + /* continue, as we are already exiting */ + } + + /* abort if errors are present */ + if (error != 0) + exit(EXIT_FAILURE); +} + diff --git a/snapraid.c b/snapraid.c index 6ae9fe76..9803c01d 100644 --- a/snapraid.c +++ b/snapraid.c @@ -82,6 +82,8 @@ void usage(void) #define OPT_TEST_SKIP_FALLOCATE 261 #define OPT_TEST_SKIP_DEVICE 262 #define OPT_TEST_SKIP_SEQUENTIAL 263 +#define OPT_TEST_FORCE_MURMUR3 264 +#define OPT_TEST_FORCE_SPOOKY2 265 #if HAVE_GETOPT_LONG @@ -131,6 +133,12 @@ struct option long_options[] = { /* Skip the device check */ { "test-skip-device", 0, 0, OPT_TEST_SKIP_DEVICE }, + /* Force Murmur3 hash */ + { "test-force-murmur3", 0, 0, OPT_TEST_FORCE_MURMUR3 }, + + /* Force Spooky2 hash */ + { "test-force-spooky2", 0, 0, OPT_TEST_FORCE_SPOOKY2 }, + { 0, 0, 0, 0 } }; #endif @@ -155,6 +163,7 @@ void signal_handler(int signal) #define OPERATION_DRY 4 #define OPERATION_DUP 5 #define OPERATION_POOL 6 +#define OPERATION_REHASH 7 int main(int argc, char* argv[]) { @@ -174,6 +183,8 @@ int main(int argc, char* argv[]) int test_skip_fallocate; int test_skip_sequential; int test_skip_device; + int test_force_murmur3; + int test_force_spooky2; const char* conf; struct snapraid_state state; int operation; @@ -207,6 +218,8 @@ int main(int argc, char* argv[]) test_skip_fallocate = 0; test_skip_sequential = 0; test_skip_device = 0; + test_force_murmur3 = 0; + test_force_spooky2 = 0; blockstart = 0; blockcount = 0; tommy_list_init(&filterlist_file); @@ -328,6 +341,12 @@ int main(int argc, char* argv[]) case OPT_TEST_SKIP_DEVICE : test_skip_device = 1; break; + case OPT_TEST_FORCE_MURMUR3 : + test_force_murmur3 = 1; + break; + case OPT_TEST_FORCE_SPOOKY2 : + test_force_spooky2 = 1; + break; default: fprintf(stderr, "Unknown option '%c'\n", (char)c); exit(EXIT_FAILURE); @@ -354,6 +373,8 @@ int main(int argc, char* argv[]) operation = OPERATION_DUP; } else if (strcmp(argv[optind], "pool") == 0) { operation = OPERATION_POOL; + } else if (strcmp(argv[optind], "rehash") == 0) { + operation = OPERATION_REHASH; } else { fprintf(stderr, "Unknown command '%s'\n", argv[optind]); exit(EXIT_FAILURE); @@ -391,6 +412,13 @@ int main(int argc, char* argv[]) fprintf(stderr, "You cannot filter with the '%s' command\n", command); exit(EXIT_FAILURE); } + } + + switch (operation) { + case OPERATION_CHECK : + case OPERATION_FIX : + break; + default: if (import != 0) { fprintf(stderr, "You cannot import with the '%s' command\n", command); exit(EXIT_FAILURE); @@ -419,11 +447,7 @@ int main(int argc, char* argv[]) state_init(&state); - state_config(&state, conf, command, verbose, gui, force_zero, force_empty, force_uuid, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_sign, test_skip_fallocate, test_skip_sequential, test_skip_device); - - if (import != 0) { - state_import(&state, import); - } + state_config(&state, conf, command, verbose, gui, force_zero, force_empty, force_uuid, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_sign, test_skip_fallocate, test_skip_sequential, test_skip_device, test_force_murmur3, test_force_spooky2); if (operation == OPERATION_DIFF) { state_read(&state); @@ -483,6 +507,17 @@ int main(int argc, char* argv[]) signal(SIGINT, &signal_handler); state_dry(&state, blockstart, blockcount); + } else if (operation == OPERATION_REHASH) { + state_read(&state); + + /* intercept Ctrl+C */ + signal(SIGINT, &signal_handler); + + state_rehash(&state); + + /* save the new state if required */ + if (state.need_write) + state_write(&state); } else if (operation == OPERATION_DUP) { state_read(&state); @@ -494,6 +529,9 @@ int main(int argc, char* argv[]) } else { state_read(&state); + if (import != 0) + state_import(&state, import); + /* apply the command line filter */ state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing); diff --git a/state.c b/state.c index 20f11862..315a3e19 100644 --- a/state.c +++ b/state.c @@ -47,21 +47,6 @@ void state_init(struct snapraid_state* state) state->pool_device = 0; state->level = 1; /* default is the lowest protection */ - /* select the fastest hash */ - -#if defined(__i386__) || defined(__x86_64__) - if (sizeof(void*) == 4 && !cpu_has_slowmult()) - state->hash = HASH_MURMUR3; - else - state->hash = HASH_SPOOKY2; -#else - if (sizeof(void*) == 4) - state->hash = HASH_MURMUR3; - else - state->hash = HASH_SPOOKY2; -#endif - state->hash = HASH_MURMUR3; - tommy_list_init(&state->disklist); tommy_list_init(&state->maplist); tommy_list_init(&state->contentlist); @@ -257,7 +242,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, i } } -void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device) +void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2) { STREAM* f; unsigned line; @@ -276,6 +261,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co state->skip_fallocate = skip_fallocate; state->skip_sequential = skip_sequential; state->command = command; + state->force_murmur3 = force_murmur3; + state->force_spooky2 = force_spooky2; if (state->gui) { fprintf(stdlog, "version:%s\n", PACKAGE_VERSION); @@ -628,6 +615,28 @@ void state_config(struct snapraid_state* state, const char* path, const char* co state_config_check(state, path, skip_device); + /* select the default hash */ + if (state->force_murmur3) { + state->besthash = HASH_MURMUR3; + } else if (state->force_spooky2) { + state->besthash = HASH_SPOOKY2; + } else { +#if defined(__i386__) || defined(__x86_64__) + if (sizeof(void*) == 4 && !cpu_has_slowmult()) + state->besthash = HASH_MURMUR3; + else + state->besthash = HASH_SPOOKY2; +#else + if (sizeof(void*) == 4) + state->besthash = HASH_MURMUR3; + else + state->besthash = HASH_SPOOKY2; +#endif + } + + /* by default use the best hash */ + state->hash = state->besthash; + if (state->gui) { tommy_node* i; fprintf(stdlog, "blocksize:%u\n", state->block_size); diff --git a/state.h b/state.h index a751b863..b8c40026 100644 --- a/state.h +++ b/state.h @@ -40,6 +40,8 @@ struct snapraid_state { int skip_sign; /**< Skip the sign check for content files. */ int skip_fallocate; /**< Skip the use of fallocate(). */ int skip_sequential; /**< Skip sequential hint. */ + int force_murmur3; /**< Force Murmur3 choice. */ + int force_spooky2; /**< Force Spooky2 choice. */ int need_write; /**< If the state is changed. */ uint32_t block_size; /**< Block size in bytes. */ char parity[PATH_MAX]; /**< Path of the parity file. */ @@ -50,6 +52,7 @@ struct snapraid_state { uint64_t pool_device; /**< Device identifier of the pool. */ unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */ unsigned hash; /**< Hash kind used. */ + unsigned besthash; /**< Best hash suggested. */ const char* command; /**< Command running. */ tommy_list contentlist; /**< List of content files. */ tommy_list disklist; /**< List of all the disks. */ @@ -77,7 +80,7 @@ void state_done(struct snapraid_state* state); /** * Reads the configuration file. */ -void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device); +void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2); /** * Reads the state. @@ -111,6 +114,11 @@ void state_check(struct snapraid_state* state, int check, int fix, block_off_t b */ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount); +/** + * Rehash the files. + */ +void state_rehash(struct snapraid_state* state); + /** * Finds duplicates. */