mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 19:01:57 -04:00
Activates spooky2 and adds the new rehash command.
This commit is contained in:
Andrea Mazzoleni
parent
afa38056f9
commit
548e90c30f
6 changed files with 319 additions and 26 deletions
5
HISTORY
5
HISTORY
|
|
@ -4,8 +4,9 @@ SnapRAID HISTORY
|
|||
3.2 2013/6
|
||||
==========
|
||||
* Better selection of the RAID6 implementation for different CPUs.
|
||||
* Adds the new Spooky hash. It's faster in 64 bit architectures.
|
||||
* Adds RAID5/RAID6 mmx and sse2 implementations with unrolling by 4.
|
||||
* Added the new Spooky hash. It's faster in 64 bit architectures.
|
||||
To convert you can use the new 'rehash' command.
|
||||
* Added RAID5/RAID6 mmx and sse2 implementations with unrolling by 4.
|
||||
They are a little faster than the previous unroll by 2.
|
||||
|
||||
3.1 2013/5
|
||||
|
|
|
|||
|
|
@ -11,6 +11,7 @@ snapraid_SOURCES = \
|
|||
sync.c \
|
||||
check.c \
|
||||
dry.c \
|
||||
rehash.c \
|
||||
dup.c \
|
||||
pool.c \
|
||||
parity.c \
|
||||
|
|
@ -95,10 +96,10 @@ check-local:
|
|||
mkdir test/pool
|
||||
echo --- Help
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -h
|
||||
echo --- As empty, sync and check
|
||||
echo --- As empty, sync and check forcing murmur3
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf diff
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf dup
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf sync
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) -v -c $(srcdir)/test-raid6.conf --test-force-murmur3 sync
|
||||
if HAVE_SYMLINK
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid5.conf pool
|
||||
endif
|
||||
|
|
@ -178,6 +179,9 @@ endif
|
|||
mv test/disk3/7* test/disk3/_a/
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf sync
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
|
||||
echo --- Rehash to spooky2
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf --test-force-spooky2 rehash
|
||||
$(TESTENV) ./snapraid$(EXEEXT) $(CHECKFLAGS) --test-skip-self -c $(srcdir)/test-raid6.conf check
|
||||
echo --- Delete one disk, fix and check with raid5
|
||||
rm -r test/disk3
|
||||
mkdir test/disk3
|
||||
|
|
|
|||
233
rehash.c
Normal file
233
rehash.c
Normal file
|
|
@ -0,0 +1,233 @@
|
|||
/*
|
||||
* Copyright (C) 2013 Andrea Mazzoleni
|
||||
*
|
||||
* This program is free software: you can redistribute it and/or modify
|
||||
* it under the terms of the GNU General Public License as published by
|
||||
* the Free Software Foundation, either version 3 of the License, or
|
||||
* (at your option) any later version.
|
||||
*
|
||||
* This program is distributed in the hope that it will be useful,
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
||||
* GNU General Public License for more details.
|
||||
*
|
||||
* You should have received a copy of the GNU General Public License
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
||||
*/
|
||||
|
||||
#include "portable.h"
|
||||
|
||||
#include "util.h"
|
||||
#include "elem.h"
|
||||
#include "import.h"
|
||||
#include "state.h"
|
||||
#include "parity.h"
|
||||
#include "handle.h"
|
||||
#include "raid.h"
|
||||
|
||||
/****************************************************************************/
|
||||
/* rehash */
|
||||
|
||||
static int state_rehash_process(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax)
|
||||
{
|
||||
struct snapraid_handle* handle;
|
||||
unsigned diskmax;
|
||||
block_off_t i;
|
||||
unsigned j;
|
||||
void* buffer_alloc;
|
||||
unsigned char* buffer_aligned;
|
||||
int ret;
|
||||
data_off_t countsize;
|
||||
block_off_t countpos;
|
||||
block_off_t countmax;
|
||||
unsigned error;
|
||||
|
||||
handle = handle_map(state, &diskmax);
|
||||
|
||||
buffer_aligned = malloc_nofail_align(state->block_size, &buffer_alloc);
|
||||
|
||||
error = 0;
|
||||
|
||||
/* check that we are in a sync complete state */
|
||||
for(i=blockstart;i<blockmax;++i) {
|
||||
/* for each disk, process the block */
|
||||
for(j=0;j<diskmax;++j) {
|
||||
struct snapraid_block* block;
|
||||
unsigned block_state;
|
||||
|
||||
/* if the disk position is not used */
|
||||
if (!handle[j].disk) {
|
||||
/* skip it */
|
||||
continue;
|
||||
}
|
||||
|
||||
/* if the disk block is not used */
|
||||
block = disk_block_get(handle[j].disk, i);
|
||||
if (block == BLOCK_EMPTY) {
|
||||
/* skip it */
|
||||
continue;
|
||||
}
|
||||
|
||||
/* get the state of the block */
|
||||
block_state = block_state_get(block);
|
||||
|
||||
/* if the block is deleted */
|
||||
if (block_state != BLOCK_STATE_BLK) {
|
||||
fprintf(stderr, "You can rehash only after a complete sync.\n");
|
||||
++error;
|
||||
goto bail;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/* rehash all the blocks in files */
|
||||
countmax = blockmax - blockstart;
|
||||
countsize = 0;
|
||||
countpos = 0;
|
||||
state_progress_begin(state, blockstart, blockmax, countmax);
|
||||
for(i=blockstart;i<blockmax;++i) {
|
||||
/* for each disk, process the block */
|
||||
for(j=0;j<diskmax;++j) {
|
||||
int read_size;
|
||||
unsigned char hash[HASH_SIZE];
|
||||
unsigned char hash_new[HASH_SIZE];
|
||||
struct snapraid_block* block;
|
||||
|
||||
/* if the disk position is not used */
|
||||
if (!handle[j].disk) {
|
||||
/* skip it */
|
||||
continue;
|
||||
}
|
||||
|
||||
/* if the disk block is not used */
|
||||
block = disk_block_get(handle[j].disk, i);
|
||||
if (block == BLOCK_EMPTY) {
|
||||
/* skip it */
|
||||
continue;
|
||||
}
|
||||
|
||||
/* here we can have only BLOCK_STATE_BLK */
|
||||
|
||||
/* if the file is closed or different than the current one */
|
||||
if (handle[j].file == 0 || handle[j].file != block_file_get(block)) {
|
||||
/* close the old one, if any */
|
||||
ret = handle_close(&handle[j]);
|
||||
if (ret == -1) {
|
||||
fprintf(stderr, "DANGER! Unexpected close error in a data disk, it isn't possible to rehash.\n");
|
||||
printf("Stopping at block %u\n", i);
|
||||
++error;
|
||||
goto bail;
|
||||
}
|
||||
|
||||
/* open the file only for reading */
|
||||
ret = handle_open(&handle[j], block_file_get(block), stdlog, state->skip_sequential);
|
||||
if (ret == -1) {
|
||||
fprintf(stderr, "DANGER! Unexpected open error in a data disk, it isn't possible to rehash.\n");
|
||||
printf("Stopping at block %u\n", i);
|
||||
++error;
|
||||
goto bail;
|
||||
}
|
||||
}
|
||||
|
||||
/* read from the file */
|
||||
read_size = handle_read(&handle[j], block, buffer_aligned, state->block_size, stdlog);
|
||||
if (read_size == -1) {
|
||||
fprintf(stderr, "DANGER! Unexpected read error in a data disk, it isn't possible to rehash.\n");
|
||||
printf("Stopping at block %u\n", i);
|
||||
++error;
|
||||
goto bail;
|
||||
}
|
||||
|
||||
countsize += read_size;
|
||||
|
||||
/* compute the new hash */
|
||||
memhash(state->besthash, hash_new, buffer_aligned, read_size);
|
||||
|
||||
/* compute the old hash */
|
||||
/* note that we intentionally compute the new one before the old */
|
||||
/* to ensure to detect any random bit flip in the memory */
|
||||
memhash(state->hash, hash, buffer_aligned, read_size);
|
||||
|
||||
/* compare the hash */
|
||||
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
|
||||
fprintf(stderr, "DANGER! Unexpected data error in a data disk, it isn't possible to rehash.\n");
|
||||
printf("Stopping at block %u\n", i);
|
||||
++error;
|
||||
}
|
||||
|
||||
/* store the new hash */
|
||||
memcpy(block->hash, hash_new, HASH_SIZE);
|
||||
}
|
||||
|
||||
/* count the number of processed block */
|
||||
++countpos;
|
||||
|
||||
/* progress */
|
||||
if (state_progress(state, i, countpos, countmax, countsize)) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
state_progress_end(state, countpos, countmax, countsize);
|
||||
|
||||
/* set the new hash */
|
||||
state->hash = state->besthash;
|
||||
|
||||
/* mark the state as needing write */
|
||||
state->need_write = 1;
|
||||
|
||||
bail:
|
||||
/* close all the files left open */
|
||||
for(j=0;j<diskmax;++j) {
|
||||
ret = handle_close(&handle[j]);
|
||||
if (ret == -1) {
|
||||
fprintf(stderr, "DANGER! Unexpected close error in a data disk.\n");
|
||||
++error;
|
||||
/* continue, as we are already exiting */
|
||||
}
|
||||
}
|
||||
|
||||
if (error) {
|
||||
printf("Rehash aborted\n");
|
||||
} else {
|
||||
printf("No error\n");
|
||||
}
|
||||
|
||||
free(handle);
|
||||
free(buffer_alloc);
|
||||
|
||||
if (error != 0)
|
||||
return -1;
|
||||
return 0;
|
||||
}
|
||||
|
||||
void state_rehash(struct snapraid_state* state)
|
||||
{
|
||||
block_off_t blockmax;
|
||||
int ret;
|
||||
unsigned error;
|
||||
|
||||
if (state->hash == state->besthash) {
|
||||
fprintf(stderr, "You don't need to rehash because you are already using the best hash.\n");
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
printf("Initializing...\n");
|
||||
|
||||
blockmax = parity_size(state);
|
||||
|
||||
printf("Rehashing...\n");
|
||||
|
||||
error = 0;
|
||||
|
||||
ret = state_rehash_process(state, 0, blockmax);
|
||||
if (ret == -1) {
|
||||
++error;
|
||||
/* continue, as we are already exiting */
|
||||
}
|
||||
|
||||
/* abort if errors are present */
|
||||
if (error != 0)
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
48
snapraid.c
48
snapraid.c
|
|
@ -82,6 +82,8 @@ void usage(void)
|
|||
#define OPT_TEST_SKIP_FALLOCATE 261
|
||||
#define OPT_TEST_SKIP_DEVICE 262
|
||||
#define OPT_TEST_SKIP_SEQUENTIAL 263
|
||||
#define OPT_TEST_FORCE_MURMUR3 264
|
||||
#define OPT_TEST_FORCE_SPOOKY2 265
|
||||
|
||||
|
||||
#if HAVE_GETOPT_LONG
|
||||
|
|
@ -131,6 +133,12 @@ struct option long_options[] = {
|
|||
/* Skip the device check */
|
||||
{ "test-skip-device", 0, 0, OPT_TEST_SKIP_DEVICE },
|
||||
|
||||
/* Force Murmur3 hash */
|
||||
{ "test-force-murmur3", 0, 0, OPT_TEST_FORCE_MURMUR3 },
|
||||
|
||||
/* Force Spooky2 hash */
|
||||
{ "test-force-spooky2", 0, 0, OPT_TEST_FORCE_SPOOKY2 },
|
||||
|
||||
{ 0, 0, 0, 0 }
|
||||
};
|
||||
#endif
|
||||
|
|
@ -155,6 +163,7 @@ void signal_handler(int signal)
|
|||
#define OPERATION_DRY 4
|
||||
#define OPERATION_DUP 5
|
||||
#define OPERATION_POOL 6
|
||||
#define OPERATION_REHASH 7
|
||||
|
||||
int main(int argc, char* argv[])
|
||||
{
|
||||
|
|
@ -174,6 +183,8 @@ int main(int argc, char* argv[])
|
|||
int test_skip_fallocate;
|
||||
int test_skip_sequential;
|
||||
int test_skip_device;
|
||||
int test_force_murmur3;
|
||||
int test_force_spooky2;
|
||||
const char* conf;
|
||||
struct snapraid_state state;
|
||||
int operation;
|
||||
|
|
@ -207,6 +218,8 @@ int main(int argc, char* argv[])
|
|||
test_skip_fallocate = 0;
|
||||
test_skip_sequential = 0;
|
||||
test_skip_device = 0;
|
||||
test_force_murmur3 = 0;
|
||||
test_force_spooky2 = 0;
|
||||
blockstart = 0;
|
||||
blockcount = 0;
|
||||
tommy_list_init(&filterlist_file);
|
||||
|
|
@ -328,6 +341,12 @@ int main(int argc, char* argv[])
|
|||
case OPT_TEST_SKIP_DEVICE :
|
||||
test_skip_device = 1;
|
||||
break;
|
||||
case OPT_TEST_FORCE_MURMUR3 :
|
||||
test_force_murmur3 = 1;
|
||||
break;
|
||||
case OPT_TEST_FORCE_SPOOKY2 :
|
||||
test_force_spooky2 = 1;
|
||||
break;
|
||||
default:
|
||||
fprintf(stderr, "Unknown option '%c'\n", (char)c);
|
||||
exit(EXIT_FAILURE);
|
||||
|
|
@ -354,6 +373,8 @@ int main(int argc, char* argv[])
|
|||
operation = OPERATION_DUP;
|
||||
} else if (strcmp(argv[optind], "pool") == 0) {
|
||||
operation = OPERATION_POOL;
|
||||
} else if (strcmp(argv[optind], "rehash") == 0) {
|
||||
operation = OPERATION_REHASH;
|
||||
} else {
|
||||
fprintf(stderr, "Unknown command '%s'\n", argv[optind]);
|
||||
exit(EXIT_FAILURE);
|
||||
|
|
@ -391,6 +412,13 @@ int main(int argc, char* argv[])
|
|||
fprintf(stderr, "You cannot filter with the '%s' command\n", command);
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
}
|
||||
|
||||
switch (operation) {
|
||||
case OPERATION_CHECK :
|
||||
case OPERATION_FIX :
|
||||
break;
|
||||
default:
|
||||
if (import != 0) {
|
||||
fprintf(stderr, "You cannot import with the '%s' command\n", command);
|
||||
exit(EXIT_FAILURE);
|
||||
|
|
@ -419,11 +447,7 @@ int main(int argc, char* argv[])
|
|||
|
||||
state_init(&state);
|
||||
|
||||
state_config(&state, conf, command, verbose, gui, force_zero, force_empty, force_uuid, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_sign, test_skip_fallocate, test_skip_sequential, test_skip_device);
|
||||
|
||||
if (import != 0) {
|
||||
state_import(&state, import);
|
||||
}
|
||||
state_config(&state, conf, command, verbose, gui, force_zero, force_empty, force_uuid, find_by_name, test_expect_unrecoverable, test_expect_recoverable, test_skip_sign, test_skip_fallocate, test_skip_sequential, test_skip_device, test_force_murmur3, test_force_spooky2);
|
||||
|
||||
if (operation == OPERATION_DIFF) {
|
||||
state_read(&state);
|
||||
|
|
@ -483,6 +507,17 @@ int main(int argc, char* argv[])
|
|||
signal(SIGINT, &signal_handler);
|
||||
|
||||
state_dry(&state, blockstart, blockcount);
|
||||
} else if (operation == OPERATION_REHASH) {
|
||||
state_read(&state);
|
||||
|
||||
/* intercept Ctrl+C */
|
||||
signal(SIGINT, &signal_handler);
|
||||
|
||||
state_rehash(&state);
|
||||
|
||||
/* save the new state if required */
|
||||
if (state.need_write)
|
||||
state_write(&state);
|
||||
} else if (operation == OPERATION_DUP) {
|
||||
state_read(&state);
|
||||
|
||||
|
|
@ -494,6 +529,9 @@ int main(int argc, char* argv[])
|
|||
} else {
|
||||
state_read(&state);
|
||||
|
||||
if (import != 0)
|
||||
state_import(&state, import);
|
||||
|
||||
/* apply the command line filter */
|
||||
state_filter(&state, &filterlist_file, &filterlist_disk, filter_missing);
|
||||
|
||||
|
|
|
|||
41
state.c
41
state.c
|
|
@ -47,21 +47,6 @@ void state_init(struct snapraid_state* state)
|
|||
state->pool_device = 0;
|
||||
state->level = 1; /* default is the lowest protection */
|
||||
|
||||
/* select the fastest hash */
|
||||
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
if (sizeof(void*) == 4 && !cpu_has_slowmult())
|
||||
state->hash = HASH_MURMUR3;
|
||||
else
|
||||
state->hash = HASH_SPOOKY2;
|
||||
#else
|
||||
if (sizeof(void*) == 4)
|
||||
state->hash = HASH_MURMUR3;
|
||||
else
|
||||
state->hash = HASH_SPOOKY2;
|
||||
#endif
|
||||
state->hash = HASH_MURMUR3;
|
||||
|
||||
tommy_list_init(&state->disklist);
|
||||
tommy_list_init(&state->maplist);
|
||||
tommy_list_init(&state->contentlist);
|
||||
|
|
@ -257,7 +242,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, i
|
|||
}
|
||||
}
|
||||
|
||||
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device)
|
||||
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2)
|
||||
{
|
||||
STREAM* f;
|
||||
unsigned line;
|
||||
|
|
@ -276,6 +261,8 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
|
|||
state->skip_fallocate = skip_fallocate;
|
||||
state->skip_sequential = skip_sequential;
|
||||
state->command = command;
|
||||
state->force_murmur3 = force_murmur3;
|
||||
state->force_spooky2 = force_spooky2;
|
||||
|
||||
if (state->gui) {
|
||||
fprintf(stdlog, "version:%s\n", PACKAGE_VERSION);
|
||||
|
|
@ -628,6 +615,28 @@ void state_config(struct snapraid_state* state, const char* path, const char* co
|
|||
|
||||
state_config_check(state, path, skip_device);
|
||||
|
||||
/* select the default hash */
|
||||
if (state->force_murmur3) {
|
||||
state->besthash = HASH_MURMUR3;
|
||||
} else if (state->force_spooky2) {
|
||||
state->besthash = HASH_SPOOKY2;
|
||||
} else {
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
if (sizeof(void*) == 4 && !cpu_has_slowmult())
|
||||
state->besthash = HASH_MURMUR3;
|
||||
else
|
||||
state->besthash = HASH_SPOOKY2;
|
||||
#else
|
||||
if (sizeof(void*) == 4)
|
||||
state->besthash = HASH_MURMUR3;
|
||||
else
|
||||
state->besthash = HASH_SPOOKY2;
|
||||
#endif
|
||||
}
|
||||
|
||||
/* by default use the best hash */
|
||||
state->hash = state->besthash;
|
||||
|
||||
if (state->gui) {
|
||||
tommy_node* i;
|
||||
fprintf(stdlog, "blocksize:%u\n", state->block_size);
|
||||
|
|
|
|||
10
state.h
10
state.h
|
|
@ -40,6 +40,8 @@ struct snapraid_state {
|
|||
int skip_sign; /**< Skip the sign check for content files. */
|
||||
int skip_fallocate; /**< Skip the use of fallocate(). */
|
||||
int skip_sequential; /**< Skip sequential hint. */
|
||||
int force_murmur3; /**< Force Murmur3 choice. */
|
||||
int force_spooky2; /**< Force Spooky2 choice. */
|
||||
int need_write; /**< If the state is changed. */
|
||||
uint32_t block_size; /**< Block size in bytes. */
|
||||
char parity[PATH_MAX]; /**< Path of the parity file. */
|
||||
|
|
@ -50,6 +52,7 @@ struct snapraid_state {
|
|||
uint64_t pool_device; /**< Device identifier of the pool. */
|
||||
unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */
|
||||
unsigned hash; /**< Hash kind used. */
|
||||
unsigned besthash; /**< Best hash suggested. */
|
||||
const char* command; /**< Command running. */
|
||||
tommy_list contentlist; /**< List of content files. */
|
||||
tommy_list disklist; /**< List of all the disks. */
|
||||
|
|
@ -77,7 +80,7 @@ void state_done(struct snapraid_state* state);
|
|||
/**
|
||||
* Reads the configuration file.
|
||||
*/
|
||||
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device);
|
||||
void state_config(struct snapraid_state* state, const char* path, const char* command, int verbose, int gui, int force_zero, int force_empty, int force_uuid, int find_by_name, int expect_unrecoverable, int expect_recoverable, int skip_sign, int skip_fallocate, int skip_sequential, int skip_device, int force_murmur3, int force_spooky2);
|
||||
|
||||
/**
|
||||
* Reads the state.
|
||||
|
|
@ -111,6 +114,11 @@ void state_check(struct snapraid_state* state, int check, int fix, block_off_t b
|
|||
*/
|
||||
void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount);
|
||||
|
||||
/**
|
||||
* Rehash the files.
|
||||
*/
|
||||
void state_rehash(struct snapraid_state* state);
|
||||
|
||||
/**
|
||||
* Finds duplicates.
|
||||
*/
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue