rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 10:51:54 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Initial RAID6 support. Not yet complete.

Browse source 
Uses Murmur3 as default hash.
Check now returns and error even if only recoverable errors are present.
This commit is contained in:
Andrea Mazzoleni 2011-04-23 21:38:26 +02:00
parent ad70b7e560
commit 08831c6641
24 changed files with 1624 additions and 394 deletions
Download patch file Download diff file Expand all files Collapse all files

9
HISTORY
View file

@ -1,6 +1,14 @@
SnapRAID HISTORY
================
1.1 2011/5
==========
* Added RAID6 support.
* The default checksum is Murmur3 hash.
MD5 is still supported for backward compatibility.
* The check command returns with an error code if any kind of error is present.
Previously it was only if unrecoverable error was present.
1.0 2011/4
==========
* No relevant change.
@ -26,7 +34,6 @@ SnapRAID HISTORY
* Precise error management.
* More regression tests.
0.1 2011/3
==========
* First public test release of SnapRAID.

28
Makefile.am
View file

@ -13,10 +13,14 @@ snapraid_SOURCES = \
dry.c \
parity.c \
handle.c \
fnmatch.c
fnmatch.c \
raid.c \
selftest.c \
speed.c
noinst_HEADERS = \
portable.h \
snapraid.h \
util.h \
elem.h \
state.h \
@ -24,6 +28,7 @@ noinst_HEADERS = \
handle.h \
md5.h \
md5.c \
murmurhash3.c \
tommyarray.h \
tommychain.h \
tommyhash.h \
@ -34,7 +39,9 @@ noinst_HEADERS = \
tommyhash.c \
tommyhashdyn.c \
tommylist.c \
fnmatch.h
fnmatch.h \
cpu.h \
raid.h
mktest_SOURCES = \
mktest.c
@ -97,26 +104,26 @@ check-local:
rm test/disk3/6*
$(TESTENV) ./mktest$(EXEEXT) generate 4 3 100 4000
$(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test.conf -t 7 sync
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test.conf -t 11 sync
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test.conf sync
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Delete some files, fix and check
rm test/disk1/1*
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Delete some dirs, fix and check
rm -r test/disk2/_a
rm -r test/disk2/_b
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Delete a whole disk, fix and check
rm -r test/disk3
mkdir test/disk3
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Sync after all the fixes with an empty drive
@ -128,12 +135,12 @@ check-local:
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Corrupt the parity, fix and check
$(TESTENV) ./mktest$(EXEEXT) damage 1 10 10000 test/parity
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Corrupt some files, fix and check
$(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/b*
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null
$(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check
echo --- Sync after all the fixes
@ -181,6 +188,9 @@ snapraid.txt : snapraid.d
snapraid.html : snapraid.d
advd2 html < $(srcdir)/$< > $@
snapraid.hh : snapraid.d
advd2 frame < $(srcdir)/$< > $@
# Windows distribution
DISTWIN = \

5
TODO
View file

@ -12,11 +12,16 @@ completely restored.
* Reduce memory occupation removing the flag field in the block structure.
* Provide an "include" logic in filtering.
* Add RAID6 redundancy support. There is the very good raid6 library from
H. Peter Anvin (hpa) in the Linux Kernel ready to use. See:
http://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf
http://lxr.linux.no/#linux+v2.6.38/lib/raid6
* Recognize that a file is moved from one disk to another, and if the parity
data doesn't overlap, do not recompute it.
* Go multithread to speed up the MD5 computation. We can do it when syncing and
checking.

443
check.c
View file

@ -22,18 +22,94 @@
#include "state.h"
#include "parity.h"
#include "handle.h"
#include "raid.h"
/****************************************************************************/
/* check */
static int state_check_process(struct snapraid_state* state, int fix, int parity_f, block_off_t blockstart, block_off_t blockmax)
struct failed_struct {
unsigned index;
struct snapraid_block* block;
struct snapraid_handle* handle;
};
/**
* Repair errors.
* Returns <0 if failure for missing stratey, >0 if data is wrong and we cannot rebuild correctly, 0 on success.
* If success, the parity and qarity are computed in the buffer variable.
*/
static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, struct failed_struct* failed, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity)
{
/* no fix required */
if (failed_count == 0) {
/* compute parity and qarity to check it */
if (state->level == 1) {
raid5_gen(buffer, diskmax + state->level, state->block_size);
} else {
raid6_gen(buffer, diskmax + state->level, state->block_size);
}
return 0;
}
/* fix with RAID5 */
if (failed_count == 1 && buffer_parity != 0) {
unsigned char hash[HASH_SIZE];
unsigned size;
unsigned j;
/* recompute the block */
memcpy(buffer[failed[0].index], buffer_parity, state->block_size);
for(j=0;j<diskmax;++j) {
if (j != failed[0].index) {
memxor(buffer[failed[0].index], buffer[j], state->block_size);
}
}
size = block_file_size(failed[0].block, state->block_size);
/* now compute the hash */
memhash(state->hash, hash, buffer[failed[0].index], size);
/* compare the hash */
if (memcmp(hash, failed[0].block->hash, HASH_SIZE) != 0) {
fprintf(stderr, "%u: Parity data error\n", i);
return 1;
}
/* compute parity and qarity to check it */
/* we recompute also the just used parity, because we may have used only a small part of it */
if (state->level == 1) {
raid5_gen(buffer, diskmax + state->level, state->block_size);
} else {
raid6_gen(buffer, diskmax + state->level, state->block_size);
}
return 0;
}
if (failed_count == 1 && buffer_qarity != 0) {
/* TODO RAID6 */
}
if (failed_count == 2 && buffer_parity != 0 && buffer_qarity != 0) {
/* TODO RAID6 */
}
/* no more stragety to fix */
return -1;
}
static int state_check_process(struct snapraid_state* state, int fix, int parity_f, int qarity_f, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_handle* handle;
unsigned diskmax = tommy_array_size(&state->diskarr);
block_off_t i;
unsigned j;
unsigned char* block_buffer;
unsigned char* xor_buffer;
void* buffer_alloc;
unsigned char* buffer_aligned;
unsigned char** buffer;
unsigned buffermax;
int ret;
data_off_t countsize;
block_off_t countpos;
@ -44,8 +120,14 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
unsigned unrecoverable_error;
unsigned recovered_error;
block_buffer = malloc_nofail(state->block_size);
xor_buffer = malloc_nofail(state->block_size);
/* we need disk + 2 for each parity level buffers */
buffermax = diskmax + state->level * 2;
buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc);
buffer = malloc_nofail(buffermax * sizeof(void*));
for(i=0;i<buffermax;++i) {
buffer[i] = buffer_aligned + i * state->block_size;
}
handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle));
for(i=0;i<diskmax;++i) {
@ -87,9 +169,8 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
start = time(0);
last = start;
for(i=blockstart;i<blockmax;++i) {
int failed;
struct snapraid_block* failed_block;
struct snapraid_handle* failed_handle;
struct failed_struct failed[2];
unsigned failed_count;
int one_tocheck;
int all_parity;
@ -110,13 +191,8 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
if (!one_tocheck)
continue;
/* start with 0 */
memset(xor_buffer, 0, state->block_size);
all_parity = 1; /* if all hashed block have parity computed */
failed = 0; /* number of failed block */
failed_block = 0; /* last failed block, but not filtered out */
failed_handle = 0;
failed_count = 0; /* number of failed blocks */
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
@ -125,14 +201,20 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
struct snapraid_block* block;
block = disk_block_get(handle[j].disk, i);
if (!block)
if (!block) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* we try to check and fix only if the block is hashed */
/* it could be that a file was added, and not yet synched */
/* so we should not include not hashed block in the parity computation */
if (!bit_has(block->flag, BLOCK_HAS_HASH))
if (!bit_has(block->flag, BLOCK_HAS_HASH)) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
/* keep track if at least one hashed block has no parity */
if (!bit_has(block->flag, BLOCK_HAS_PARITY))
@ -160,11 +242,12 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
ret = handle_open(&handle[j], block->file);
if (ret == -1) {
/* save the failed block for the parity check */
++failed;
if (!block->file->is_filtered) {
failed_block = block;
failed_handle = &handle[j];
if (failed_count < 2) {
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
}
++failed_count;
fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
++error;
@ -172,14 +255,15 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
}
read_size = handle_read(&handle[j], block, block_buffer, state->block_size);
read_size = handle_read(&handle[j], block, buffer[j], state->block_size);
if (read_size == -1) {
/* save the failed block for the parity check */
++failed;
if (!block->file->is_filtered) {
failed_block = block;
failed_handle = &handle[j];
if (failed_count < 2) {
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
}
++failed_count;
fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
++error;
@ -187,123 +271,144 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity
}
/* now compute the hash */
memhash(hash, block_buffer, read_size);
memhash(state->hash, hash, buffer[j], read_size);
/* compare the hash */
if (memcmp(hash, block->hash, HASH_SIZE) != 0) {
/* save the failed block for the parity check */
++failed;
if (!block->file->is_filtered) {
failed_block = block;
failed_handle = &handle[j];
if (failed_count < 2) {
failed[failed_count].index = j;
failed[failed_count].block = block;
failed[failed_count].handle = &handle[j];
}
++failed_count;
fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
++error;
continue;
}
/* compute the parity */
memxor(xor_buffer, block_buffer, read_size);
countsize += read_size;
}
if (failed != 0 && !failed_block) {
/* if the failed blocks are filtered out, do nothing */
} else if (parity_f == -1 || !all_parity) {
/* all the cases with no parity file */
if (failed != 0) {
fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i);
++unrecoverable_error;
}
} else if (failed == 0) {
int ret;
int fixable;
{
unsigned char* buffer_parity;
unsigned char* buffer_qarity;
fixable = 0;
/* read the parity */
ret = parity_read(state->parity, parity_f, i, block_buffer, state->block_size);
if (ret == -1) {
fprintf(stderr, "%u: Parity read error\n", i);
++error;
fixable = 1;
/* buffers for parity read and not computed */
if (state->level == 1) {
buffer_parity = buffer[diskmax + 1];
buffer_qarity = 0;
} else {
/* compare it */
if (memcmp(xor_buffer, block_buffer, state->block_size) != 0) {
buffer_parity = buffer[diskmax + 2];
buffer_qarity = buffer[diskmax + 3];
}
/* read the parity */
if (parity_f != -1) {
ret = parity_read(state->parity, parity_f, i, buffer_parity, state->block_size);
if (ret == -1) {
buffer_parity = 0; /* no parity to use */
fprintf(stderr, "%u: Parity read error\n", i);
++error;
}
} else {
buffer_parity = 0;
}
/* read the qarity */
if (state->level >= 2) {
if (qarity_f != -1) {
ret = parity_read(state->qarity, qarity_f, i, buffer_qarity, state->block_size);
if (ret == -1) {
buffer_qarity = 0; /* no qarity to use */
fprintf(stderr, "%u: Q-Parity read error\n", i);
++error;
}
} else {
buffer_qarity = 0;
}
}
ret = repair(state, i, diskmax, failed, failed_count, buffer, buffer_parity, buffer_qarity);
if (ret != 0) {
/* increment the number of errors */
if (ret > 0)
error += ret;
fprintf(stderr, "%u: UNRECOVERABLE error for this block\n", i);
++unrecoverable_error;
} else {
/* check the parity */
if (buffer_parity != 0 && memcmp(buffer_parity, buffer[diskmax], state->block_size) != 0) {
buffer_parity = 0;
fprintf(stderr, "%u: Parity data error\n", i);
++error;
fixable = 1;
}
}
if (fix && fixable) {
/* fix the parity */
/* if all the file hashes are matching, we can recompute the parity */
ret = parity_write(state->parity, parity_f, i, xor_buffer, state->block_size);
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working parity disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
fprintf(stderr, "%u: Fixed\n", i);
++recovered_error;
}
} else if (failed == 1) {
unsigned char hash[HASH_SIZE];
unsigned failed_size;
int ret;
int fixable;
/* check the qarity */
if (state->level >= 2) {
if (buffer_qarity != 0 && memcmp(buffer_qarity, buffer[diskmax + 1], state->block_size) != 0) {
buffer_qarity = 0;
fixable = 0;
/* read the parity */
ret = parity_read(state->parity, parity_f, i, block_buffer, state->block_size);
if (ret == -1) {
fprintf(stderr, "%u: Parity read error\n", i);
fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i);
++unrecoverable_error;
} else {
/* compute the failed block */
memxor(block_buffer, xor_buffer, state->block_size);
failed_size = block_file_size(failed_block, state->block_size);
/* now compute the hash */
memhash(hash, block_buffer, failed_size);
/* compare the hash */
if (memcmp(hash, failed_block->hash, HASH_SIZE) != 0) {
fprintf(stderr, "%u: Parity data error\n", i);
fprintf(stderr, "%u: UNRECOVERABLE erros for this block\n", i);
++unrecoverable_error;
} else {
fixable = 1;
}
}
if (fix && fixable) {
/* fix the file */
/* if only one file is wrong, we already have recomputed it */
ret = handle_write(failed_handle, failed_block, block_buffer, state->block_size);
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
fprintf(stderr, "%u: Q-Parity data error\n", i);
++error;
}
}
fprintf(stderr, "%u: Fixed\n", i);
++recovered_error;
if (fix) {
/* update the fixed files */
for(j=0;j<failed_count;++j) {
/* do not fix if the file filtered out */
if (failed[j].block->file->is_filtered)
continue;
ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size);
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
fprintf(stderr, "%u: Fixed data error for file %s at position %u\n", i, failed[j].block->file->sub, block_file_pos(failed[j].block));
++recovered_error;
}
/* update the parity */
if (buffer_parity == 0 && parity_f != -1) {
ret = parity_write(state->parity, parity_f, i, buffer[diskmax], state->block_size);
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working parity disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
fprintf(stderr, "%u: Fixed parity error\n", i);
++recovered_error;
}
/* update the qarity */
if (state->level >= 2) {
if (buffer_qarity == 0 && qarity_f != -1) {
ret = parity_write(state->qarity, qarity_f, i, buffer[diskmax + 1], state->block_size);
if (ret == -1) {
fprintf(stderr, "WARNING! Without a working Q-Parity disk, it isn't possible to fix errors on it.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
}
fprintf(stderr, "%u: Fixed Q-Parity error\n", i);
++recovered_error;
}
}
}
} else {
fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i);
++unrecoverable_error;
}
/* count the number of processed block */
@ -353,22 +458,47 @@ bail:
}
free(handle);
free(block_buffer);
free(xor_buffer);
free(buffer_alloc);
free(buffer);
/* fails if some error are present after the run */
if (fix) {
if (state->expect_unrecoverable) {
if (unrecoverable_error == 0)
return -1;
} else {
if (unrecoverable_error != 0)
return -1;
}
} else {
if (state->expect_unrecoverable && state->expect_recoverable) {
if (error == 0 || unrecoverable_error == 0)
return -1;
} else if (state->expect_unrecoverable) {
if (unrecoverable_error == 0)
return -1;
} if (state->expect_recoverable) {
if (error == 0)
return -1;
} else {
if (error != 0 || unrecoverable_error != 0)
return -1;
}
}
if (unrecoverable_error != 0)
return -1;
return 0;
}
void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount)
{
char path[PATH_MAX];
char parity_path[PATH_MAX];
char qarity_path[PATH_MAX];
block_off_t blockmax;
data_off_t size;
int ret;
int f;
unsigned unrecoverable_error;
int parity_f;
int qarity_f;
unsigned error;
if (fix)
printf("Checking and fixing...\n");
@ -388,48 +518,81 @@ void state_check(struct snapraid_state* state, int fix, block_off_t blockstart,
blockmax = blockstart + blockcount;
}
pathcpy(path, sizeof(path), state->parity);
pathcpy(parity_path, sizeof(parity_path), state->parity);
if (fix) {
/* if fixing, create the file and open for writing */
/* if it fails, we cannot continue */
f = parity_create(path, size);
if (f == -1) {
fprintf(stderr, "WARNING! Without an accessible parity file, it isn't possible to fix any error.\n");
parity_f = parity_create(parity_path, size);
if (parity_f == -1) {
fprintf(stderr, "WARNING! Without an accessible Parity file, it isn't possible to fix any error.\n");
exit(EXIT_FAILURE);
}
if (state->level >= 2) {
pathcpy(qarity_path, sizeof(qarity_path), state->qarity);
qarity_f = parity_create(qarity_path, size);
if (qarity_f == -1) {
fprintf(stderr, "WARNING! Without an accessible Q-Parity file, it isn't possible to fix any error.\n");
exit(EXIT_FAILURE);
}
} else {
qarity_f = -1;
}
} else {
/* if checking, open the file for reading */
/* it may fail if the file doesn't exist, in this case we continue to check the files */
f = parity_open(path);
if (f == -1) {
printf("No accessible parity file, all the errors are going to be UNRECOVERABLE.\n");
parity_f = parity_open(parity_path);
if (parity_f == -1) {
printf("No accessible Parity file, only files will be checked.\n");
/* continue anyway */
}
if (state->level >= 2) {
pathcpy(qarity_path, sizeof(qarity_path), state->qarity);
qarity_f = parity_open(qarity_path);
if (qarity_f == -1) {
printf("No accessible Q-Parity file, only files will be checked.\n");
/* continue anyway */
}
} else {
qarity_f = -1;
}
}
unrecoverable_error = 0;
error = 0;
/* skip degenerated cases of empty parity, or skipping all */
if (blockstart < blockmax) {
ret = state_check_process(state, fix, f, blockstart, blockmax);
ret = state_check_process(state, fix, parity_f, qarity_f, blockstart, blockmax);
if (ret == -1) {
++unrecoverable_error;
++error;
/* continue, as we are already exiting */
}
}
/* try to close only if opened */
if (f != -1) {
ret = parity_close(path, f);
if (parity_f != -1) {
ret = parity_close(parity_path, parity_f);
if (ret == -1) {
fprintf(stderr, "WARNING! A 'snapraid check' command is highly suggested.\n");
++unrecoverable_error;
++error;
/* continue, as we are already exiting */
}
}
/* abort if required */
if (unrecoverable_error != 0)
if (state->level >= 2) {
if (qarity_f != -1) {
ret = parity_close(qarity_path, qarity_f);
if (ret == -1) {
fprintf(stderr, "WARNING! A 'snapraid check' command is highly suggested.\n");
++error;
/* continue, as we are already exiting */
}
}
}
/* abort if error are present */
if (error != 0)
exit(EXIT_FAILURE);
}

2
configure.windows Executable file
View file

@ -0,0 +1,2 @@
./configure --host=i686-pc-mingw32 --build=`./config.guess` $@

57
cpu.h Normal file
View file

@ -0,0 +1,57 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __CPU_H
#define __CPU_H
#if defined(__i386__) || defined(__x86_64__)
#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */
#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions (fast save and restore) */
#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */
#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */
#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */
static inline int cpu_has(uint32_t flag)
{
uint32_t eax = (flag >> 5) ? 0x80000001 : 1;
uint32_t edx;
asm volatile("cpuid"
: "+a" (eax), "=d" (edx)
: : "ecx", "ebx");
return (edx >> (flag & 31)) & 1;
}
static inline int cpu_has_mmx(void)
{
return cpu_has(X86_FEATURE_MMX);
}
static inline int cpu_has_sse2(void)
{
return cpu_has(X86_FEATURE_MMX) &&
cpu_has(X86_FEATURE_FXSR) &&
cpu_has(X86_FEATURE_XMM) &&
cpu_has(X86_FEATURE_XMM2);
}
#endif
#endif

71
dry.c
View file

@ -26,13 +26,13 @@
/****************************************************************************/
/* dry */
static int state_dry_process(struct snapraid_state* state, int parity_f, block_off_t blockstart, block_off_t blockmax)
static int state_dry_process(struct snapraid_state* state, int parity_f, int qarity_f, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_handle* handle;
unsigned diskmax = tommy_array_size(&state->diskarr);
block_off_t i;
unsigned j;
unsigned char* block_buffer;
unsigned char* buffer;
int ret;
data_off_t countsize;
block_off_t countpos;
@ -41,7 +41,7 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, block_o
time_t last;
unsigned error;
block_buffer = malloc_nofail(state->block_size);
buffer = malloc_nofail(state->block_size);
handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle));
for(i=0;i<diskmax;++i) {
@ -81,7 +81,7 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, block_o
continue;
}
read_size = handle_read(&handle[j], block, block_buffer, state->block_size);
read_size = handle_read(&handle[j], block, buffer, state->block_size);
if (read_size == -1) {
fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block));
++error;
@ -92,12 +92,22 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, block_o
}
/* read the parity */
ret = parity_read(state->parity, parity_f, i, block_buffer, state->block_size);
ret = parity_read(state->parity, parity_f, i, buffer, state->block_size);
if (ret == -1) {
fprintf(stderr, "%u: Parity read error\n", i);
++error;
}
/* read the qarity */
if (state->level >= 2) {
ret = parity_read(state->qarity, qarity_f, i, buffer, state->block_size);
if (ret == -1) {
fprintf(stderr, "%u: Syndrome read error\n", i);
++error;
}
}
/* count the number of processed block */
++countpos;
@ -131,7 +141,7 @@ bail:
}
free(handle);
free(block_buffer);
free(buffer);
if (error != 0)
return -1;
@ -140,12 +150,14 @@ bail:
void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount)
{
char path[PATH_MAX];
char parity_path[PATH_MAX];
char qarity_path[PATH_MAX];
block_off_t blockmax;
data_off_t size;
int ret;
int f;
unsigned unrecoverable_error;
int parity_f;
int qarity_f;
unsigned error;
printf("Drying...\n");
@ -162,37 +174,58 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t
blockmax = blockstart + blockcount;
}
pathcpy(path, sizeof(path), state->parity);
pathcpy(parity_path, sizeof(parity_path), state->parity);
/* if drying, open the file for reading */
/* it may fail if the file doesn't exist, in this case we continue to dry the files */
f = parity_open(path);
if (f == -1) {
parity_f = parity_open(parity_path);
if (parity_f == -1) {
printf("No accessible parity file.\n");
/* continue anyway */
}
unrecoverable_error = 0;
if (state->level >= 2) {
pathcpy(qarity_path, sizeof(qarity_path), state->qarity);
qarity_f = parity_open(qarity_path);
if (qarity_f == -1) {
printf("No accessible qarity file.\n");
/* continue anyway */
}
} else {
qarity_f = -1;
}
error = 0;
/* skip degenerated cases of empty parity, or skipping all */
if (blockstart < blockmax) {
ret = state_dry_process(state, f, blockstart, blockmax);
ret = state_dry_process(state, parity_f, qarity_f, blockstart, blockmax);
if (ret == -1) {
++unrecoverable_error;
++error;
/* continue, as we are already exiting */
}
}
/* try to close only if opened */
if (f != -1) {
ret = parity_close(path, f);
if (parity_f != -1) {
ret = parity_close(parity_path, parity_f);
if (ret == -1) {
++unrecoverable_error;
++error;
/* continue, as we are already exiting */
}
}
if (state->level >= 2) {
if (qarity_f != -1) {
ret = parity_close(qarity_path, qarity_f);
if (ret == -1) {
++error;
/* continue, as we are already exiting */
}
}
}
/* abort if required */
if (unrecoverable_error != 0)
if (error != 0)
exit(EXIT_FAILURE);
}

5
handle.c
View file

@ -256,6 +256,11 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un
return -1;
}
/* pad with 0 */
if (read_size < block_size) {
memset(block_buffer + read_size, 0, block_size - read_size);
}
/* Here isn't needed to call posix_fadvise(..., POSIX_FADV_DONTNEED) because */
/* we already advised sequential access with POSIX_FADV_SEQUENTIAL. */
/* In Linux 2.6.33 it's enough to ensure that data is not kept in the cache. */

1
handle.h
View file

@ -53,6 +53,7 @@ int handle_close(struct snapraid_handle* handle);
/**
* Read a block from a file.
* If the read block is shorter, it's padded with 0.
*/
int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size);

155
murmurhash3.c Normal file
View file

@ -0,0 +1,155 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* Derivative work from MurmorHash3.cpp from http://code.google.com/p/smhasher/ */
// MurmurHash3 was written by Austin Appleby, and is placed in the public
// domain. The author hereby disclaims copyright to this source code.
// Note - The x86 and x64 versions do _not_ produce the same results, as the
// algorithms are optimized for their respective platforms. You can still
// compile and run any of them on any platform, but your performance with the
// non-native version will be less than optimal.
/* Rotate left */
inline uint32_t rotl32 ( uint32_t x, int8_t r )
{
return (x << r) | (x >> (32 - r));
}
/* Swap endianess */
#if WORDS_BIGENDIAN
static inline uint32_t swap32(uint32_t v)
{
return (((v) & 0xff) << 24 | ((v) & 0xff00) << 8 | ((v) & 0xff0000) >> 8 | ((v) >> 24) & 0xff);
}
#else
static inline uint32_t swap32(uint32_t v)
{
return v;
}
#endif
/* Finalization mix - force all bits of a hash block to avalanche */
static inline uint32_t fmix32(uint32_t h)
{
h ^= h >> 16;
h *= 0x85ebca6b;
h ^= h >> 13;
h *= 0xc2b2ae35;
h ^= h >> 16;
return h;
}
uint32_t c1 = 0x239b961b;
uint32_t c2 = 0xab0e9789;
uint32_t c3 = 0x38b34ae5;
uint32_t c4 = 0xa1e38b93;
void MurmurHash3_x86_128(const void * key, unsigned len, uint32_t seed, void* out)
{
const uint8_t * data = (const uint8_t*)key;
unsigned nblocks = len / 16;
unsigned i;
uint32_t h1 = seed;
uint32_t h2 = seed;
uint32_t h3 = seed;
uint32_t h4 = seed;
const uint32_t* blocks = (const uint32_t*)data;
/* body */
for(i=0;i<nblocks;++i) {
uint32_t k1 = swap32(blocks[0]);
uint32_t k2 = swap32(blocks[1]);
uint32_t k3 = swap32(blocks[2]);
uint32_t k4 = swap32(blocks[3]);
k1 *= c1; k1 = rotl32(k1,15); k1 *= c2; h1 ^= k1;
h1 = rotl32(h1,19); h1 += h2; h1 = h1*5+0x561ccd1b;
k2 *= c2; k2 = rotl32(k2,16); k2 *= c3; h2 ^= k2;
h2 = rotl32(h2,17); h2 += h3; h2 = h2*5+0x0bcaa747;
k3 *= c3; k3 = rotl32(k3,17); k3 *= c4; h3 ^= k3;
h3 = rotl32(h3,15); h3 += h4; h3 = h3*5+0x96cd1c35;
k4 *= c4; k4 = rotl32(k4,18); k4 *= c1; h4 ^= k4;
h4 = rotl32(h4,13); h4 += h1; h4 = h4*5+0x32ac3b17;
blocks += 4;
}
/* tail */
{
const uint8_t* tail = (const uint8_t*)blocks;
uint32_t k1 = 0;
uint32_t k2 = 0;
uint32_t k3 = 0;
uint32_t k4 = 0;
switch (len & 15) {
case 15 : k4 ^= tail[14] << 16;
case 14 : k4 ^= tail[13] << 8;
case 13 : k4 ^= tail[12] << 0;
k4 *= c4; k4 = rotl32(k4,18); k4 *= c1; h4 ^= k4;
case 12 : k3 ^= tail[11] << 24;
case 11 : k3 ^= tail[10] << 16;
case 10 : k3 ^= tail[ 9] << 8;
case 9 : k3 ^= tail[ 8] << 0;
k3 *= c3; k3 = rotl32(k3,17); k3 *= c4; h3 ^= k3;
case 8 : k2 ^= tail[ 7] << 24;
case 7 : k2 ^= tail[ 6] << 16;
case 6 : k2 ^= tail[ 5] << 8;
case 5 : k2 ^= tail[ 4] << 0;
k2 *= c2; k2 = rotl32(k2,16); k2 *= c3; h2 ^= k2;
case 4 : k1 ^= tail[ 3] << 24;
case 3 : k1 ^= tail[ 2] << 16;
case 2 : k1 ^= tail[ 1] << 8;
case 1 : k1 ^= tail[ 0] << 0;
k1 *= c1; k1 = rotl32(k1,15); k1 *= c2; h1 ^= k1;
};
}
/* finalization */
h1 ^= len; h2 ^= len; h3 ^= len; h4 ^= len;
h1 += h2; h1 += h3; h1 += h4;
h2 += h1; h3 += h1; h4 += h1;
h1 = fmix32(h1);
h2 = fmix32(h2);
h3 = fmix32(h3);
h4 = fmix32(h4);
h1 += h2; h1 += h3; h1 += h4;
h2 += h1; h3 += h1; h4 += h1;
((uint32_t*)out)[0] = swap32(h1);
((uint32_t*)out)[1] = swap32(h2);
((uint32_t*)out)[2] = swap32(h3);
((uint32_t*)out)[3] = swap32(h4);
}

331
raid.c Normal file
View file

@ -0,0 +1,331 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
/* -*- linux-c -*- ------------------------------------------------------- *
*
* Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, Inc., 53 Temple Place Ste 330,
* Boston MA 02111-1307, USA; either version 2 of the License, or
* (at your option) any later version; incorporated herein by reference.
*
* ----------------------------------------------------------------------- */
#include "portable.h"
#include "raid.h"
#include "cpu.h"
/*
* 2-way unrolled portable integer math RAID-5 instruction set
*/
void raid5_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size)
{
unsigned char* p;
int z, z0;
unsigned d;
uint32_t wp0;
uint32_t wp1;
z0 = diskmax - 2; /* Highest data disk */
p = dptr[z0+1]; /* XOR parity */
for(d=0;d<block_size;d+=8) {
wp0 = *(uint32_t *)&dptr[z0][d+0*4];
wp1 = *(uint32_t *)&dptr[z0][d+1*4];
for(z=z0-1;z>=0;--z) {
wp0 ^= *(uint32_t *)&dptr[z][d+0*4];
wp1 ^= *(uint32_t *)&dptr[z][d+1*4];
}
*(uint32_t *)&p[d+4*0] = wp0;
*(uint32_t *)&p[d+4*1] = wp1;
}
}
/*
* Unrolled-by-2 MMX implementation
*/
void raid5_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size)
{
unsigned char* p;
int z, z0;
unsigned d;
z0 = diskmax - 2; /* Highest data disk */
p = dptr[z0+1]; /* XOR parity */
for(d=0;d<block_size;d+=16) {
asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d]));
asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
for(z=z0-1;z>=0;--z) {
asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
asm volatile("pxor %mm5,%mm2");
asm volatile("pxor %mm7,%mm3");
}
asm volatile("movq %%mm2,%0" : "=m" (p[d]));
asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
}
asm volatile("emms" : : : "memory");
}
void raid5_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size)
{
unsigned char* p;
int z, z0;
unsigned d;
z0 = diskmax - 2; /* Highest data disk */
p = dptr[z0+1]; /* XOR parity */
for(d=0;d<block_size;d+=32) {
asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d]));
asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16]));
for(z=z0-1;z>=0;--z) {
asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
asm volatile("pxor %xmm5,%xmm2");
asm volatile("pxor %xmm7,%xmm3");
}
asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
}
asm volatile("sfence" : : : "memory");
}
/*
* Repeats an 8 bit nibble to a full 32 bits value.
*/
#define NBYTES(x) ((x) * 0x01010101U)
/*
* These sub-operations are separate inlines since they can sometimes be
* specially optimized using architecture-specific hacks.
*/
/*
* The SHLBYTE() operation shifts each byte left by 1, *not*
* rolling over into the next byte.
*/
static inline uint32_t SHLBYTE(uint32_t v)
{
uint32_t vv;
vv = (v << 1) & NBYTES(0xfe);
return vv;
}
/*
* The MASK() operation returns 0xFF in any byte for which the high
* bit is 1, 0x00 for any byte for which the high bit is 0.
*/
static inline uint32_t MASK(uint32_t v)
{
uint32_t vv;
vv = v & NBYTES(0x80);
vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */
return vv;
}
/*
* 2-way unrolled portable integer math RAID-6 instruction set
*/
void raid6_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size)
{
unsigned char* p;
unsigned char* q;
int z, z0;
unsigned d;
uint32_t wd0, wq0, wp0, w10, w20;
uint32_t wd1, wq1, wp1, w11, w21;
z0 = diskmax - 3; /* Highest data disk */
p = dptr[z0+1]; /* XOR parity */
q = dptr[z0+2]; /* RS qarity */
for(d=0;d<block_size;d+=8) {
wq0 = wp0 = *(uint32_t *)&dptr[z0][d+0*4];
wq1 = wp1 = *(uint32_t *)&dptr[z0][d+1*4];
for (z=z0-1;z>= 0;--z) {
wd0 = *(uint32_t *)&dptr[z][d+0*4];
wd1 = *(uint32_t *)&dptr[z][d+1*4];
wp0 ^= wd0;
wp1 ^= wd1;
w20 = MASK(wq0);
w21 = MASK(wq1);
w10 = SHLBYTE(wq0);
w11 = SHLBYTE(wq1);
w20 &= NBYTES(0x1d);
w21 &= NBYTES(0x1d);
w10 ^= w20;
w11 ^= w21;
wq0 = w10 ^ wd0;
wq1 = w11 ^ wd1;
}
*(uint32_t *)&p[d+4*0] = wp0;
*(uint32_t *)&p[d+4*1] = wp1;
*(uint32_t *)&q[d+4*0] = wq0;
*(uint32_t *)&q[d+4*1] = wq1;
}
}
#if defined(__i386__) || defined(__x86_64__)
static const struct raid6_mmx_constants {
uint64_t x1d;
} raid6_mmx_constants = {
0x1d1d1d1d1d1d1d1dULL,
};
/*
* Unrolled-by-2 MMX implementation
*/
void raid6_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size)
{
unsigned char* p;
unsigned char* q;
int z, z0;
unsigned d;
z0 = diskmax - 3; /* Highest data disk */
p = dptr[z0+1]; /* XOR parity */
q = dptr[z0+2]; /* RS qarity */
asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
asm volatile("pxor %mm5,%mm5"); /* Zero temp */
asm volatile("pxor %mm7,%mm7"); /* Zero temp */
for(d=0;d<block_size;d+=16) {
asm volatile("movq %0,%%mm2" : : "m" (dptr[z0][d]));
asm volatile("movq %0,%%mm3" : : "m" (dptr[z0][d+8]));
asm volatile("movq %mm2,%mm4");
asm volatile("movq %mm3,%mm6");
for(z=z0-1;z>=0;--z) {
asm volatile("pcmpgtb %mm4,%mm5");
asm volatile("pcmpgtb %mm6,%mm7");
asm volatile("paddb %mm4,%mm4");
asm volatile("paddb %mm6,%mm6");
asm volatile("pand %mm0,%mm5");
asm volatile("pand %mm0,%mm7");
asm volatile("pxor %mm5,%mm4");
asm volatile("pxor %mm7,%mm6");
asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d]));
asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8]));
asm volatile("pxor %mm5,%mm2");
asm volatile("pxor %mm7,%mm3");
asm volatile("pxor %mm5,%mm4");
asm volatile("pxor %mm7,%mm6");
asm volatile("pxor %mm5,%mm5");
asm volatile("pxor %mm7,%mm7");
}
asm volatile("movq %%mm2,%0" : "=m" (p[d]));
asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
asm volatile("movq %%mm4,%0" : "=m" (q[d]));
asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
}
asm volatile("emms" : : : "memory");
}
static const struct raid6_sse_constants {
uint64_t x1d[2];
} raid6_sse_constants __attribute__((aligned(16))) = {
{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
};
/*
* Unrolled-by-2 SSE2 implementation
*/
void raid6_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size)
{
unsigned char* p;
unsigned char* q;
int z, z0;
unsigned d;
z0 = diskmax - 3; /* Highest data disk */
p = dptr[z0+1]; /* XOR parity */
q = dptr[z0+2]; /* RS qarity */
asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0]));
asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */
asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */
/* We uniformly assume a single prefetch covers at least 32 bytes */
for(d=0;d<block_size;d+=32) {
asm volatile("prefetchnta %0" : : "m" (dptr[z0][d]));
asm volatile("movdqa %0,%%xmm2" : : "m" (dptr[z0][d])); /* P[0] */
asm volatile("movdqa %0,%%xmm3" : : "m" (dptr[z0][d+16])); /* P[1] */
asm volatile("movdqa %xmm2,%xmm4"); /* Q[0] */
asm volatile("movdqa %xmm3,%xmm6"); /* Q[1] */
for(z=z0-1;z>=0;--z) {
asm volatile("prefetchnta %0" : : "m" (dptr[z][d]));
asm volatile("pcmpgtb %xmm4,%xmm5");
asm volatile("pcmpgtb %xmm6,%xmm7");
asm volatile("paddb %xmm4,%xmm4");
asm volatile("paddb %xmm6,%xmm6");
asm volatile("pand %xmm0,%xmm5");
asm volatile("pand %xmm0,%xmm7");
asm volatile("pxor %xmm5,%xmm4");
asm volatile("pxor %xmm7,%xmm6");
asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d]));
asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16]));
asm volatile("pxor %xmm5,%xmm2");
asm volatile("pxor %xmm7,%xmm3");
asm volatile("pxor %xmm5,%xmm4");
asm volatile("pxor %xmm7,%xmm6");
asm volatile("pxor %xmm5,%xmm5");
asm volatile("pxor %xmm7,%xmm7");
}
asm volatile("movntdq %%xmm2,%0" : "=m" (p[d]));
asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16]));
asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16]));
}
asm volatile("sfence" : : : "memory");
}
#endif
void (*raid5_gen)(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void (*raid6_gen)(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void raid_init(void)
{
raid5_gen = raid5_int32r2;
raid6_gen = raid6_int32r2;
#if defined(__i386__) || defined(__x86_64__)
/*
if (cpu_has_mmx()) {
raid5_gen = raid5_mmxr2;
raid6_gen = raid6_mmxr2;
}
if (cpu_has_sse2()) {
raid5_gen = raid5_sse2r2;
raid6_gen = raid6_sse2r2;
}
*/
#endif
}

41
raid.h Normal file
View file

@ -0,0 +1,41 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __RAID_H
#define __RAID_H
/****************************************************************************/
/* raid */
void raid5_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void raid5_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void raid5_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void raid6_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void raid6_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size);
void raid6_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size);
extern void (*raid5_gen)(uint8_t** ptrs, unsigned disks, unsigned block_size);
extern void (*raid6_gen)(uint8_t** ptrs, unsigned disks, unsigned block_size);
/**
* Initializes the RAID system.
*/
void raid_init(void);
#endif

111
selftest.c Normal file
View file

@ -0,0 +1,111 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "portable.h"
#include "snapraid.h"
#include "util.h"
#include "raid.h"
struct test_vector {
const char* data;
unsigned char digest[HASH_SIZE];
};
/**
* Test vectors for MD5 from RFC1321.
*
* MD5("") = d41d8cd98f00b204e9800998ecf8427e
* MD5("a") = 0cc175b9c0f1b6a831c399e269772661
* MD5("abc") = 900150983cd24fb0d6963f7d28e17f72
* MD5("message digest") = f96b697d7cb7938d525a2f31aaf161d0
* MD5("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
* MD5("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
* MD5("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
*/
struct test_vector TEST_MD5[] = {
{ "", { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } },
{ "a", { 0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8, 0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61 } },
{ "abc", { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } },
{ "message digest", { 0xf9, 0x6b, 0x69, 0x7d, 0x7c, 0xb7, 0x93, 0x8d, 0x52, 0x5a, 0x2f, 0x31, 0xaa, 0xf1, 0x61, 0xd0 } },
{ "abcdefghijklmnopqrstuvwxyz", { 0xc3, 0xfc, 0xd3, 0xd7, 0x61, 0x92, 0xe4, 0x00, 0x7d, 0xfb, 0x49, 0x6c, 0xca, 0x67, 0xe1, 0x3b } },
{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xd1, 0x74, 0xab, 0x98, 0xd2, 0x77, 0xd9, 0xf5, 0xa5, 0x61, 0x1c, 0x2c, 0x9f, 0x41, 0x9d, 0x9f } },
{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0x57, 0xed, 0xf4, 0xa2, 0x2b, 0xe3, 0xc9, 0x55, 0xac, 0x49, 0xda, 0x2e, 0x21, 0x07, 0xb6, 0x7a, } },
{ 0, { 0 } }
};
/**
* Test vectors for MurmorHash3_x86_128
*/
struct test_vector TEST_MURMUR3[] = {
{ "", { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } },
{ "a", { 0x3c, 0x93, 0x94, 0xa7, 0x1b, 0xb0, 0x56, 0x55, 0x1b, 0xb0, 0x56, 0x55, 0x1b, 0xb0, 0x56, 0x55 } },
{ "abc", { 0xd1, 0xc6, 0xcd, 0x75, 0xa5, 0x06, 0xb0, 0xa2, 0xa5, 0x06, 0xb0, 0xa2, 0xa5, 0x06, 0xb0, 0xa2 } },
{ "message digest", { 0xea, 0x7b, 0x82, 0xc1, 0x89, 0x77, 0xdb, 0x26, 0xb0, 0x30, 0xfe, 0xbc, 0xef, 0x42, 0x4b, 0xda } },
{ "abcdefghijklmnopqrstuvwxyz", { 0x13, 0x06, 0x34, 0x3e, 0x66, 0x2f, 0x6f, 0x66, 0x6e, 0x56, 0xf6, 0x17, 0x2c, 0x3d, 0xe3, 0x44 } },
{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xe7, 0xd6, 0x62, 0x29, 0x32, 0xf7, 0xdd, 0x48, 0x29, 0x38, 0xed, 0x07, 0x32, 0x0c, 0x72, 0x12 } },
{ "1234567890123456789012345678901234567890123456789012345678901234", { 0x8e, 0x2e, 0xd6, 0x0c, 0xbb, 0x4d, 0x10, 0x7e, 0x7a, 0x78, 0x72, 0x2e, 0xab, 0x9a, 0x6c, 0x0d } },
{ "12345678901234567890123456789012345678901234567890123456789012345", { 0x45, 0xe6, 0x26, 0xa0, 0xf5, 0xdb, 0xf8, 0x22, 0x3b, 0x5e, 0xc5, 0x41, 0x10, 0xc5, 0x06, 0x36 } },
{ "123456789012345678901234567890123456789012345678901234567890123456", { 0xb5, 0x5e, 0xdd, 0xa2, 0xa6, 0x65, 0xfd, 0x53, 0xec, 0xb8, 0xc9, 0x93, 0x97, 0x97, 0x65, 0x22 } },
{ "1234567890123456789012345678901234567890123456789012345678901234567", { 0x19, 0x40, 0x6f, 0xb8, 0xc4, 0x30, 0x59, 0x00, 0x56, 0x10, 0xe7, 0xdd, 0x9a, 0x48, 0x83, 0xa5 } },
{ "12345678901234567890123456789012345678901234567890123456789012345678", { 0x77, 0xa8, 0xb1, 0xe6, 0xb2, 0xbc, 0x2c, 0x4f, 0x91, 0x0c, 0x32, 0xb3, 0x1d, 0x28, 0xf2, 0xd4 } },
{ "123456789012345678901234567890123456789012345678901234567890123456789", { 0xe0, 0x5a, 0x5f, 0xe7, 0x0c, 0x2a, 0xa9, 0x63, 0x9a, 0x8e, 0x41, 0xbe, 0x73, 0xa3, 0x25, 0xe4 } },
{ "1234567890123456789012345678901234567890123456789012345678901234567890", { 0x8e, 0x8f, 0x93, 0xcf, 0xdf, 0x1f, 0xe5, 0xfe, 0x09, 0x35, 0xe3, 0x53, 0x1e, 0x5b, 0x75, 0x37 } },
{ "12345678901234567890123456789012345678901234567890123456789012345678901", { 0x46, 0xd5, 0xd2, 0xa3, 0x6d, 0xc9, 0x20, 0x0d, 0xd6, 0x6a, 0x90, 0x9e, 0xda, 0xfb, 0x1c, 0x26 } },
{ "123456789012345678901234567890123456789012345678901234567890123456789012", { 0x4a, 0x26, 0xe4, 0xae, 0x00, 0x32, 0x08, 0xfd, 0x0b, 0xee, 0x6c, 0x0f, 0xb2, 0x64, 0xa4, 0x25 } },
{ "1234567890123456789012345678901234567890123456789012345678901234567890123", { 0x2a, 0x97, 0x3f, 0xe8, 0xb4, 0xf9, 0x4e, 0xf9, 0x45, 0xe2, 0xe4, 0x90, 0x69, 0x14, 0xa6, 0x24 } },
{ "12345678901234567890123456789012345678901234567890123456789012345678901234", { 0x4b, 0xd7, 0x48, 0x1a, 0x25, 0x8a, 0x2a, 0xab, 0x50, 0xc3, 0xc6, 0xd0, 0x24, 0x62, 0x78, 0xba } },
{ "123456789012345678901234567890123456789012345678901234567890123456789012345", { 0x88, 0xe4, 0x8f, 0x6f, 0xee, 0x22, 0xe5, 0x89, 0x78, 0x24, 0x99, 0x07, 0x73, 0x7e, 0x5b, 0xfd } },
{ "1234567890123456789012345678901234567890123456789012345678901234567890123456", { 0xdd, 0x18, 0x02, 0x02, 0xa9, 0xcd, 0x6e, 0xcb, 0xfe, 0x91, 0x8e, 0x82, 0x4e, 0xa8, 0x33, 0xfe } },
{ "12345678901234567890123456789012345678901234567890123456789012345678901234567", { 0x80, 0xf2, 0xb8, 0x67, 0x5a, 0x02, 0x78, 0x0c, 0xcb, 0x31, 0xd0, 0xf3, 0xb6, 0x67, 0x52, 0xca } },
{ "123456789012345678901234567890123456789012345678901234567890123456789012345678", { 0x66, 0x85, 0x66, 0x0f, 0xc7, 0x56, 0xfb, 0xb7, 0x9b, 0xa2, 0x28, 0x37, 0x54, 0x89, 0xbb, 0x48 } },
{ "1234567890123456789012345678901234567890123456789012345678901234567890123456789", { 0x5b, 0x59, 0x21, 0x42, 0xae, 0x72, 0xbe, 0xd4, 0x4a, 0xa7, 0xf2, 0x12, 0x35, 0x48, 0xc6, 0xd8 } },
{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0xf2, 0x6c, 0x2a, 0x8f, 0x58, 0x23, 0x4c, 0x8e, 0x2c, 0x74, 0x41, 0x66, 0x0e, 0xc2, 0xd3, 0x91 } },
{ 0, { 0 } }
};
void selftest(void)
{
unsigned i;
printf("Self test...\n");
/* large file check */
if (sizeof(off_t) < sizeof(uint64_t)) {
fprintf(stderr, "Missing support for large files\n");
exit(EXIT_FAILURE);
}
for(i=0;TEST_MD5[i].data;++i) {
unsigned char digest[HASH_SIZE];
memhash(HASH_MD5, digest, TEST_MD5[i].data, strlen(TEST_MD5[i].data));
if (memcmp(digest, TEST_MD5[i].digest, HASH_SIZE) != 0) {
fprintf(stderr, "Failed MD5 test vector\n");
exit(EXIT_FAILURE);
}
}
for(i=0;TEST_MURMUR3[i].data;++i) {
unsigned char digest[HASH_SIZE];
memhash(HASH_MURMUR3, digest, TEST_MURMUR3[i].data, strlen(TEST_MURMUR3[i].data));
if (memcmp(digest, TEST_MURMUR3[i].digest, HASH_SIZE) != 0) {
fprintf(stderr, "Failed Murmur3 test vector\n");
exit(EXIT_FAILURE);
}
}
}

82
snapraid.1
View file

@ -23,7 +23,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hyf, \(hy\(hyfilter PATTERN]
.PP
.PD
.PP
snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp] [\(hyT, \(hy\(hyspeed\(hytest]
snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp]
.PD 0
.PP
.PD
@ -31,7 +31,7 @@ snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp] [\(hyT, \(hy\(hyspeed\(h
SnapRAID is a backup program for a disk array using redundancy.
.PP
SnapRAID uses a disk of the array to store redundancy information,
and it allows to recover from a disk failure.
and it allows to recover from up to two disk failures.
.PP
SnapRAID is mainly targeted for a home media center, where you have
a lot of big files that rarely change.
@ -49,7 +49,7 @@ You can add more disks at the array at any time.
If you accidentally delete some files in a disk, you can
recover them.
.IP \(bu
If more than one disk fails, you lose the data only on the
If more than two disk fails, you lose the data only on the
failed disks. All the data in the other disks is safe.
.IP \(bu
It doesn\(cqt lock\(hyin your data. You can stop using SnapRAID at any
@ -96,7 +96,7 @@ With RAID you get a speed boost with striping.
It doesn\(cqt support real\(hytime recovery.
With RAID you do not have to stop working when a disk fails.
.IP \(bu
It\(cqs able to recover damages only from a single disk.
It\(cqs able to recover damages only from up to two disks.
With a Backup you are able to recover from a complete
failure of the whole disk array.
.IP \(bu
@ -109,18 +109,25 @@ hard\(hylinks, symbolic\(hylinks are not saved.
.PD
.SH GETTING STARTED
To use SnapRAID you need first select one disk of your disk array
to dedicate at the redundancy information.
to dedicate at the parity information. With parity you will be able
to recover from a single disk failure.
.PP
You have to pick the biggest disk in the array, as the redundancy
information may grow in size as the biggest data disk in the array.
If you want to recover from two disk failures you must reserve
another disk for the q\(hyparity information.
.PP
This disk will be dedicated to store the parity and content metadata
information. You should not store other data on it.
For both disks, you have to pick the biggest disks in the array,
as the redundancy information may grow in size as the biggest data
disk in the array.
.PP
For example, if all your disks are present in:
This disk will be dedicated to store the parity and q\(hyparity.
You should not store other data on it, with the exception of
the content metadata information.
.PP
For example, suppose that you are interested only at the parity level
of protection, and that all your disks are present in:
.PP
.RS 4
/mnt/diskpar <\(hy selected disk for parity and metadata content
/mnt/diskpar <\(hy selected disk for parity
.PD 0
.PP
.PD
@ -258,7 +265,7 @@ meaning that you can move them on the disk without triggering
any redundancy recomputation.
You can stop this process at any time pressing Ctrl+C,
without losing the work already done.
The \(a"content\(a" and \(a"parity\(a" files are modified if necessary.
The \(a"content\(a", \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary.
The files in the array are NOT modified.
.TP
.B check
@ -275,7 +282,7 @@ disk array at the previous \(a"sync\(a" command.
After a successful \(a"fix\(a", you should also run a \(a"sync\(a"
command to update the new state of the files.
The \(a"content\(a" file is NOT modified.
The \(a"parity\(a" file is modified if necessary.
The \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary.
The files in the array are modified if necessary.
.SH OPTIONS
.TP
@ -315,17 +322,11 @@ or fix some specific block, in case of a damaged disk.
.TP
.B \(hyt, \(hy\(hycount BLKCOUNT
Process only the specified number of blocks.
It\(cqs present mainly for testing.
It\(cqs present mainly for advanced manual recovering.
.TP
.B \(hyv, \(hy\(hyverbose
Prints more information in the processing.
.TP
.B \(hyT, \(hy\(hyspeed\(hytest
Executes a speed test of the MD5 implementation used.
If available, SnapRAID uses the fast MD5 implementation
from the OpenSSL crypto library. Otherwise it uses a
slower C implementation.
.TP
.B \(hyh, \(hy\(hyhelp
Prints a short help screen.
.TP
@ -339,19 +340,29 @@ This configuration file is located in /etc/snapraid.conf and
it should contains the following options:
.TP
.B parity FILE
Defines the file to use to store the redundancy information.
Defines the file to use to store the parity information.
It must be placed in a disk dedicated for this purpose with
as much free space as the biggest disk in the array.
Leaving the parity disk reserved for only this file, ensures that
it doesn\(cqt get fragmented, improving the performance.
This option can be used only one time.
This option is mandatory and it can be used only one time.
.TP
.B q\(hyparity FILE
Defines the file to use to store the q\(hyparity information.
This option is optional, and if present it enables the protection
from two disk failures.
It must be placed in a disk dedicated for this purpose with
as much free space as the biggest disk in the array.
Leaving the q\(hyparity disk reserved for only this file, ensures that
it doesn\(cqt get fragmented, improving the performance.
This option is optional and it can be used only one time.
.TP
.B content FILE
Defines the file to use to store the content of the redundancy
organization.
It can be placed in the same disk of the parity file, or better
in another disk, but NOT in a data disk of the array.
This option can be used only one time.
This option is mandatory and it can be used only one time.
.TP
.B disk NAME DIR
Defines the name and the mount point of the disks of the array.
@ -550,7 +561,7 @@ to identify the file.
Defines the ordered parity block list used by the last defined file.
BLOCK is the block position in the \(a"parity\(a" file.
0 for the first block, 1 for the second one and so on.
HASH is the md5 of the block. In the last block of the file,
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
.TP
.B inv BLOCK [HASH]
@ -559,18 +570,25 @@ The HASH may be missing if not yet computed.
This field is used only when you interrupt manually the \(a"sync\(a"
command.
.SH PARITY
SnapRAID creates a parity file containing the redundancy information
of your disk array.
SnapRAID creates the parity and q\(hyparity files containing the redundancy
information of your disk array.
.PP
It\(cqs a binary file, containing the computed parity of all the blocks
defined in the \(a"content\(a" file. You do not need to understand its format,
but it\(cqs described here for documentation.
They are binary files, containing the computed redundancy of all the
blocks defined in the \(a"content\(a" file. You do not need to understand its
format, but it\(cqs described here for documentation.
.PP
This file is read and written by the \(a"sync\(a" and \(a"fix\(a" commands, and
These files are read and written by the \(a"sync\(a" and \(a"fix\(a" commands, and
only read by \(a"check\(a".
.PP
For all the blocks at a given position, the parity information is
computed with the XOR operator applied to all the blocks.
For all the blocks at a given position, the parity and the q\(hyparity
are computed as specified in:
.PP
.RS 4
kernel.org/pub/linux/kernel/people/hpa/raid6.pdf
.PD 0
.PP
.PD
.RE
.PP
When a file block is shorter than the default block size, for example
because it\(cqs the last block of a file, it\(cqs assumed as filled with 0

106
snapraid.c
View file

@ -17,9 +17,11 @@
#include "portable.h"
#include "snapraid.h"
#include "util.h"
#include "elem.h"
#include "state.h"
#include "raid.h"
/****************************************************************************/
/* main */
@ -33,91 +35,6 @@
#define CONF "/etc/" PACKAGE ".conf"
#endif
/**
* Test vectors for MD5 from RFC1321.
*
* MD5("") = d41d8cd98f00b204e9800998ecf8427e
* MD5("a") = 0cc175b9c0f1b6a831c399e269772661
* MD5("abc") = 900150983cd24fb0d6963f7d28e17f72
* MD5("message digest") = f96b697d7cb7938d525a2f31aaf161d0
* MD5("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b
* MD5("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f
* MD5("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a
*/
struct test_vector {
const char* data;
unsigned char digest[HASH_SIZE];
} TEST[] = {
{ "", { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } },
{ "a", { 0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8, 0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61 } },
{ "abc", { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } },
{ "message digest", { 0xf9, 0x6b, 0x69, 0x7d, 0x7c, 0xb7, 0x93, 0x8d, 0x52, 0x5a, 0x2f, 0x31, 0xaa, 0xf1, 0x61, 0xd0 } },
{ "abcdefghijklmnopqrstuvwxyz", { 0xc3, 0xfc, 0xd3, 0xd7, 0x61, 0x92, 0xe4, 0x00, 0x7d, 0xfb, 0x49, 0x6c, 0xca, 0x67, 0xe1, 0x3b } },
{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xd1, 0x74, 0xab, 0x98, 0xd2, 0x77, 0xd9, 0xf5, 0xa5, 0x61, 0x1c, 0x2c, 0x9f, 0x41, 0x9d, 0x9f } },
{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0x57, 0xed, 0xf4, 0xa2, 0x2b, 0xe3, 0xc9, 0x55, 0xac, 0x49, 0xda, 0x2e, 0x21, 0x07, 0xb6, 0x7a, } },
{ 0, { 0 } }
};
void selftest(void)
{
unsigned i;
printf("Self test...\n");
/* large file check */
if (sizeof(off_t) < sizeof(uint64_t)) {
fprintf(stderr, "Missing support for large files\n");
exit(EXIT_FAILURE);
}
/* test vectors for MD5. It could be dynamically linked, so we have to test it at runtime */
for(i=0;TEST[i].data;++i) {
unsigned char digest[HASH_SIZE];
memhash(digest, TEST[i].data, strlen(TEST[i].data));
if (memcmp(digest, TEST[i].digest, HASH_SIZE) != 0) {
fprintf(stderr, "Failed MD5 test vector\n");
exit(EXIT_FAILURE);
}
}
}
void speed(void)
{
struct timeval start;
struct timeval stop;
int64_t ds;
int64_t dt;
unsigned i;
unsigned char digest[HASH_SIZE];
unsigned count = 10000;
unsigned blocksize = 256 * 1024;
void* block = malloc_nofail(blocksize);
#if HAVE_LIBCRYPTO
printf("Using OpenSSL libcrypto MD5\n");
#else
printf("Using portable MD5\n");
#endif
#ifdef __GNUC__
printf("Compiled with gcc " __VERSION__ "\n");
#endif
printf("Hashing speed test...\n");
gettimeofday(&start, 0);
for(i=0;i<count;++i)
memhash(digest, block, blocksize);
gettimeofday(&stop, 0);
ds = blocksize * (int64_t)count;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("%llu [MB/s]\n", ds / dt);
free(block);
}
void version(void)
{
printf(PACKAGE " v" VERSION " by Andrea Mazzoleni, " PACKAGE_URL "\n");
@ -136,7 +53,6 @@ void usage(void)
printf(" " SWITCH_GETOPT_LONG("-E, --force-empty ", "-E") " Force synching of disks that get empty\n");
printf(" " SWITCH_GETOPT_LONG("-s, --start BLKSTART", "-s") " Start from the specified block number\n");
printf(" " SWITCH_GETOPT_LONG("-t, --count BLKCOUNT", "-t") " Count of block to process\n");
printf(" " SWITCH_GETOPT_LONG("-T, --speed-test ", "-T") " Speed test of the MD5 implementation\n");
printf(" " SWITCH_GETOPT_LONG("-v, --verbose ", "-v") " Verbose\n");
printf(" " SWITCH_GETOPT_LONG("-h, --help ", "-h") " Help\n");
printf(" " SWITCH_GETOPT_LONG("-V, --version ", "-V") " Version\n");
@ -154,11 +70,13 @@ struct option long_options[] = {
{ "verbose", 0, 0, 'v' },
{ "help", 0, 0, 'h' },
{ "version", 0, 0, 'V' },
{ "expect-unrecoverable", 0, 0, 'U' },
{ "expect-recoverable", 0, 0, 'R' },
{ 0, 0, 0, 0 }
};
#endif
#define OPTIONS "c:f:s:t:ZETvhV"
#define OPTIONS "c:f:s:t:ZEURTvhV"
volatile int global_interrupt = 0;
@ -182,6 +100,8 @@ int main(int argc, char* argv[])
int verbose;
int force_zero;
int force_empty;
int expect_unrecoverable;
int expect_recoverable;
const char* conf;
struct snapraid_state state;
int operation;
@ -195,6 +115,8 @@ int main(int argc, char* argv[])
verbose = 0;
force_zero = 0;
force_empty = 0;
expect_unrecoverable = 0;
expect_recoverable = 0;
blockstart = 0;
blockcount = 0;
tommy_list_init(&filterlist);
@ -237,6 +159,12 @@ int main(int argc, char* argv[])
case 'E' :
force_empty = 1;
break;
case 'U' :
expect_unrecoverable = 1;
break;
case 'R' :
expect_recoverable = 1;
break;
case 'v' :
verbose = 1;
break;
@ -275,9 +203,11 @@ int main(int argc, char* argv[])
selftest();
raid_init();
state_init(&state);
state_config(&state, conf, verbose, force_zero, force_empty);
state_config(&state, conf, verbose, force_zero, force_empty, expect_unrecoverable, expect_recoverable);
if (operation == OPERATION_SYNC) {
if (!tommy_list_empty(&filterlist)) {

77
snapraid.d
View file

@ -8,13 +8,13 @@ Synopsis
: [-v, --verbose]
: COMMAND
:snapraid [-V, --version] [-h, --help] [-T, --speed-test]
:snapraid [-V, --version] [-h, --help]
Description
SnapRAID is a backup program for a disk array using redundancy.
SnapRAID uses a disk of the array to store redundancy information,
and it allows to recover from a disk failure.
and it allows to recover from up to two disk failures.
SnapRAID is mainly targeted for a home media center, where you have
a lot of big files that rarely change.
@ -27,7 +27,7 @@ Description
* You can add more disks at the array at any time.
* If you accidentally delete some files in a disk, you can
recover them.
* If more than one disk fails, you lose the data only on the
* If more than two disk fails, you lose the data only on the
failed disks. All the data in the other disks is safe.
* It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data.
@ -63,7 +63,7 @@ Limitations
With RAID you get a speed boost with striping.
* It doesn't support real-time recovery.
With RAID you do not have to stop working when a disk fails.
* It's able to recover damages only from a single disk.
* It's able to recover damages only from up to two disks.
With a Backup you are able to recover from a complete
failure of the whole disk array.
* It identifies changes checking file time and size. If you have
@ -74,17 +74,24 @@ Limitations
Getting Started
To use SnapRAID you need first select one disk of your disk array
to dedicate at the redundancy information.
to dedicate at the parity information. With parity you will be able
to recover from a single disk failure.
You have to pick the biggest disk in the array, as the redundancy
information may grow in size as the biggest data disk in the array.
If you want to recover from two disk failures you must reserve
another disk for the q-parity information.
This disk will be dedicated to store the parity and content metadata
information. You should not store other data on it.
For both disks, you have to pick the biggest disks in the array,
as the redundancy information may grow in size as the biggest data
disk in the array.
For example, if all your disks are present in:
This disk will be dedicated to store the parity and q-parity.
You should not store other data on it, with the exception of
the content metadata information.
:/mnt/diskpar <- selected disk for parity and metadata content
For example, suppose that you are interested only at the parity level
of protection, and that all your disks are present in:
:/mnt/diskpar <- selected disk for parity
:/mnt/disk1 <- first disk to backup
:/mnt/disk2 <- second disk to backup
:/mnt/disk3 <- third disk to backup
@ -157,7 +164,7 @@ Commands
any redundancy recomputation.
You can stop this process at any time pressing Ctrl+C,
without losing the work already done.
The "content" and "parity" files are modified if necessary.
The "content", "parity" and "q-parity" files are modified if necessary.
The files in the array are NOT modified.
=check
@ -174,7 +181,7 @@ Commands
After a successful "fix", you should also run a "sync"
command to update the new state of the files.
The "content" file is NOT modified.
The "parity" file is modified if necessary.
The "parity" and "q-parity" files are modified if necessary.
The files in the array are modified if necessary.
Options
@ -214,17 +221,11 @@ Options
-t, --count BLKCOUNT
Process only the specified number of blocks.
It's present mainly for testing.
It's present mainly for advanced manual recovering.
-v, --verbose
Prints more information in the processing.
-T, --speed-test
Executes a speed test of the MD5 implementation used.
If available, SnapRAID uses the fast MD5 implementation
from the OpenSSL crypto library. Otherwise it uses a
slower C implementation.
-h, --help
Prints a short help screen.
@ -239,19 +240,29 @@ Configuration
it should contains the following options:
=parity FILE
Defines the file to use to store the redundancy information.
Defines the file to use to store the parity information.
It must be placed in a disk dedicated for this purpose with
as much free space as the biggest disk in the array.
Leaving the parity disk reserved for only this file, ensures that
it doesn't get fragmented, improving the performance.
This option can be used only one time.
This option is mandatory and it can be used only one time.
=q-parity FILE
Defines the file to use to store the q-parity information.
This option is optional, and if present it enables the protection
from two disk failures.
It must be placed in a disk dedicated for this purpose with
as much free space as the biggest disk in the array.
Leaving the q-parity disk reserved for only this file, ensures that
it doesn't get fragmented, improving the performance.
This option is optional and it can be used only one time.
=content FILE
Defines the file to use to store the content of the redundancy
organization.
It can be placed in the same disk of the parity file, or better
in another disk, but NOT in a data disk of the array.
This option can be used only one time.
This option is mandatory and it can be used only one time.
=disk NAME DIR
Defines the name and the mount point of the disks of the array.
@ -379,7 +390,7 @@ Content
Defines the ordered parity block list used by the last defined file.
BLOCK is the block position in the "parity" file.
0 for the first block, 1 for the second one and so on.
HASH is the md5 of the block. In the last block of the file,
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
=inv BLOCK [HASH]
@ -389,18 +400,20 @@ Content
command.
Parity
SnapRAID creates a parity file containing the redundancy information
of your disk array.
SnapRAID creates the parity and q-parity files containing the redundancy
information of your disk array.
It's a binary file, containing the computed parity of all the blocks
defined in the "content" file. You do not need to understand its format,
but it's described here for documentation.
They are binary files, containing the computed redundancy of all the
blocks defined in the "content" file. You do not need to understand its
format, but it's described here for documentation.
This file is read and written by the "sync" and "fix" commands, and
These files are read and written by the "sync" and "fix" commands, and
only read by "check".
For all the blocks at a given position, the parity information is
computed with the XOR operator applied to all the blocks.
For all the blocks at a given position, the parity and the q-parity
are computed as specified in:
:kernel.org/pub/linux/kernel/people/hpa/raid6.pdf
When a file block is shorter than the default block size, for example
because it's the last block of a file, it's assumed as filled with 0

28
snapraid.h Normal file
View file

@ -0,0 +1,28 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#ifndef __SNAPRAID_H
#define __SNAPRAID_H
/****************************************************************************/
/* snapraid */
void speed(void);
void selftest(void);
#endif

77
snapraid.txt
View file

@ -12,7 +12,7 @@ snapraid [-c, --conf CONFIG] [-f, --filter PATTERN]
[-v, --verbose]
COMMAND
snapraid [-V, --version] [-h, --help] [-T, --speed-test]
snapraid [-V, --version] [-h, --help]
2 DESCRIPTION
@ -21,7 +21,7 @@ snapraid [-V, --version] [-h, --help] [-T, --speed-test]
SnapRAID is a backup program for a disk array using redundancy.
SnapRAID uses a disk of the array to store redundancy information,
and it allows to recover from a disk failure.
and it allows to recover from up to two disk failures.
SnapRAID is mainly targeted for a home media center, where you have
a lot of big files that rarely change.
@ -34,7 +34,7 @@ features of SnapRAID are:
* You can add more disks at the array at any time.
* If you accidentally delete some files in a disk, you can
recover them.
* If more than one disk fails, you lose the data only on the
* If more than two disk fails, you lose the data only on the
failed disks. All the data in the other disks is safe.
* It doesn't lock-in your data. You can stop using SnapRAID at any
time without the need to reformat or move data.
@ -73,7 +73,7 @@ Other limitations are:
With RAID you get a speed boost with striping.
* It doesn't support real-time recovery.
With RAID you do not have to stop working when a disk fails.
* It's able to recover damages only from a single disk.
* It's able to recover damages only from up to two disks.
With a Backup you are able to recover from a complete
failure of the whole disk array.
* It identifies changes checking file time and size. If you have
@ -87,17 +87,24 @@ Other limitations are:
=================
To use SnapRAID you need first select one disk of your disk array
to dedicate at the redundancy information.
to dedicate at the parity information. With parity you will be able
to recover from a single disk failure.
You have to pick the biggest disk in the array, as the redundancy
information may grow in size as the biggest data disk in the array.
If you want to recover from two disk failures you must reserve
another disk for the q-parity information.
This disk will be dedicated to store the parity and content metadata
information. You should not store other data on it.
For both disks, you have to pick the biggest disks in the array,
as the redundancy information may grow in size as the biggest data
disk in the array.
For example, if all your disks are present in:
This disk will be dedicated to store the parity and q-parity.
You should not store other data on it, with the exception of
the content metadata information.
/mnt/diskpar <- selected disk for parity and metadata content
For example, suppose that you are interested only at the parity level
of protection, and that all your disks are present in:
/mnt/diskpar <- selected disk for parity
/mnt/disk1 <- first disk to backup
/mnt/disk2 <- second disk to backup
/mnt/disk3 <- third disk to backup
@ -173,7 +180,7 @@ SnapRAID provides three simple commands that allow to:
any redundancy recomputation.
You can stop this process at any time pressing Ctrl+C,
without losing the work already done.
The "content" and "parity" files are modified if necessary.
The "content", "parity" and "q-parity" files are modified if necessary.
The files in the array are NOT modified.
check
@ -190,7 +197,7 @@ SnapRAID provides three simple commands that allow to:
After a successful "fix", you should also run a "sync"
command to update the new state of the files.
The "content" file is NOT modified.
The "parity" file is modified if necessary.
The "parity" and "q-parity" files are modified if necessary.
The files in the array are modified if necessary.
@ -233,17 +240,11 @@ SnapRAID provides three simple commands that allow to:
-t, --count BLKCOUNT
Process only the specified number of blocks.
It's present mainly for testing.
It's present mainly for advanced manual recovering.
-v, --verbose
Prints more information in the processing.
-T, --speed-test
Executes a speed test of the MD5 implementation used.
If available, SnapRAID uses the fast MD5 implementation
from the OpenSSL crypto library. Otherwise it uses a
slower C implementation.
-h, --help
Prints a short help screen.
@ -261,19 +262,29 @@ This configuration file is located in /etc/snapraid.conf and
it should contains the following options:
parity FILE
Defines the file to use to store the redundancy information.
Defines the file to use to store the parity information.
It must be placed in a disk dedicated for this purpose with
as much free space as the biggest disk in the array.
Leaving the parity disk reserved for only this file, ensures that
it doesn't get fragmented, improving the performance.
This option can be used only one time.
This option is mandatory and it can be used only one time.
q-parity FILE
Defines the file to use to store the q-parity information.
This option is optional, and if present it enables the protection
from two disk failures.
It must be placed in a disk dedicated for this purpose with
as much free space as the biggest disk in the array.
Leaving the q-parity disk reserved for only this file, ensures that
it doesn't get fragmented, improving the performance.
This option is optional and it can be used only one time.
content FILE
Defines the file to use to store the content of the redundancy
organization.
It can be placed in the same disk of the parity file, or better
in another disk, but NOT in a data disk of the array.
This option can be used only one time.
This option is mandatory and it can be used only one time.
disk NAME DIR
Defines the name and the mount point of the disks of the array.
@ -407,7 +418,7 @@ is described here.
Defines the ordered parity block list used by the last defined file.
BLOCK is the block position in the "parity" file.
0 for the first block, 1 for the second one and so on.
HASH is the md5 of the block. In the last block of the file,
HASH is the hash of the block. In the last block of the file,
the HASH is the hash of only the used part of the block.
inv BLOCK [HASH]
@ -420,18 +431,20 @@ is described here.
10 PARITY
=========
SnapRAID creates a parity file containing the redundancy information
of your disk array.
SnapRAID creates the parity and q-parity files containing the redundancy
information of your disk array.
It's a binary file, containing the computed parity of all the blocks
defined in the "content" file. You do not need to understand its format,
but it's described here for documentation.
They are binary files, containing the computed redundancy of all the
blocks defined in the "content" file. You do not need to understand its
format, but it's described here for documentation.
This file is read and written by the "sync" and "fix" commands, and
These files are read and written by the "sync" and "fix" commands, and
only read by "check".
For all the blocks at a given position, the parity information is
computed with the XOR operator applied to all the blocks.
For all the blocks at a given position, the parity and the q-parity
are computed as specified in:
kernel.org/pub/linux/kernel/people/hpa/raid6.pdf
When a file block is shorter than the default block size, for example
because it's the last block of a file, it's assumed as filled with 0

174
speed.c Normal file
View file

@ -0,0 +1,174 @@
/*
* Copyright (C) 2011 Andrea Mazzoleni
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <http://www.gnu.org/licenses/>.
*/
#include "portable.h"
#include "snapraid.h"
#include "util.h"
#include "raid.h"
#include "cpu.h"
void speed(void)
{
struct timeval start;
struct timeval stop;
int64_t ds;
int64_t dt;
unsigned i, j;
unsigned char digest[HASH_SIZE];
unsigned count = 5000;
unsigned block_size = 256 * 1024;
unsigned diskmax = 4;
void* buffer_alloc;
unsigned char* buffer_aligned;
unsigned char** buffer;
buffer_aligned = malloc_nofail_align((diskmax + 2) * block_size, &buffer_alloc);
buffer = malloc_nofail((diskmax + 2) * sizeof(void*));
for(i=0;i<diskmax+2;++i) {
buffer[i] = buffer_aligned + i * block_size;
memset(buffer[i], i, block_size);
}
#ifdef __GNUC__
printf("Compiled with gcc " __VERSION__ "\n");
#endif
printf("Speed test with %d disk and %d buffer size...\n", diskmax, block_size);
gettimeofday(&start, 0);
for(i=0;i<count;++i) {
for(j=0;j<diskmax;++j) {
memset(buffer[j], j, block_size);
}
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("memset %llu [MB/s]\n", ds / dt);
gettimeofday(&start, 0);
for(i=0;i<count;++i) {
for(j=0;j<diskmax;++j) {
memhash(HASH_MD5, digest, buffer[j], block_size);
}
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("MD5 %llu [MB/s]\n", ds / dt);
gettimeofday(&start, 0);
for(i=0;i<count;++i) {
for(j=0;j<diskmax;++j) {
memhash(HASH_MURMUR3, digest, buffer[j], block_size);
}
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("Murmur3 %llu [MB/s]\n", ds / dt);
gettimeofday(&start, 0);
for(j=0;j<count;++j) {
raid5_int32r2(buffer, diskmax + 1, block_size);
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("RAID5 int32x2 %llu [MB/s]\n", ds / dt);
#if defined(__i386__) || defined(__x86_64__)
if (cpu_has_mmx()) {
gettimeofday(&start, 0);
for(j=0;j<count;++j) {
raid5_mmxr2(buffer, diskmax + 1, block_size);
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("RAID5 mmxx2 %llu [MB/s]\n", ds / dt);
}
if (cpu_has_sse2()) {
gettimeofday(&start, 0);
for(j=0;j<count;++j) {
raid5_sse2r2(buffer, diskmax + 1, block_size);
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("RAID5 sse2x2 %llu [MB/s]\n", ds / dt);
}
#endif
gettimeofday(&start, 0);
for(j=0;j<count;++j) {
raid6_int32r2(buffer, diskmax + 2, block_size);
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("RAID6 int32x2 %llu [MB/s]\n", ds / dt);
#if defined(__i386__) || defined(__x86_64__)
if (cpu_has_mmx()) {
gettimeofday(&start, 0);
for(j=0;j<count;++j) {
raid6_mmxr2(buffer, diskmax + 2, block_size);
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("RAID6 mmxx2 %llu [MB/s]\n", ds / dt);
}
if (cpu_has_sse2()) {
gettimeofday(&start, 0);
for(j=0;j<count;++j) {
raid6_sse2r2(buffer, diskmax + 2, block_size);
}
gettimeofday(&stop, 0);
ds = block_size * (int64_t)count * diskmax;
dt = (int64_t)1000000 * (stop.tv_sec - start.tv_sec) + (stop.tv_usec - start.tv_usec);
printf("RAID6 sse2x2 %llu [MB/s]\n", ds / dt);
}
#endif
free(buffer_alloc);
free(buffer);
}

36
state.c
View file

@ -31,10 +31,15 @@ void state_init(struct snapraid_state* state)
state->verbose = 0;
state->force_zero = 0;
state->force_empty = 0;
state->expect_unrecoverable = 0;
state->expect_recoverable = 0;
state->need_write = 0;
state->block_size = 256 * 1024; /* default 256 KiB */
state->content[0] = 0;
state->parity[0] = 0;
state->qarity[0] = 0;
state->level = 1; /* default is the lowest protection */
state->hash = HASH_MURMUR3; /* default is the fastest */
tommy_array_init(&state->diskarr);
tommy_list_init(&state->excludelist);
}
@ -49,7 +54,7 @@ void state_done(struct snapraid_state* state)
tommy_list_foreach(&state->excludelist, (tommy_foreach_func*)filter_free);
}
void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty)
void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable)
{
FILE* f;
unsigned line;
@ -57,6 +62,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
state->verbose = verbose;
state->force_zero = force_zero;
state->force_empty = force_empty;
state->expect_unrecoverable = expect_unrecoverable;
state->expect_recoverable = expect_recoverable;
f = fopen(path, "rt");
if (!f) {
@ -124,6 +131,13 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i
exit(EXIT_FAILURE);
}
pathcpy(state->parity, sizeof(state->parity), s);
} else if (strcmp(tag, "q-parity") == 0) {
if (*state->qarity) {
fprintf(stderr, "Multiple 'q-parity' specification in '%s' at line %u\n", path, line);
exit(EXIT_FAILURE);
}
pathcpy(state->qarity, sizeof(state->qarity), s);
state->level = 2;
} else if (strcmp(tag, "content") == 0) {
if (*state->content) {
fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line);
@ -186,6 +200,10 @@ void state_read(struct snapraid_state* state)
printf("Loading state...\n");
/* start with a MD5 default. */
/* it's for compatibility with version 1.0 where MD5 was implicit. */
state->hash = HASH_MD5;
disk = 0;
file = 0;
line = 0;
@ -379,6 +397,15 @@ void state_read(struct snapraid_state* state)
fprintf(stderr, "Please restore the 'block_size' value in the configuration file to '%u'\n", blksize / 1024);
exit(EXIT_FAILURE);
}
} else if (strcmp(tag, "checksum") == 0) {
if (strcmp(s, "md5") == 0) {
state->hash = HASH_MD5;
} else if (strcmp(s, "murmur3") == 0) {
state->hash = HASH_MURMUR3;
} else {
fprintf(stderr, "Invalid 'checksum' specification '%s' in '%s' at line %u\n", s, path, line);
exit(EXIT_FAILURE);
}
} else {
fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line);
exit(EXIT_FAILURE);
@ -425,6 +452,12 @@ void state_write(struct snapraid_state* state)
exit(EXIT_FAILURE);
}
ret = fprintf(f, "checksum %s\n", state->hash == HASH_MD5 ? "md5" : "murmur3");
if (ret < 0) {
fprintf(stderr, "Error writing the content file '%s' in fprintf(). %s.\n", path, strerror(errno));
exit(EXIT_FAILURE);
}
/* for each disk */
for(i=0;i<tommy_array_size(&state->diskarr);++i) {
tommy_node* j;
@ -594,7 +627,6 @@ int state_progress(time_t* start, time_t* last, block_off_t countpos, block_off_
*last = now;
}
/* stop if requested */
if (global_interrupt) {
printf("\n");

10
state.h
View file

@ -26,14 +26,22 @@
*/
extern volatile int global_interrupt;
#define HASH_MURMUR3 0
#define HASH_MD5 1
struct snapraid_state {
int verbose; /**< Verbose output. */
int force_zero; /**< Forced dangerous operations of synching file now with zero size. */
int force_empty; /**< Forced dangerous operations of synching disk now empty. */
int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */
int expect_recoverable; /**< Expect presence of recoverable error in checking. */
int need_write; /**< If the state is changed. */
uint32_t block_size; /**< Block size in bytes. */
char content[PATH_MAX]; /**< Path of the content file. */
char parity[PATH_MAX]; /**< Path of the parity file. */
char qarity[PATH_MAX]; /**< Path of the qarity file. */
unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */
unsigned hash; /**< Hash kind used. */
tommy_array diskarr; /**< Disk array. */
tommy_list excludelist; /**< List of exclusion. */
};
@ -51,7 +59,7 @@ void state_done(struct snapraid_state* state);
/**
* Read the configuration file.
*/
void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty);
void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable);
/**
* Read the state.

113
sync.c
View file

@ -22,18 +22,21 @@
#include "state.h"
#include "parity.h"
#include "handle.h"
#include "raid.h"
/****************************************************************************/
/* sync */
static int state_sync_process(struct snapraid_state* state, int parity_f, block_off_t blockstart, block_off_t blockmax)
static int state_sync_process(struct snapraid_state* state, int parity_f, int qarity_f, block_off_t blockstart, block_off_t blockmax)
{
struct snapraid_handle* handle;
unsigned diskmax = tommy_array_size(&state->diskarr);
block_off_t i;
unsigned j;
unsigned char* block_buffer;
unsigned char* xor_buffer;
void* buffer_alloc;
unsigned char* buffer_aligned;
unsigned char** buffer;
unsigned buffermax;
data_off_t countsize;
block_off_t countpos;
block_off_t countmax;
@ -42,8 +45,14 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
int ret;
unsigned unrecoverable_error;
block_buffer = malloc_nofail(state->block_size);
xor_buffer = malloc_nofail(state->block_size);
/* we need disk + 1 for each parity level buffers */
buffermax = diskmax + state->level;
buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc);
buffer = malloc_nofail(buffermax * sizeof(void*));
for(i=0;i<buffermax;++i) {
buffer[i] = buffer_aligned + i * state->block_size;
}
handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle));
for(i=0;i<diskmax;++i) {
@ -97,9 +106,6 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
if (!one_invalid)
continue;
/* start with 0 */
memset(xor_buffer, 0, state->block_size);
/* for each disk, process the block */
for(j=0;j<diskmax;++j) {
int read_size;
@ -107,8 +113,11 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
struct snapraid_block* block;
block = disk_block_get(handle[j].disk, i);
if (!block)
if (!block) {
/* use an empty block */
memset(buffer[j], 0, state->block_size);
continue;
}
ret = handle_close_if_different(&handle[j], block->file);
if (ret == -1) {
@ -150,7 +159,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
goto bail;
}
read_size = handle_read(&handle[j], block, block_buffer, state->block_size);
read_size = handle_read(&handle[j], block, buffer[j], state->block_size);
if (read_size == -1) {
fprintf(stderr, "DANGER! Unexpected read error in a data disk, it isn't possible to sync.\n");
fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid check'\n");
@ -159,7 +168,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
}
/* now compute the hash */
memhash(hash, block_buffer, read_size);
memhash(state->hash, hash, buffer[j], read_size);
if (bit_has(block->flag, BLOCK_HAS_HASH)) {
/* compare the hash */
@ -176,21 +185,36 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_
memcpy(block->hash, hash, HASH_SIZE);
}
/* compute the parity */
memxor(xor_buffer, block_buffer, read_size);
countsize += read_size;
}
/* compute the parity */
if (state->level == 1) {
raid5_gen(buffer, diskmax + state->level, state->block_size);
} else {
raid6_gen(buffer, diskmax + state->level, state->block_size);
}
/* write the parity */
ret = parity_write(state->parity, parity_f, i, xor_buffer, state->block_size);
ret = parity_write(state->parity, parity_f, i, buffer[diskmax], state->block_size);
if (ret == -1) {
fprintf(stderr, "DANGER! Write error in the parity disk, it isn't possible to sync.\n");
fprintf(stderr, "DANGER! Write error in the Parity disk, it isn't possible to sync.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
/* write the qarity */
if (state->level >= 2) {
ret = parity_write(state->qarity, qarity_f, i, buffer[diskmax+1], state->block_size);
if (ret == -1) {
fprintf(stderr, "DANGER! Write error in the Q-Parity disk, it isn't possible to sync.\n");
fprintf(stderr, "Stopping at block %u\n", i);
++unrecoverable_error;
goto bail;
}
}
/* for each disk, mark the blocks as processed */
for(j=0;j<diskmax;++j) {
struct snapraid_block* block;
@ -231,8 +255,8 @@ bail:
}
free(handle);
free(block_buffer);
free(xor_buffer);
free(buffer_alloc);
free(buffer);
if (unrecoverable_error != 0)
return -1;
@ -241,11 +265,13 @@ bail:
int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount)
{
char path[PATH_MAX];
char parity_path[PATH_MAX];
char qarity_path[PATH_MAX];
block_off_t blockmax;
data_off_t size;
int ret;
int f;
int parity_f;
int qarity_f;
unsigned unrecoverable_error;
printf("Syncing...\n");
@ -258,44 +284,71 @@ int state_sync(struct snapraid_state* state, block_off_t blockstart, block_off_t
fprintf(stderr, "Error in the starting block %u. It's bigger than the parity size %u.\n", blockstart, blockmax);
exit(EXIT_FAILURE);
}
/* adjust the number of block to process */
if (blockcount != 0 && blockstart + blockcount < blockmax) {
blockmax = blockstart + blockcount;
}
pathcpy(path, sizeof(path), state->parity);
f = parity_create(path, size);
if (f == -1) {
fprintf(stderr, "WARNING! Without an accessible parity file, it isn't possible to sync.\n");
pathcpy(parity_path, sizeof(parity_path), state->parity);
parity_f = parity_create(parity_path, size);
if (parity_f == -1) {
fprintf(stderr, "WARNING! Without an accessible Parity file, it isn't possible to sync.\n");
exit(EXIT_FAILURE);
}
if (state->level >= 2) {
pathcpy(qarity_path, sizeof(qarity_path), state->qarity);
qarity_f = parity_create(qarity_path, size);
if (qarity_f == -1) {
fprintf(stderr, "WARNING! Without an accessible Q-Parity file, it isn't possible to sync.\n");
exit(EXIT_FAILURE);
}
} else {
qarity_f = -1;
}
unrecoverable_error = 0;
/* skip degenerated cases of empty parity, or skipping all */
if (blockstart < blockmax) {
ret = state_sync_process(state, f, blockstart, blockmax);
ret = state_sync_process(state, parity_f, qarity_f, blockstart, blockmax);
if (ret == -1) {
++unrecoverable_error;
/* continue, as we are already exiting */
}
}
ret = parity_sync(path, f);
ret = parity_sync(parity_path, parity_f);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected sync error in parity disk.\n");
fprintf(stderr, "DANGER! Unexpected sync error in Parity disk.\n");
++unrecoverable_error;
/* continue, as we are already exiting */
}
ret = parity_close(path, f);
ret = parity_close(parity_path, parity_f);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in parity disk.\n");
fprintf(stderr, "DANGER! Unexpected close error in Parity disk.\n");
++unrecoverable_error;
/* continue, as we are already exiting */
}
if (state->level >= 2) {
ret = parity_sync(qarity_path, qarity_f);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected sync error in Q-Parity disk.\n");
++unrecoverable_error;
/* continue, as we are already exiting */
}
ret = parity_close(qarity_path, qarity_f);
if (ret == -1) {
fprintf(stderr, "DANGER! Unexpected close error in Q-Parity disk.\n");
++unrecoverable_error;
/* continue, as we are already exiting */
}
}
/* abort if required */
if (unrecoverable_error != 0)
return -1;

40
util.c
View file

@ -223,6 +223,24 @@ void* malloc_nofail(size_t size)
return ptr;
}
#define ALIGN 256
void* malloc_nofail_align(size_t size, void** freeptr)
{
unsigned char* ptr = malloc_nofail(size + ALIGN);
uintptr_t offset;
*freeptr = ptr;
offset = ((uintptr_t)ptr) % ALIGN;
if (offset != 0) {
ptr += ALIGN - offset;
}
return ptr;
}
void memxor(unsigned char* xor, const unsigned char* block, unsigned size)
{
while (size >= 16) {
@ -263,15 +281,25 @@ void memxor(unsigned char* xor, const unsigned char* block, unsigned size)
#include "md5.c"
#endif
void memhash(void* digest, const void* src, unsigned size)
#include "murmurhash3.c"
void memhash(unsigned kind, void* digest, const void* src, unsigned size)
{
if (kind == HASH_MURMUR3) {
MurmurHash3_x86_128(src, size, 0, digest);
return;
}
if (kind == HASH_MD5) {
#if HAVE_LIBCRYPTO
MD5((void*)src, size, digest);
MD5((void*)src, size, digest);
#else
struct md5_t md5;
md5_init(&md5);
md5_update(&md5, src, size);
md5_final(&md5, digest);
struct md5_t md5;
md5_init(&md5);
md5_update(&md5, src, size);
md5_final(&md5, digest);
#endif
return;
}
}

16
util.h
View file

@ -121,6 +121,12 @@ static inline unsigned bit_clear(unsigned bit, unsigned mask)
*/
void* malloc_nofail(size_t size);
/**
* Safe aligned malloc.
* If no memory is available, it aborts.
*/
void* malloc_nofail_align(size_t size, void** freeptr);
/**
* Xor two memory blocks.
*/
@ -133,9 +139,15 @@ void memxor(unsigned char* xor, const unsigned char* block, unsigned size);
#define HASH_SIZE 16
/**
* Computes the hash of a memory block.
* Hash kinds.
*/
void memhash(void* digest, const void* src, unsigned size);
#define HASH_MURMUR3 0
#define HASH_MD5 1
/**
* Computes the HASH of a memory block.
*/
void memhash(unsigned kind, void* digest, const void* src, unsigned size);
#endif