From 08831c664194ea6ca0bf3572bc061d155c735a81 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 23 Apr 2011 21:38:26 +0200 Subject: [PATCH] Initial RAID6 support. Not yet complete. Uses Murmur3 as default hash. Check now returns and error even if only recoverable errors are present. --- HISTORY | 9 +- Makefile.am | 28 ++- TODO | 5 + check.c | 443 +++++++++++++++++++++++++++++++--------------- configure.windows | 2 + cpu.h | 57 ++++++ dry.c | 71 ++++++-- handle.c | 5 + handle.h | 1 + murmurhash3.c | 155 ++++++++++++++++ raid.c | 331 ++++++++++++++++++++++++++++++++++ raid.h | 41 +++++ selftest.c | 111 ++++++++++++ snapraid.1 | 82 +++++---- snapraid.c | 106 ++--------- snapraid.d | 77 ++++---- snapraid.h | 28 +++ snapraid.txt | 77 ++++---- speed.c | 174 ++++++++++++++++++ state.c | 36 +++- state.h | 10 +- sync.c | 113 ++++++++---- util.c | 40 ++++- util.h | 16 +- 24 files changed, 1624 insertions(+), 394 deletions(-) create mode 100755 configure.windows create mode 100644 cpu.h create mode 100644 murmurhash3.c create mode 100644 raid.c create mode 100644 raid.h create mode 100644 selftest.c create mode 100644 snapraid.h create mode 100644 speed.c diff --git a/HISTORY b/HISTORY index 1b4e879d..b1c16cc6 100644 --- a/HISTORY +++ b/HISTORY @@ -1,6 +1,14 @@ SnapRAID HISTORY ================ +1.1 2011/5 +========== +* Added RAID6 support. +* The default checksum is Murmur3 hash. + MD5 is still supported for backward compatibility. +* The check command returns with an error code if any kind of error is present. + Previously it was only if unrecoverable error was present. + 1.0 2011/4 ========== * No relevant change. @@ -26,7 +34,6 @@ SnapRAID HISTORY * Precise error management. * More regression tests. - 0.1 2011/3 ========== * First public test release of SnapRAID. diff --git a/Makefile.am b/Makefile.am index 9d63ac5f..7cf5f3cd 100644 --- a/Makefile.am +++ b/Makefile.am @@ -13,10 +13,14 @@ snapraid_SOURCES = \ dry.c \ parity.c \ handle.c \ - fnmatch.c + fnmatch.c \ + raid.c \ + selftest.c \ + speed.c noinst_HEADERS = \ portable.h \ + snapraid.h \ util.h \ elem.h \ state.h \ @@ -24,6 +28,7 @@ noinst_HEADERS = \ handle.h \ md5.h \ md5.c \ + murmurhash3.c \ tommyarray.h \ tommychain.h \ tommyhash.h \ @@ -34,7 +39,9 @@ noinst_HEADERS = \ tommyhash.c \ tommyhashdyn.c \ tommylist.c \ - fnmatch.h + fnmatch.h \ + cpu.h \ + raid.h mktest_SOURCES = \ mktest.c @@ -97,26 +104,26 @@ check-local: rm test/disk3/6* $(TESTENV) ./mktest$(EXEEXT) generate 4 3 100 4000 $(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test.conf -t 7 sync - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test.conf -t 11 sync - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -v -c $(srcdir)/test.conf sync $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Delete some files, fix and check rm test/disk1/1* - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Delete some dirs, fix and check rm -r test/disk2/_a rm -r test/disk2/_b - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Delete a whole disk, fix and check rm -r test/disk3 mkdir test/disk3 - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Sync after all the fixes with an empty drive @@ -128,12 +135,12 @@ check-local: $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Corrupt the parity, fix and check $(TESTENV) ./mktest$(EXEEXT) damage 1 10 10000 test/parity - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Corrupt some files, fix and check $(TESTENV) ./mktest$(EXEEXT) damage 1 1 500 test/disk2/b* - $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check 2>/dev/null + $(TESTENV) ./snapraid$(EXEEXT) -R -c $(srcdir)/test.conf check 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf fix 2>/dev/null $(TESTENV) ./snapraid$(EXEEXT) -c $(srcdir)/test.conf check echo --- Sync after all the fixes @@ -181,6 +188,9 @@ snapraid.txt : snapraid.d snapraid.html : snapraid.d advd2 html < $(srcdir)/$< > $@ +snapraid.hh : snapraid.d + advd2 frame < $(srcdir)/$< > $@ + # Windows distribution DISTWIN = \ diff --git a/TODO b/TODO index 0e3b97ce..3afc369f 100644 --- a/TODO +++ b/TODO @@ -12,11 +12,16 @@ completely restored. * Reduce memory occupation removing the flag field in the block structure. +* Provide an "include" logic in filtering. + * Add RAID6 redundancy support. There is the very good raid6 library from H. Peter Anvin (hpa) in the Linux Kernel ready to use. See: http://kernel.org/pub/linux/kernel/people/hpa/raid6.pdf http://lxr.linux.no/#linux+v2.6.38/lib/raid6 +* Recognize that a file is moved from one disk to another, and if the parity +data doesn't overlap, do not recompute it. + * Go multithread to speed up the MD5 computation. We can do it when syncing and checking. diff --git a/check.c b/check.c index 4d075e0a..07b32d73 100644 --- a/check.c +++ b/check.c @@ -22,18 +22,94 @@ #include "state.h" #include "parity.h" #include "handle.h" +#include "raid.h" /****************************************************************************/ /* check */ -static int state_check_process(struct snapraid_state* state, int fix, int parity_f, block_off_t blockstart, block_off_t blockmax) +struct failed_struct { + unsigned index; + struct snapraid_block* block; + struct snapraid_handle* handle; +}; + +/** + * Repair errors. + * Returns <0 if failure for missing stratey, >0 if data is wrong and we cannot rebuild correctly, 0 on success. + * If success, the parity and qarity are computed in the buffer variable. + */ +static int repair(struct snapraid_state* state, unsigned i, unsigned diskmax, struct failed_struct* failed, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_parity, unsigned char* buffer_qarity) +{ + /* no fix required */ + if (failed_count == 0) { + /* compute parity and qarity to check it */ + if (state->level == 1) { + raid5_gen(buffer, diskmax + state->level, state->block_size); + } else { + raid6_gen(buffer, diskmax + state->level, state->block_size); + } + + return 0; + } + + /* fix with RAID5 */ + if (failed_count == 1 && buffer_parity != 0) { + unsigned char hash[HASH_SIZE]; + unsigned size; + unsigned j; + + /* recompute the block */ + memcpy(buffer[failed[0].index], buffer_parity, state->block_size); + for(j=0;jblock_size); + } + } + + size = block_file_size(failed[0].block, state->block_size); + + /* now compute the hash */ + memhash(state->hash, hash, buffer[failed[0].index], size); + + /* compare the hash */ + if (memcmp(hash, failed[0].block->hash, HASH_SIZE) != 0) { + fprintf(stderr, "%u: Parity data error\n", i); + return 1; + } + + /* compute parity and qarity to check it */ + /* we recompute also the just used parity, because we may have used only a small part of it */ + if (state->level == 1) { + raid5_gen(buffer, diskmax + state->level, state->block_size); + } else { + raid6_gen(buffer, diskmax + state->level, state->block_size); + } + + return 0; + } + + if (failed_count == 1 && buffer_qarity != 0) { + /* TODO RAID6 */ + } + + if (failed_count == 2 && buffer_parity != 0 && buffer_qarity != 0) { + /* TODO RAID6 */ + } + + /* no more stragety to fix */ + return -1; +} + +static int state_check_process(struct snapraid_state* state, int fix, int parity_f, int qarity_f, block_off_t blockstart, block_off_t blockmax) { struct snapraid_handle* handle; unsigned diskmax = tommy_array_size(&state->diskarr); block_off_t i; unsigned j; - unsigned char* block_buffer; - unsigned char* xor_buffer; + void* buffer_alloc; + unsigned char* buffer_aligned; + unsigned char** buffer; + unsigned buffermax; int ret; data_off_t countsize; block_off_t countpos; @@ -44,8 +120,14 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity unsigned unrecoverable_error; unsigned recovered_error; - block_buffer = malloc_nofail(state->block_size); - xor_buffer = malloc_nofail(state->block_size); + /* we need disk + 2 for each parity level buffers */ + buffermax = diskmax + state->level * 2; + + buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc); + buffer = malloc_nofail(buffermax * sizeof(void*)); + for(i=0;iblock_size; + } handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle)); for(i=0;iblock_size); - all_parity = 1; /* if all hashed block have parity computed */ - failed = 0; /* number of failed block */ - failed_block = 0; /* last failed block, but not filtered out */ - failed_handle = 0; + failed_count = 0; /* number of failed blocks */ /* for each disk, process the block */ for(j=0;jblock_size); continue; + } /* we try to check and fix only if the block is hashed */ /* it could be that a file was added, and not yet synched */ /* so we should not include not hashed block in the parity computation */ - if (!bit_has(block->flag, BLOCK_HAS_HASH)) + if (!bit_has(block->flag, BLOCK_HAS_HASH)) { + /* use an empty block */ + memset(buffer[j], 0, state->block_size); continue; + } /* keep track if at least one hashed block has no parity */ if (!bit_has(block->flag, BLOCK_HAS_PARITY)) @@ -160,11 +242,12 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity ret = handle_open(&handle[j], block->file); if (ret == -1) { /* save the failed block for the parity check */ - ++failed; - if (!block->file->is_filtered) { - failed_block = block; - failed_handle = &handle[j]; + if (failed_count < 2) { + failed[failed_count].index = j; + failed[failed_count].block = block; + failed[failed_count].handle = &handle[j]; } + ++failed_count; fprintf(stderr, "%u: Open error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); ++error; @@ -172,14 +255,15 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } } - read_size = handle_read(&handle[j], block, block_buffer, state->block_size); + read_size = handle_read(&handle[j], block, buffer[j], state->block_size); if (read_size == -1) { /* save the failed block for the parity check */ - ++failed; - if (!block->file->is_filtered) { - failed_block = block; - failed_handle = &handle[j]; + if (failed_count < 2) { + failed[failed_count].index = j; + failed[failed_count].block = block; + failed[failed_count].handle = &handle[j]; } + ++failed_count; fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); ++error; @@ -187,123 +271,144 @@ static int state_check_process(struct snapraid_state* state, int fix, int parity } /* now compute the hash */ - memhash(hash, block_buffer, read_size); + memhash(state->hash, hash, buffer[j], read_size); /* compare the hash */ if (memcmp(hash, block->hash, HASH_SIZE) != 0) { /* save the failed block for the parity check */ - ++failed; - if (!block->file->is_filtered) { - failed_block = block; - failed_handle = &handle[j]; + if (failed_count < 2) { + failed[failed_count].index = j; + failed[failed_count].block = block; + failed[failed_count].handle = &handle[j]; } + ++failed_count; fprintf(stderr, "%u: Data error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); ++error; continue; } - /* compute the parity */ - memxor(xor_buffer, block_buffer, read_size); - countsize += read_size; } - if (failed != 0 && !failed_block) { - /* if the failed blocks are filtered out, do nothing */ - } else if (parity_f == -1 || !all_parity) { - /* all the cases with no parity file */ - if (failed != 0) { - fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i); - ++unrecoverable_error; - } - } else if (failed == 0) { - int ret; - int fixable; + { + unsigned char* buffer_parity; + unsigned char* buffer_qarity; - fixable = 0; - - /* read the parity */ - ret = parity_read(state->parity, parity_f, i, block_buffer, state->block_size); - if (ret == -1) { - fprintf(stderr, "%u: Parity read error\n", i); - ++error; - - fixable = 1; + /* buffers for parity read and not computed */ + if (state->level == 1) { + buffer_parity = buffer[diskmax + 1]; + buffer_qarity = 0; } else { - /* compare it */ - if (memcmp(xor_buffer, block_buffer, state->block_size) != 0) { + buffer_parity = buffer[diskmax + 2]; + buffer_qarity = buffer[diskmax + 3]; + } + + /* read the parity */ + if (parity_f != -1) { + ret = parity_read(state->parity, parity_f, i, buffer_parity, state->block_size); + if (ret == -1) { + buffer_parity = 0; /* no parity to use */ + + fprintf(stderr, "%u: Parity read error\n", i); + ++error; + } + } else { + buffer_parity = 0; + } + + /* read the qarity */ + if (state->level >= 2) { + if (qarity_f != -1) { + ret = parity_read(state->qarity, qarity_f, i, buffer_qarity, state->block_size); + if (ret == -1) { + buffer_qarity = 0; /* no qarity to use */ + + fprintf(stderr, "%u: Q-Parity read error\n", i); + ++error; + } + } else { + buffer_qarity = 0; + } + } + + ret = repair(state, i, diskmax, failed, failed_count, buffer, buffer_parity, buffer_qarity); + if (ret != 0) { + /* increment the number of errors */ + if (ret > 0) + error += ret; + + fprintf(stderr, "%u: UNRECOVERABLE error for this block\n", i); + ++unrecoverable_error; + } else { + /* check the parity */ + if (buffer_parity != 0 && memcmp(buffer_parity, buffer[diskmax], state->block_size) != 0) { + buffer_parity = 0; + fprintf(stderr, "%u: Parity data error\n", i); ++error; - - fixable = 1; - } - } - - if (fix && fixable) { - /* fix the parity */ - /* if all the file hashes are matching, we can recompute the parity */ - ret = parity_write(state->parity, parity_f, i, xor_buffer, state->block_size); - if (ret == -1) { - fprintf(stderr, "WARNING! Without a working parity disk, it isn't possible to fix errors on it.\n"); - fprintf(stderr, "Stopping at block %u\n", i); - ++unrecoverable_error; - goto bail; } - fprintf(stderr, "%u: Fixed\n", i); - ++recovered_error; - } - } else if (failed == 1) { - unsigned char hash[HASH_SIZE]; - unsigned failed_size; - int ret; - int fixable; + /* check the qarity */ + if (state->level >= 2) { + if (buffer_qarity != 0 && memcmp(buffer_qarity, buffer[diskmax + 1], state->block_size) != 0) { + buffer_qarity = 0; - fixable = 0; - - /* read the parity */ - ret = parity_read(state->parity, parity_f, i, block_buffer, state->block_size); - if (ret == -1) { - fprintf(stderr, "%u: Parity read error\n", i); - fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i); - ++unrecoverable_error; - } else { - /* compute the failed block */ - memxor(block_buffer, xor_buffer, state->block_size); - - failed_size = block_file_size(failed_block, state->block_size); - - /* now compute the hash */ - memhash(hash, block_buffer, failed_size); - - /* compare the hash */ - if (memcmp(hash, failed_block->hash, HASH_SIZE) != 0) { - fprintf(stderr, "%u: Parity data error\n", i); - fprintf(stderr, "%u: UNRECOVERABLE erros for this block\n", i); - ++unrecoverable_error; - } else { - fixable = 1; - } - } - - if (fix && fixable) { - /* fix the file */ - /* if only one file is wrong, we already have recomputed it */ - ret = handle_write(failed_handle, failed_block, block_buffer, state->block_size); - if (ret == -1) { - fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); - fprintf(stderr, "Stopping at block %u\n", i); - ++unrecoverable_error; - goto bail; + fprintf(stderr, "%u: Q-Parity data error\n", i); + ++error; + } } - fprintf(stderr, "%u: Fixed\n", i); - ++recovered_error; + if (fix) { + /* update the fixed files */ + for(j=0;jfile->is_filtered) + continue; + + ret = handle_write(failed[j].handle, failed[j].block, buffer[failed[j].index], state->block_size); + if (ret == -1) { + fprintf(stderr, "WARNING! Without a working data disk, it isn't possible to fix errors on it.\n"); + fprintf(stderr, "Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } + + fprintf(stderr, "%u: Fixed data error for file %s at position %u\n", i, failed[j].block->file->sub, block_file_pos(failed[j].block)); + ++recovered_error; + } + + /* update the parity */ + if (buffer_parity == 0 && parity_f != -1) { + ret = parity_write(state->parity, parity_f, i, buffer[diskmax], state->block_size); + if (ret == -1) { + fprintf(stderr, "WARNING! Without a working parity disk, it isn't possible to fix errors on it.\n"); + fprintf(stderr, "Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } + + fprintf(stderr, "%u: Fixed parity error\n", i); + ++recovered_error; + } + + /* update the qarity */ + if (state->level >= 2) { + if (buffer_qarity == 0 && qarity_f != -1) { + ret = parity_write(state->qarity, qarity_f, i, buffer[diskmax + 1], state->block_size); + if (ret == -1) { + fprintf(stderr, "WARNING! Without a working Q-Parity disk, it isn't possible to fix errors on it.\n"); + fprintf(stderr, "Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } + } + + fprintf(stderr, "%u: Fixed Q-Parity error\n", i); + ++recovered_error; + } + } } - } else { - fprintf(stderr, "%u: UNRECOVERABLE errors for this block\n", i); - ++unrecoverable_error; } /* count the number of processed block */ @@ -353,22 +458,47 @@ bail: } free(handle); - free(block_buffer); - free(xor_buffer); + free(buffer_alloc); + free(buffer); + + /* fails if some error are present after the run */ + if (fix) { + if (state->expect_unrecoverable) { + if (unrecoverable_error == 0) + return -1; + } else { + if (unrecoverable_error != 0) + return -1; + } + } else { + if (state->expect_unrecoverable && state->expect_recoverable) { + if (error == 0 || unrecoverable_error == 0) + return -1; + } else if (state->expect_unrecoverable) { + if (unrecoverable_error == 0) + return -1; + } if (state->expect_recoverable) { + if (error == 0) + return -1; + } else { + if (error != 0 || unrecoverable_error != 0) + return -1; + } + } - if (unrecoverable_error != 0) - return -1; return 0; } void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, block_off_t blockcount) { - char path[PATH_MAX]; + char parity_path[PATH_MAX]; + char qarity_path[PATH_MAX]; block_off_t blockmax; data_off_t size; int ret; - int f; - unsigned unrecoverable_error; + int parity_f; + int qarity_f; + unsigned error; if (fix) printf("Checking and fixing...\n"); @@ -388,48 +518,81 @@ void state_check(struct snapraid_state* state, int fix, block_off_t blockstart, blockmax = blockstart + blockcount; } - pathcpy(path, sizeof(path), state->parity); + pathcpy(parity_path, sizeof(parity_path), state->parity); if (fix) { /* if fixing, create the file and open for writing */ /* if it fails, we cannot continue */ - f = parity_create(path, size); - if (f == -1) { - fprintf(stderr, "WARNING! Without an accessible parity file, it isn't possible to fix any error.\n"); + parity_f = parity_create(parity_path, size); + if (parity_f == -1) { + fprintf(stderr, "WARNING! Without an accessible Parity file, it isn't possible to fix any error.\n"); exit(EXIT_FAILURE); } + + if (state->level >= 2) { + pathcpy(qarity_path, sizeof(qarity_path), state->qarity); + qarity_f = parity_create(qarity_path, size); + if (qarity_f == -1) { + fprintf(stderr, "WARNING! Without an accessible Q-Parity file, it isn't possible to fix any error.\n"); + exit(EXIT_FAILURE); + } + } else { + qarity_f = -1; + } } else { /* if checking, open the file for reading */ /* it may fail if the file doesn't exist, in this case we continue to check the files */ - f = parity_open(path); - if (f == -1) { - printf("No accessible parity file, all the errors are going to be UNRECOVERABLE.\n"); + parity_f = parity_open(parity_path); + if (parity_f == -1) { + printf("No accessible Parity file, only files will be checked.\n"); /* continue anyway */ } + + if (state->level >= 2) { + pathcpy(qarity_path, sizeof(qarity_path), state->qarity); + qarity_f = parity_open(qarity_path); + if (qarity_f == -1) { + printf("No accessible Q-Parity file, only files will be checked.\n"); + /* continue anyway */ + } + } else { + qarity_f = -1; + } } - unrecoverable_error = 0; + error = 0; /* skip degenerated cases of empty parity, or skipping all */ if (blockstart < blockmax) { - ret = state_check_process(state, fix, f, blockstart, blockmax); + ret = state_check_process(state, fix, parity_f, qarity_f, blockstart, blockmax); if (ret == -1) { - ++unrecoverable_error; + ++error; /* continue, as we are already exiting */ } } /* try to close only if opened */ - if (f != -1) { - ret = parity_close(path, f); + if (parity_f != -1) { + ret = parity_close(parity_path, parity_f); if (ret == -1) { fprintf(stderr, "WARNING! A 'snapraid check' command is highly suggested.\n"); - ++unrecoverable_error; + ++error; /* continue, as we are already exiting */ } } - /* abort if required */ - if (unrecoverable_error != 0) + if (state->level >= 2) { + if (qarity_f != -1) { + ret = parity_close(qarity_path, qarity_f); + if (ret == -1) { + fprintf(stderr, "WARNING! A 'snapraid check' command is highly suggested.\n"); + ++error; + /* continue, as we are already exiting */ + } + } + } + + /* abort if error are present */ + if (error != 0) exit(EXIT_FAILURE); } diff --git a/configure.windows b/configure.windows new file mode 100755 index 00000000..f682a966 --- /dev/null +++ b/configure.windows @@ -0,0 +1,2 @@ +./configure --host=i686-pc-mingw32 --build=`./config.guess` $@ + diff --git a/cpu.h b/cpu.h new file mode 100644 index 00000000..d4426948 --- /dev/null +++ b/cpu.h @@ -0,0 +1,57 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __CPU_H +#define __CPU_H + +#if defined(__i386__) || defined(__x86_64__) + +#define X86_FEATURE_MMX (0*32+23) /* Multimedia Extensions */ +#define X86_FEATURE_FXSR (0*32+24) /* FXSAVE and FXRSTOR instructions (fast save and restore) */ +#define X86_FEATURE_XMM (0*32+25) /* Streaming SIMD Extensions */ +#define X86_FEATURE_XMM2 (0*32+26) /* Streaming SIMD Extensions-2 */ +#define X86_FEATURE_MMXEXT (1*32+22) /* AMD MMX extensions */ + +static inline int cpu_has(uint32_t flag) +{ + uint32_t eax = (flag >> 5) ? 0x80000001 : 1; + uint32_t edx; + + asm volatile("cpuid" + : "+a" (eax), "=d" (edx) + : : "ecx", "ebx"); + + return (edx >> (flag & 31)) & 1; +} + +static inline int cpu_has_mmx(void) +{ + return cpu_has(X86_FEATURE_MMX); +} + +static inline int cpu_has_sse2(void) +{ + return cpu_has(X86_FEATURE_MMX) && + cpu_has(X86_FEATURE_FXSR) && + cpu_has(X86_FEATURE_XMM) && + cpu_has(X86_FEATURE_XMM2); +} + +#endif + +#endif + diff --git a/dry.c b/dry.c index 238b2e08..e507eda6 100644 --- a/dry.c +++ b/dry.c @@ -26,13 +26,13 @@ /****************************************************************************/ /* dry */ -static int state_dry_process(struct snapraid_state* state, int parity_f, block_off_t blockstart, block_off_t blockmax) +static int state_dry_process(struct snapraid_state* state, int parity_f, int qarity_f, block_off_t blockstart, block_off_t blockmax) { struct snapraid_handle* handle; unsigned diskmax = tommy_array_size(&state->diskarr); block_off_t i; unsigned j; - unsigned char* block_buffer; + unsigned char* buffer; int ret; data_off_t countsize; block_off_t countpos; @@ -41,7 +41,7 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, block_o time_t last; unsigned error; - block_buffer = malloc_nofail(state->block_size); + buffer = malloc_nofail(state->block_size); handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle)); for(i=0;iblock_size); + read_size = handle_read(&handle[j], block, buffer, state->block_size); if (read_size == -1) { fprintf(stderr, "%u: Read error for file %s at position %u\n", i, block->file->sub, block_file_pos(block)); ++error; @@ -92,12 +92,22 @@ static int state_dry_process(struct snapraid_state* state, int parity_f, block_o } /* read the parity */ - ret = parity_read(state->parity, parity_f, i, block_buffer, state->block_size); + ret = parity_read(state->parity, parity_f, i, buffer, state->block_size); if (ret == -1) { fprintf(stderr, "%u: Parity read error\n", i); ++error; } + /* read the qarity */ + if (state->level >= 2) { + ret = parity_read(state->qarity, qarity_f, i, buffer, state->block_size); + if (ret == -1) { + fprintf(stderr, "%u: Syndrome read error\n", i); + ++error; + } + } + + /* count the number of processed block */ ++countpos; @@ -131,7 +141,7 @@ bail: } free(handle); - free(block_buffer); + free(buffer); if (error != 0) return -1; @@ -140,12 +150,14 @@ bail: void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockcount) { - char path[PATH_MAX]; + char parity_path[PATH_MAX]; + char qarity_path[PATH_MAX]; block_off_t blockmax; data_off_t size; int ret; - int f; - unsigned unrecoverable_error; + int parity_f; + int qarity_f; + unsigned error; printf("Drying...\n"); @@ -162,37 +174,58 @@ void state_dry(struct snapraid_state* state, block_off_t blockstart, block_off_t blockmax = blockstart + blockcount; } - pathcpy(path, sizeof(path), state->parity); + pathcpy(parity_path, sizeof(parity_path), state->parity); /* if drying, open the file for reading */ /* it may fail if the file doesn't exist, in this case we continue to dry the files */ - f = parity_open(path); - if (f == -1) { + parity_f = parity_open(parity_path); + if (parity_f == -1) { printf("No accessible parity file.\n"); /* continue anyway */ } - unrecoverable_error = 0; + if (state->level >= 2) { + pathcpy(qarity_path, sizeof(qarity_path), state->qarity); + qarity_f = parity_open(qarity_path); + if (qarity_f == -1) { + printf("No accessible qarity file.\n"); + /* continue anyway */ + } + } else { + qarity_f = -1; + } + + error = 0; /* skip degenerated cases of empty parity, or skipping all */ if (blockstart < blockmax) { - ret = state_dry_process(state, f, blockstart, blockmax); + ret = state_dry_process(state, parity_f, qarity_f, blockstart, blockmax); if (ret == -1) { - ++unrecoverable_error; + ++error; /* continue, as we are already exiting */ } } /* try to close only if opened */ - if (f != -1) { - ret = parity_close(path, f); + if (parity_f != -1) { + ret = parity_close(parity_path, parity_f); if (ret == -1) { - ++unrecoverable_error; + ++error; /* continue, as we are already exiting */ } } + if (state->level >= 2) { + if (qarity_f != -1) { + ret = parity_close(qarity_path, qarity_f); + if (ret == -1) { + ++error; + /* continue, as we are already exiting */ + } + } + } + /* abort if required */ - if (unrecoverable_error != 0) + if (error != 0) exit(EXIT_FAILURE); } diff --git a/handle.c b/handle.c index 50f53a74..51270f56 100644 --- a/handle.c +++ b/handle.c @@ -256,6 +256,11 @@ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, un return -1; } + /* pad with 0 */ + if (read_size < block_size) { + memset(block_buffer + read_size, 0, block_size - read_size); + } + /* Here isn't needed to call posix_fadvise(..., POSIX_FADV_DONTNEED) because */ /* we already advised sequential access with POSIX_FADV_SEQUENTIAL. */ /* In Linux 2.6.33 it's enough to ensure that data is not kept in the cache. */ diff --git a/handle.h b/handle.h index 19833201..ec000e0b 100644 --- a/handle.h +++ b/handle.h @@ -53,6 +53,7 @@ int handle_close(struct snapraid_handle* handle); /** * Read a block from a file. + * If the read block is shorter, it's padded with 0. */ int handle_read(struct snapraid_handle* handle, struct snapraid_block* block, unsigned char* block_buffer, unsigned block_size); diff --git a/murmurhash3.c b/murmurhash3.c new file mode 100644 index 00000000..196dd053 --- /dev/null +++ b/murmurhash3.c @@ -0,0 +1,155 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* Derivative work from MurmorHash3.cpp from http://code.google.com/p/smhasher/ */ + +// MurmurHash3 was written by Austin Appleby, and is placed in the public +// domain. The author hereby disclaims copyright to this source code. + +// Note - The x86 and x64 versions do _not_ produce the same results, as the +// algorithms are optimized for their respective platforms. You can still +// compile and run any of them on any platform, but your performance with the +// non-native version will be less than optimal. + +/* Rotate left */ +inline uint32_t rotl32 ( uint32_t x, int8_t r ) +{ + return (x << r) | (x >> (32 - r)); +} + +/* Swap endianess */ +#if WORDS_BIGENDIAN +static inline uint32_t swap32(uint32_t v) +{ + return (((v) & 0xff) << 24 | ((v) & 0xff00) << 8 | ((v) & 0xff0000) >> 8 | ((v) >> 24) & 0xff); +} +#else +static inline uint32_t swap32(uint32_t v) +{ + return v; +} +#endif + +/* Finalization mix - force all bits of a hash block to avalanche */ +static inline uint32_t fmix32(uint32_t h) +{ + h ^= h >> 16; + h *= 0x85ebca6b; + h ^= h >> 13; + h *= 0xc2b2ae35; + h ^= h >> 16; + return h; +} + +uint32_t c1 = 0x239b961b; +uint32_t c2 = 0xab0e9789; +uint32_t c3 = 0x38b34ae5; +uint32_t c4 = 0xa1e38b93; + +void MurmurHash3_x86_128(const void * key, unsigned len, uint32_t seed, void* out) +{ + const uint8_t * data = (const uint8_t*)key; + unsigned nblocks = len / 16; + unsigned i; + + uint32_t h1 = seed; + uint32_t h2 = seed; + uint32_t h3 = seed; + uint32_t h4 = seed; + + const uint32_t* blocks = (const uint32_t*)data; + + /* body */ + + for(i=0;i. + */ + +/* -*- linux-c -*- ------------------------------------------------------- * + * + * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, Inc., 53 Temple Place Ste 330, + * Boston MA 02111-1307, USA; either version 2 of the License, or + * (at your option) any later version; incorporated herein by reference. + * + * ----------------------------------------------------------------------- */ + +#include "portable.h" + +#include "raid.h" +#include "cpu.h" + +/* + * 2-way unrolled portable integer math RAID-5 instruction set + */ +void raid5_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size) +{ + unsigned char* p; + int z, z0; + unsigned d; + + uint32_t wp0; + uint32_t wp1; + + z0 = diskmax - 2; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + + for(d=0;d=0;--z) { + wp0 ^= *(uint32_t *)&dptr[z][d+0*4]; + wp1 ^= *(uint32_t *)&dptr[z][d+1*4]; + } + *(uint32_t *)&p[d+4*0] = wp0; + *(uint32_t *)&p[d+4*1] = wp1; + } +} + +/* + * Unrolled-by-2 MMX implementation + */ +void raid5_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size) +{ + unsigned char* p; + int z, z0; + unsigned d; + + z0 = diskmax - 2; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + + for(d=0;d=0;--z) { + asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d])); + asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8])); + asm volatile("pxor %mm5,%mm2"); + asm volatile("pxor %mm7,%mm3"); + } + asm volatile("movq %%mm2,%0" : "=m" (p[d])); + asm volatile("movq %%mm3,%0" : "=m" (p[d+8])); + } + + asm volatile("emms" : : : "memory"); +} + +void raid5_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size) +{ + unsigned char* p; + int z, z0; + unsigned d; + + z0 = diskmax - 2; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + + for(d=0;d=0;--z) { + asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d])); + asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16])); + asm volatile("pxor %xmm5,%xmm2"); + asm volatile("pxor %xmm7,%xmm3"); + } + asm volatile("movntdq %%xmm2,%0" : "=m" (p[d])); + asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16])); + } + + asm volatile("sfence" : : : "memory"); +} + +/* + * Repeats an 8 bit nibble to a full 32 bits value. + */ +#define NBYTES(x) ((x) * 0x01010101U) + +/* + * These sub-operations are separate inlines since they can sometimes be + * specially optimized using architecture-specific hacks. + */ + +/* + * The SHLBYTE() operation shifts each byte left by 1, *not* + * rolling over into the next byte. + */ +static inline uint32_t SHLBYTE(uint32_t v) +{ + uint32_t vv; + + vv = (v << 1) & NBYTES(0xfe); + return vv; +} + +/* + * The MASK() operation returns 0xFF in any byte for which the high + * bit is 1, 0x00 for any byte for which the high bit is 0. + */ +static inline uint32_t MASK(uint32_t v) +{ + uint32_t vv; + + vv = v & NBYTES(0x80); + vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */ + return vv; +} + +/* + * 2-way unrolled portable integer math RAID-6 instruction set + */ +void raid6_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size) +{ + unsigned char* p; + unsigned char* q; + int z, z0; + unsigned d; + + uint32_t wd0, wq0, wp0, w10, w20; + uint32_t wd1, wq1, wp1, w11, w21; + + z0 = diskmax - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS qarity */ + + for(d=0;d= 0;--z) { + wd0 = *(uint32_t *)&dptr[z][d+0*4]; + wd1 = *(uint32_t *)&dptr[z][d+1*4]; + wp0 ^= wd0; + wp1 ^= wd1; + w20 = MASK(wq0); + w21 = MASK(wq1); + w10 = SHLBYTE(wq0); + w11 = SHLBYTE(wq1); + w20 &= NBYTES(0x1d); + w21 &= NBYTES(0x1d); + w10 ^= w20; + w11 ^= w21; + wq0 = w10 ^ wd0; + wq1 = w11 ^ wd1; + } + *(uint32_t *)&p[d+4*0] = wp0; + *(uint32_t *)&p[d+4*1] = wp1; + *(uint32_t *)&q[d+4*0] = wq0; + *(uint32_t *)&q[d+4*1] = wq1; + } +} + +#if defined(__i386__) || defined(__x86_64__) + +static const struct raid6_mmx_constants { + uint64_t x1d; +} raid6_mmx_constants = { + 0x1d1d1d1d1d1d1d1dULL, +}; + +/* + * Unrolled-by-2 MMX implementation + */ +void raid6_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size) +{ + unsigned char* p; + unsigned char* q; + int z, z0; + unsigned d; + + z0 = diskmax - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS qarity */ + + asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d)); + asm volatile("pxor %mm5,%mm5"); /* Zero temp */ + asm volatile("pxor %mm7,%mm7"); /* Zero temp */ + + for(d=0;d=0;--z) { + asm volatile("pcmpgtb %mm4,%mm5"); + asm volatile("pcmpgtb %mm6,%mm7"); + asm volatile("paddb %mm4,%mm4"); + asm volatile("paddb %mm6,%mm6"); + asm volatile("pand %mm0,%mm5"); + asm volatile("pand %mm0,%mm7"); + asm volatile("pxor %mm5,%mm4"); + asm volatile("pxor %mm7,%mm6"); + asm volatile("movq %0,%%mm5" : : "m" (dptr[z][d])); + asm volatile("movq %0,%%mm7" : : "m" (dptr[z][d+8])); + asm volatile("pxor %mm5,%mm2"); + asm volatile("pxor %mm7,%mm3"); + asm volatile("pxor %mm5,%mm4"); + asm volatile("pxor %mm7,%mm6"); + asm volatile("pxor %mm5,%mm5"); + asm volatile("pxor %mm7,%mm7"); + } + asm volatile("movq %%mm2,%0" : "=m" (p[d])); + asm volatile("movq %%mm3,%0" : "=m" (p[d+8])); + asm volatile("movq %%mm4,%0" : "=m" (q[d])); + asm volatile("movq %%mm6,%0" : "=m" (q[d+8])); + } + + asm volatile("emms" : : : "memory"); +} + +static const struct raid6_sse_constants { + uint64_t x1d[2]; +} raid6_sse_constants __attribute__((aligned(16))) = { + { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, +}; + +/* + * Unrolled-by-2 SSE2 implementation + */ +void raid6_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size) +{ + unsigned char* p; + unsigned char* q; + int z, z0; + unsigned d; + + z0 = diskmax - 3; /* Highest data disk */ + p = dptr[z0+1]; /* XOR parity */ + q = dptr[z0+2]; /* RS qarity */ + + asm volatile("movdqa %0,%%xmm0" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("pxor %xmm5,%xmm5"); /* Zero temp */ + asm volatile("pxor %xmm7,%xmm7"); /* Zero temp */ + + /* We uniformly assume a single prefetch covers at least 32 bytes */ + for(d=0;d=0;--z) { + asm volatile("prefetchnta %0" : : "m" (dptr[z][d])); + asm volatile("pcmpgtb %xmm4,%xmm5"); + asm volatile("pcmpgtb %xmm6,%xmm7"); + asm volatile("paddb %xmm4,%xmm4"); + asm volatile("paddb %xmm6,%xmm6"); + asm volatile("pand %xmm0,%xmm5"); + asm volatile("pand %xmm0,%xmm7"); + asm volatile("pxor %xmm5,%xmm4"); + asm volatile("pxor %xmm7,%xmm6"); + asm volatile("movdqa %0,%%xmm5" : : "m" (dptr[z][d])); + asm volatile("movdqa %0,%%xmm7" : : "m" (dptr[z][d+16])); + asm volatile("pxor %xmm5,%xmm2"); + asm volatile("pxor %xmm7,%xmm3"); + asm volatile("pxor %xmm5,%xmm4"); + asm volatile("pxor %xmm7,%xmm6"); + asm volatile("pxor %xmm5,%xmm5"); + asm volatile("pxor %xmm7,%xmm7"); + } + asm volatile("movntdq %%xmm2,%0" : "=m" (p[d])); + asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+16])); + asm volatile("movntdq %%xmm4,%0" : "=m" (q[d])); + asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+16])); + } + + asm volatile("sfence" : : : "memory"); +} +#endif + +void (*raid5_gen)(unsigned char** dptr, unsigned diskmax, unsigned block_size); +void (*raid6_gen)(unsigned char** dptr, unsigned diskmax, unsigned block_size); + +void raid_init(void) +{ + raid5_gen = raid5_int32r2; + raid6_gen = raid6_int32r2; +#if defined(__i386__) || defined(__x86_64__) +/* + if (cpu_has_mmx()) { + raid5_gen = raid5_mmxr2; + raid6_gen = raid6_mmxr2; + } + if (cpu_has_sse2()) { + raid5_gen = raid5_sse2r2; + raid6_gen = raid6_sse2r2; + } +*/ +#endif +} + diff --git a/raid.h b/raid.h new file mode 100644 index 00000000..610d6c7d --- /dev/null +++ b/raid.h @@ -0,0 +1,41 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __RAID_H +#define __RAID_H + +/****************************************************************************/ +/* raid */ + +void raid5_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size); +void raid5_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size); +void raid5_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size); + +void raid6_int32r2(unsigned char** dptr, unsigned diskmax, unsigned block_size); +void raid6_mmxr2(unsigned char** dptr, unsigned diskmax, unsigned block_size); +void raid6_sse2r2(unsigned char** dptr, unsigned diskmax, unsigned block_size); + +extern void (*raid5_gen)(uint8_t** ptrs, unsigned disks, unsigned block_size); +extern void (*raid6_gen)(uint8_t** ptrs, unsigned disks, unsigned block_size); + +/** + * Initializes the RAID system. + */ +void raid_init(void); + +#endif + diff --git a/selftest.c b/selftest.c new file mode 100644 index 00000000..1460a0e9 --- /dev/null +++ b/selftest.c @@ -0,0 +1,111 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "snapraid.h" +#include "util.h" +#include "raid.h" + +struct test_vector { + const char* data; + unsigned char digest[HASH_SIZE]; +}; + +/** + * Test vectors for MD5 from RFC1321. + * + * MD5("") = d41d8cd98f00b204e9800998ecf8427e + * MD5("a") = 0cc175b9c0f1b6a831c399e269772661 + * MD5("abc") = 900150983cd24fb0d6963f7d28e17f72 + * MD5("message digest") = f96b697d7cb7938d525a2f31aaf161d0 + * MD5("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b + * MD5("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f + * MD5("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a + */ +struct test_vector TEST_MD5[] = { +{ "", { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } }, +{ "a", { 0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8, 0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61 } }, +{ "abc", { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } }, +{ "message digest", { 0xf9, 0x6b, 0x69, 0x7d, 0x7c, 0xb7, 0x93, 0x8d, 0x52, 0x5a, 0x2f, 0x31, 0xaa, 0xf1, 0x61, 0xd0 } }, +{ "abcdefghijklmnopqrstuvwxyz", { 0xc3, 0xfc, 0xd3, 0xd7, 0x61, 0x92, 0xe4, 0x00, 0x7d, 0xfb, 0x49, 0x6c, 0xca, 0x67, 0xe1, 0x3b } }, +{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xd1, 0x74, 0xab, 0x98, 0xd2, 0x77, 0xd9, 0xf5, 0xa5, 0x61, 0x1c, 0x2c, 0x9f, 0x41, 0x9d, 0x9f } }, +{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0x57, 0xed, 0xf4, 0xa2, 0x2b, 0xe3, 0xc9, 0x55, 0xac, 0x49, 0xda, 0x2e, 0x21, 0x07, 0xb6, 0x7a, } }, +{ 0, { 0 } } +}; + +/** + * Test vectors for MurmorHash3_x86_128 + */ +struct test_vector TEST_MURMUR3[] = { +{ "", { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 } }, +{ "a", { 0x3c, 0x93, 0x94, 0xa7, 0x1b, 0xb0, 0x56, 0x55, 0x1b, 0xb0, 0x56, 0x55, 0x1b, 0xb0, 0x56, 0x55 } }, +{ "abc", { 0xd1, 0xc6, 0xcd, 0x75, 0xa5, 0x06, 0xb0, 0xa2, 0xa5, 0x06, 0xb0, 0xa2, 0xa5, 0x06, 0xb0, 0xa2 } }, +{ "message digest", { 0xea, 0x7b, 0x82, 0xc1, 0x89, 0x77, 0xdb, 0x26, 0xb0, 0x30, 0xfe, 0xbc, 0xef, 0x42, 0x4b, 0xda } }, +{ "abcdefghijklmnopqrstuvwxyz", { 0x13, 0x06, 0x34, 0x3e, 0x66, 0x2f, 0x6f, 0x66, 0x6e, 0x56, 0xf6, 0x17, 0x2c, 0x3d, 0xe3, 0x44 } }, +{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xe7, 0xd6, 0x62, 0x29, 0x32, 0xf7, 0xdd, 0x48, 0x29, 0x38, 0xed, 0x07, 0x32, 0x0c, 0x72, 0x12 } }, +{ "1234567890123456789012345678901234567890123456789012345678901234", { 0x8e, 0x2e, 0xd6, 0x0c, 0xbb, 0x4d, 0x10, 0x7e, 0x7a, 0x78, 0x72, 0x2e, 0xab, 0x9a, 0x6c, 0x0d } }, +{ "12345678901234567890123456789012345678901234567890123456789012345", { 0x45, 0xe6, 0x26, 0xa0, 0xf5, 0xdb, 0xf8, 0x22, 0x3b, 0x5e, 0xc5, 0x41, 0x10, 0xc5, 0x06, 0x36 } }, +{ "123456789012345678901234567890123456789012345678901234567890123456", { 0xb5, 0x5e, 0xdd, 0xa2, 0xa6, 0x65, 0xfd, 0x53, 0xec, 0xb8, 0xc9, 0x93, 0x97, 0x97, 0x65, 0x22 } }, +{ "1234567890123456789012345678901234567890123456789012345678901234567", { 0x19, 0x40, 0x6f, 0xb8, 0xc4, 0x30, 0x59, 0x00, 0x56, 0x10, 0xe7, 0xdd, 0x9a, 0x48, 0x83, 0xa5 } }, +{ "12345678901234567890123456789012345678901234567890123456789012345678", { 0x77, 0xa8, 0xb1, 0xe6, 0xb2, 0xbc, 0x2c, 0x4f, 0x91, 0x0c, 0x32, 0xb3, 0x1d, 0x28, 0xf2, 0xd4 } }, +{ "123456789012345678901234567890123456789012345678901234567890123456789", { 0xe0, 0x5a, 0x5f, 0xe7, 0x0c, 0x2a, 0xa9, 0x63, 0x9a, 0x8e, 0x41, 0xbe, 0x73, 0xa3, 0x25, 0xe4 } }, +{ "1234567890123456789012345678901234567890123456789012345678901234567890", { 0x8e, 0x8f, 0x93, 0xcf, 0xdf, 0x1f, 0xe5, 0xfe, 0x09, 0x35, 0xe3, 0x53, 0x1e, 0x5b, 0x75, 0x37 } }, +{ "12345678901234567890123456789012345678901234567890123456789012345678901", { 0x46, 0xd5, 0xd2, 0xa3, 0x6d, 0xc9, 0x20, 0x0d, 0xd6, 0x6a, 0x90, 0x9e, 0xda, 0xfb, 0x1c, 0x26 } }, +{ "123456789012345678901234567890123456789012345678901234567890123456789012", { 0x4a, 0x26, 0xe4, 0xae, 0x00, 0x32, 0x08, 0xfd, 0x0b, 0xee, 0x6c, 0x0f, 0xb2, 0x64, 0xa4, 0x25 } }, +{ "1234567890123456789012345678901234567890123456789012345678901234567890123", { 0x2a, 0x97, 0x3f, 0xe8, 0xb4, 0xf9, 0x4e, 0xf9, 0x45, 0xe2, 0xe4, 0x90, 0x69, 0x14, 0xa6, 0x24 } }, +{ "12345678901234567890123456789012345678901234567890123456789012345678901234", { 0x4b, 0xd7, 0x48, 0x1a, 0x25, 0x8a, 0x2a, 0xab, 0x50, 0xc3, 0xc6, 0xd0, 0x24, 0x62, 0x78, 0xba } }, +{ "123456789012345678901234567890123456789012345678901234567890123456789012345", { 0x88, 0xe4, 0x8f, 0x6f, 0xee, 0x22, 0xe5, 0x89, 0x78, 0x24, 0x99, 0x07, 0x73, 0x7e, 0x5b, 0xfd } }, +{ "1234567890123456789012345678901234567890123456789012345678901234567890123456", { 0xdd, 0x18, 0x02, 0x02, 0xa9, 0xcd, 0x6e, 0xcb, 0xfe, 0x91, 0x8e, 0x82, 0x4e, 0xa8, 0x33, 0xfe } }, +{ "12345678901234567890123456789012345678901234567890123456789012345678901234567", { 0x80, 0xf2, 0xb8, 0x67, 0x5a, 0x02, 0x78, 0x0c, 0xcb, 0x31, 0xd0, 0xf3, 0xb6, 0x67, 0x52, 0xca } }, +{ "123456789012345678901234567890123456789012345678901234567890123456789012345678", { 0x66, 0x85, 0x66, 0x0f, 0xc7, 0x56, 0xfb, 0xb7, 0x9b, 0xa2, 0x28, 0x37, 0x54, 0x89, 0xbb, 0x48 } }, +{ "1234567890123456789012345678901234567890123456789012345678901234567890123456789", { 0x5b, 0x59, 0x21, 0x42, 0xae, 0x72, 0xbe, 0xd4, 0x4a, 0xa7, 0xf2, 0x12, 0x35, 0x48, 0xc6, 0xd8 } }, +{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0xf2, 0x6c, 0x2a, 0x8f, 0x58, 0x23, 0x4c, 0x8e, 0x2c, 0x74, 0x41, 0x66, 0x0e, 0xc2, 0xd3, 0x91 } }, +{ 0, { 0 } } +}; + +void selftest(void) +{ + unsigned i; + + printf("Self test...\n"); + + /* large file check */ + if (sizeof(off_t) < sizeof(uint64_t)) { + fprintf(stderr, "Missing support for large files\n"); + exit(EXIT_FAILURE); + } + + for(i=0;TEST_MD5[i].data;++i) { + unsigned char digest[HASH_SIZE]; + memhash(HASH_MD5, digest, TEST_MD5[i].data, strlen(TEST_MD5[i].data)); + if (memcmp(digest, TEST_MD5[i].digest, HASH_SIZE) != 0) { + fprintf(stderr, "Failed MD5 test vector\n"); + exit(EXIT_FAILURE); + } + } + + for(i=0;TEST_MURMUR3[i].data;++i) { + unsigned char digest[HASH_SIZE]; + memhash(HASH_MURMUR3, digest, TEST_MURMUR3[i].data, strlen(TEST_MURMUR3[i].data)); + if (memcmp(digest, TEST_MURMUR3[i].digest, HASH_SIZE) != 0) { + fprintf(stderr, "Failed Murmur3 test vector\n"); + exit(EXIT_FAILURE); + } + } +} + diff --git a/snapraid.1 b/snapraid.1 index 9ffd99f2..6e8e5024 100644 --- a/snapraid.1 +++ b/snapraid.1 @@ -23,7 +23,7 @@ snapraid [\(hyc, \(hy\(hyconf CONFIG] [\(hyf, \(hy\(hyfilter PATTERN] .PP .PD .PP -snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp] [\(hyT, \(hy\(hyspeed\(hytest] +snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp] .PD 0 .PP .PD @@ -31,7 +31,7 @@ snapraid [\(hyV, \(hy\(hyversion] [\(hyh, \(hy\(hyhelp] [\(hyT, \(hy\(hyspeed\(h SnapRAID is a backup program for a disk array using redundancy. .PP SnapRAID uses a disk of the array to store redundancy information, -and it allows to recover from a disk failure. +and it allows to recover from up to two disk failures. .PP SnapRAID is mainly targeted for a home media center, where you have a lot of big files that rarely change. @@ -49,7 +49,7 @@ You can add more disks at the array at any time. If you accidentally delete some files in a disk, you can recover them. .IP \(bu -If more than one disk fails, you lose the data only on the +If more than two disk fails, you lose the data only on the failed disks. All the data in the other disks is safe. .IP \(bu It doesn\(cqt lock\(hyin your data. You can stop using SnapRAID at any @@ -96,7 +96,7 @@ With RAID you get a speed boost with striping. It doesn\(cqt support real\(hytime recovery. With RAID you do not have to stop working when a disk fails. .IP \(bu -It\(cqs able to recover damages only from a single disk. +It\(cqs able to recover damages only from up to two disks. With a Backup you are able to recover from a complete failure of the whole disk array. .IP \(bu @@ -109,18 +109,25 @@ hard\(hylinks, symbolic\(hylinks are not saved. .PD .SH GETTING STARTED To use SnapRAID you need first select one disk of your disk array -to dedicate at the redundancy information. +to dedicate at the parity information. With parity you will be able +to recover from a single disk failure. .PP -You have to pick the biggest disk in the array, as the redundancy -information may grow in size as the biggest data disk in the array. +If you want to recover from two disk failures you must reserve +another disk for the q\(hyparity information. .PP -This disk will be dedicated to store the parity and content metadata -information. You should not store other data on it. +For both disks, you have to pick the biggest disks in the array, +as the redundancy information may grow in size as the biggest data +disk in the array. .PP -For example, if all your disks are present in: +This disk will be dedicated to store the parity and q\(hyparity. +You should not store other data on it, with the exception of +the content metadata information. +.PP +For example, suppose that you are interested only at the parity level +of protection, and that all your disks are present in: .PP .RS 4 -/mnt/diskpar <\(hy selected disk for parity and metadata content +/mnt/diskpar <\(hy selected disk for parity .PD 0 .PP .PD @@ -258,7 +265,7 @@ meaning that you can move them on the disk without triggering any redundancy recomputation. You can stop this process at any time pressing Ctrl+C, without losing the work already done. -The \(a"content\(a" and \(a"parity\(a" files are modified if necessary. +The \(a"content\(a", \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary. The files in the array are NOT modified. .TP .B check @@ -275,7 +282,7 @@ disk array at the previous \(a"sync\(a" command. After a successful \(a"fix\(a", you should also run a \(a"sync\(a" command to update the new state of the files. The \(a"content\(a" file is NOT modified. -The \(a"parity\(a" file is modified if necessary. +The \(a"parity\(a" and \(a"q\(hyparity\(a" files are modified if necessary. The files in the array are modified if necessary. .SH OPTIONS .TP @@ -315,17 +322,11 @@ or fix some specific block, in case of a damaged disk. .TP .B \(hyt, \(hy\(hycount BLKCOUNT Process only the specified number of blocks. -It\(cqs present mainly for testing. +It\(cqs present mainly for advanced manual recovering. .TP .B \(hyv, \(hy\(hyverbose Prints more information in the processing. .TP -.B \(hyT, \(hy\(hyspeed\(hytest -Executes a speed test of the MD5 implementation used. -If available, SnapRAID uses the fast MD5 implementation -from the OpenSSL crypto library. Otherwise it uses a -slower C implementation. -.TP .B \(hyh, \(hy\(hyhelp Prints a short help screen. .TP @@ -339,19 +340,29 @@ This configuration file is located in /etc/snapraid.conf and it should contains the following options: .TP .B parity FILE -Defines the file to use to store the redundancy information. +Defines the file to use to store the parity information. It must be placed in a disk dedicated for this purpose with as much free space as the biggest disk in the array. Leaving the parity disk reserved for only this file, ensures that it doesn\(cqt get fragmented, improving the performance. -This option can be used only one time. +This option is mandatory and it can be used only one time. +.TP +.B q\(hyparity FILE +Defines the file to use to store the q\(hyparity information. +This option is optional, and if present it enables the protection +from two disk failures. +It must be placed in a disk dedicated for this purpose with +as much free space as the biggest disk in the array. +Leaving the q\(hyparity disk reserved for only this file, ensures that +it doesn\(cqt get fragmented, improving the performance. +This option is optional and it can be used only one time. .TP .B content FILE Defines the file to use to store the content of the redundancy organization. It can be placed in the same disk of the parity file, or better in another disk, but NOT in a data disk of the array. -This option can be used only one time. +This option is mandatory and it can be used only one time. .TP .B disk NAME DIR Defines the name and the mount point of the disks of the array. @@ -550,7 +561,7 @@ to identify the file. Defines the ordered parity block list used by the last defined file. BLOCK is the block position in the \(a"parity\(a" file. 0 for the first block, 1 for the second one and so on. -HASH is the md5 of the block. In the last block of the file, +HASH is the hash of the block. In the last block of the file, the HASH is the hash of only the used part of the block. .TP .B inv BLOCK [HASH] @@ -559,18 +570,25 @@ The HASH may be missing if not yet computed. This field is used only when you interrupt manually the \(a"sync\(a" command. .SH PARITY -SnapRAID creates a parity file containing the redundancy information -of your disk array. +SnapRAID creates the parity and q\(hyparity files containing the redundancy +information of your disk array. .PP -It\(cqs a binary file, containing the computed parity of all the blocks -defined in the \(a"content\(a" file. You do not need to understand its format, -but it\(cqs described here for documentation. +They are binary files, containing the computed redundancy of all the +blocks defined in the \(a"content\(a" file. You do not need to understand its +format, but it\(cqs described here for documentation. .PP -This file is read and written by the \(a"sync\(a" and \(a"fix\(a" commands, and +These files are read and written by the \(a"sync\(a" and \(a"fix\(a" commands, and only read by \(a"check\(a". .PP -For all the blocks at a given position, the parity information is -computed with the XOR operator applied to all the blocks. +For all the blocks at a given position, the parity and the q\(hyparity +are computed as specified in: +.PP +.RS 4 +kernel.org/pub/linux/kernel/people/hpa/raid6.pdf +.PD 0 +.PP +.PD +.RE .PP When a file block is shorter than the default block size, for example because it\(cqs the last block of a file, it\(cqs assumed as filled with 0 diff --git a/snapraid.c b/snapraid.c index a442bea4..90024344 100644 --- a/snapraid.c +++ b/snapraid.c @@ -17,9 +17,11 @@ #include "portable.h" +#include "snapraid.h" #include "util.h" #include "elem.h" #include "state.h" +#include "raid.h" /****************************************************************************/ /* main */ @@ -33,91 +35,6 @@ #define CONF "/etc/" PACKAGE ".conf" #endif -/** - * Test vectors for MD5 from RFC1321. - * - * MD5("") = d41d8cd98f00b204e9800998ecf8427e - * MD5("a") = 0cc175b9c0f1b6a831c399e269772661 - * MD5("abc") = 900150983cd24fb0d6963f7d28e17f72 - * MD5("message digest") = f96b697d7cb7938d525a2f31aaf161d0 - * MD5("abcdefghijklmnopqrstuvwxyz") = c3fcd3d76192e4007dfb496cca67e13b - * MD5("ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789") = d174ab98d277d9f5a5611c2c9f419d9f - * MD5("12345678901234567890123456789012345678901234567890123456789012345678901234567890") = 57edf4a22be3c955ac49da2e2107b67a - */ -struct test_vector { - const char* data; - unsigned char digest[HASH_SIZE]; -} TEST[] = { -{ "", { 0xd4, 0x1d, 0x8c, 0xd9, 0x8f, 0x00, 0xb2, 0x04, 0xe9, 0x80, 0x09, 0x98, 0xec, 0xf8, 0x42, 0x7e } }, -{ "a", { 0x0c, 0xc1, 0x75, 0xb9, 0xc0, 0xf1, 0xb6, 0xa8, 0x31, 0xc3, 0x99, 0xe2, 0x69, 0x77, 0x26, 0x61 } }, -{ "abc", { 0x90, 0x01, 0x50, 0x98, 0x3c, 0xd2, 0x4f, 0xb0, 0xd6, 0x96, 0x3f, 0x7d, 0x28, 0xe1, 0x7f, 0x72 } }, -{ "message digest", { 0xf9, 0x6b, 0x69, 0x7d, 0x7c, 0xb7, 0x93, 0x8d, 0x52, 0x5a, 0x2f, 0x31, 0xaa, 0xf1, 0x61, 0xd0 } }, -{ "abcdefghijklmnopqrstuvwxyz", { 0xc3, 0xfc, 0xd3, 0xd7, 0x61, 0x92, 0xe4, 0x00, 0x7d, 0xfb, 0x49, 0x6c, 0xca, 0x67, 0xe1, 0x3b } }, -{ "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789", { 0xd1, 0x74, 0xab, 0x98, 0xd2, 0x77, 0xd9, 0xf5, 0xa5, 0x61, 0x1c, 0x2c, 0x9f, 0x41, 0x9d, 0x9f } }, -{ "12345678901234567890123456789012345678901234567890123456789012345678901234567890", { 0x57, 0xed, 0xf4, 0xa2, 0x2b, 0xe3, 0xc9, 0x55, 0xac, 0x49, 0xda, 0x2e, 0x21, 0x07, 0xb6, 0x7a, } }, -{ 0, { 0 } } -}; - -void selftest(void) -{ - unsigned i; - - printf("Self test...\n"); - - /* large file check */ - if (sizeof(off_t) < sizeof(uint64_t)) { - fprintf(stderr, "Missing support for large files\n"); - exit(EXIT_FAILURE); - } - - /* test vectors for MD5. It could be dynamically linked, so we have to test it at runtime */ - for(i=0;TEST[i].data;++i) { - unsigned char digest[HASH_SIZE]; - memhash(digest, TEST[i].data, strlen(TEST[i].data)); - if (memcmp(digest, TEST[i].digest, HASH_SIZE) != 0) { - fprintf(stderr, "Failed MD5 test vector\n"); - exit(EXIT_FAILURE); - } - } -} - -void speed(void) -{ - struct timeval start; - struct timeval stop; - int64_t ds; - int64_t dt; - unsigned i; - unsigned char digest[HASH_SIZE]; - - unsigned count = 10000; - unsigned blocksize = 256 * 1024; - void* block = malloc_nofail(blocksize); - -#if HAVE_LIBCRYPTO - printf("Using OpenSSL libcrypto MD5\n"); -#else - printf("Using portable MD5\n"); -#endif -#ifdef __GNUC__ - printf("Compiled with gcc " __VERSION__ "\n"); -#endif - - printf("Hashing speed test...\n"); - - gettimeofday(&start, 0); - for(i=0;i. + */ + +#ifndef __SNAPRAID_H +#define __SNAPRAID_H + +/****************************************************************************/ +/* snapraid */ + +void speed(void); +void selftest(void); + +#endif + diff --git a/snapraid.txt b/snapraid.txt index c03baeff..14b94570 100644 --- a/snapraid.txt +++ b/snapraid.txt @@ -12,7 +12,7 @@ snapraid [-c, --conf CONFIG] [-f, --filter PATTERN] [-v, --verbose] COMMAND -snapraid [-V, --version] [-h, --help] [-T, --speed-test] +snapraid [-V, --version] [-h, --help] 2 DESCRIPTION @@ -21,7 +21,7 @@ snapraid [-V, --version] [-h, --help] [-T, --speed-test] SnapRAID is a backup program for a disk array using redundancy. SnapRAID uses a disk of the array to store redundancy information, -and it allows to recover from a disk failure. +and it allows to recover from up to two disk failures. SnapRAID is mainly targeted for a home media center, where you have a lot of big files that rarely change. @@ -34,7 +34,7 @@ features of SnapRAID are: * You can add more disks at the array at any time. * If you accidentally delete some files in a disk, you can recover them. -* If more than one disk fails, you lose the data only on the +* If more than two disk fails, you lose the data only on the failed disks. All the data in the other disks is safe. * It doesn't lock-in your data. You can stop using SnapRAID at any time without the need to reformat or move data. @@ -73,7 +73,7 @@ Other limitations are: With RAID you get a speed boost with striping. * It doesn't support real-time recovery. With RAID you do not have to stop working when a disk fails. -* It's able to recover damages only from a single disk. +* It's able to recover damages only from up to two disks. With a Backup you are able to recover from a complete failure of the whole disk array. * It identifies changes checking file time and size. If you have @@ -87,17 +87,24 @@ Other limitations are: ================= To use SnapRAID you need first select one disk of your disk array -to dedicate at the redundancy information. +to dedicate at the parity information. With parity you will be able +to recover from a single disk failure. -You have to pick the biggest disk in the array, as the redundancy -information may grow in size as the biggest data disk in the array. +If you want to recover from two disk failures you must reserve +another disk for the q-parity information. -This disk will be dedicated to store the parity and content metadata -information. You should not store other data on it. +For both disks, you have to pick the biggest disks in the array, +as the redundancy information may grow in size as the biggest data +disk in the array. -For example, if all your disks are present in: +This disk will be dedicated to store the parity and q-parity. +You should not store other data on it, with the exception of +the content metadata information. - /mnt/diskpar <- selected disk for parity and metadata content +For example, suppose that you are interested only at the parity level +of protection, and that all your disks are present in: + + /mnt/diskpar <- selected disk for parity /mnt/disk1 <- first disk to backup /mnt/disk2 <- second disk to backup /mnt/disk3 <- third disk to backup @@ -173,7 +180,7 @@ SnapRAID provides three simple commands that allow to: any redundancy recomputation. You can stop this process at any time pressing Ctrl+C, without losing the work already done. - The "content" and "parity" files are modified if necessary. + The "content", "parity" and "q-parity" files are modified if necessary. The files in the array are NOT modified. check @@ -190,7 +197,7 @@ SnapRAID provides three simple commands that allow to: After a successful "fix", you should also run a "sync" command to update the new state of the files. The "content" file is NOT modified. - The "parity" file is modified if necessary. + The "parity" and "q-parity" files are modified if necessary. The files in the array are modified if necessary. @@ -233,17 +240,11 @@ SnapRAID provides three simple commands that allow to: -t, --count BLKCOUNT Process only the specified number of blocks. - It's present mainly for testing. + It's present mainly for advanced manual recovering. -v, --verbose Prints more information in the processing. - -T, --speed-test - Executes a speed test of the MD5 implementation used. - If available, SnapRAID uses the fast MD5 implementation - from the OpenSSL crypto library. Otherwise it uses a - slower C implementation. - -h, --help Prints a short help screen. @@ -261,19 +262,29 @@ This configuration file is located in /etc/snapraid.conf and it should contains the following options: parity FILE - Defines the file to use to store the redundancy information. + Defines the file to use to store the parity information. It must be placed in a disk dedicated for this purpose with as much free space as the biggest disk in the array. Leaving the parity disk reserved for only this file, ensures that it doesn't get fragmented, improving the performance. - This option can be used only one time. + This option is mandatory and it can be used only one time. + + q-parity FILE + Defines the file to use to store the q-parity information. + This option is optional, and if present it enables the protection + from two disk failures. + It must be placed in a disk dedicated for this purpose with + as much free space as the biggest disk in the array. + Leaving the q-parity disk reserved for only this file, ensures that + it doesn't get fragmented, improving the performance. + This option is optional and it can be used only one time. content FILE Defines the file to use to store the content of the redundancy organization. It can be placed in the same disk of the parity file, or better in another disk, but NOT in a data disk of the array. - This option can be used only one time. + This option is mandatory and it can be used only one time. disk NAME DIR Defines the name and the mount point of the disks of the array. @@ -407,7 +418,7 @@ is described here. Defines the ordered parity block list used by the last defined file. BLOCK is the block position in the "parity" file. 0 for the first block, 1 for the second one and so on. - HASH is the md5 of the block. In the last block of the file, + HASH is the hash of the block. In the last block of the file, the HASH is the hash of only the used part of the block. inv BLOCK [HASH] @@ -420,18 +431,20 @@ is described here. 10 PARITY ========= -SnapRAID creates a parity file containing the redundancy information -of your disk array. +SnapRAID creates the parity and q-parity files containing the redundancy +information of your disk array. -It's a binary file, containing the computed parity of all the blocks -defined in the "content" file. You do not need to understand its format, -but it's described here for documentation. +They are binary files, containing the computed redundancy of all the +blocks defined in the "content" file. You do not need to understand its +format, but it's described here for documentation. -This file is read and written by the "sync" and "fix" commands, and +These files are read and written by the "sync" and "fix" commands, and only read by "check". -For all the blocks at a given position, the parity information is -computed with the XOR operator applied to all the blocks. +For all the blocks at a given position, the parity and the q-parity +are computed as specified in: + + kernel.org/pub/linux/kernel/people/hpa/raid6.pdf When a file block is shorter than the default block size, for example because it's the last block of a file, it's assumed as filled with 0 diff --git a/speed.c b/speed.c new file mode 100644 index 00000000..4bc8e80f --- /dev/null +++ b/speed.c @@ -0,0 +1,174 @@ +/* + * Copyright (C) 2011 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "portable.h" + +#include "snapraid.h" +#include "util.h" +#include "raid.h" +#include "cpu.h" + +void speed(void) +{ + struct timeval start; + struct timeval stop; + int64_t ds; + int64_t dt; + unsigned i, j; + unsigned char digest[HASH_SIZE]; + + unsigned count = 5000; + unsigned block_size = 256 * 1024; + unsigned diskmax = 4; + void* buffer_alloc; + unsigned char* buffer_aligned; + unsigned char** buffer; + + buffer_aligned = malloc_nofail_align((diskmax + 2) * block_size, &buffer_alloc); + buffer = malloc_nofail((diskmax + 2) * sizeof(void*)); + for(i=0;iverbose = 0; state->force_zero = 0; state->force_empty = 0; + state->expect_unrecoverable = 0; + state->expect_recoverable = 0; state->need_write = 0; state->block_size = 256 * 1024; /* default 256 KiB */ state->content[0] = 0; state->parity[0] = 0; + state->qarity[0] = 0; + state->level = 1; /* default is the lowest protection */ + state->hash = HASH_MURMUR3; /* default is the fastest */ tommy_array_init(&state->diskarr); tommy_list_init(&state->excludelist); } @@ -49,7 +54,7 @@ void state_done(struct snapraid_state* state) tommy_list_foreach(&state->excludelist, (tommy_foreach_func*)filter_free); } -void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty) +void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable) { FILE* f; unsigned line; @@ -57,6 +62,8 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i state->verbose = verbose; state->force_zero = force_zero; state->force_empty = force_empty; + state->expect_unrecoverable = expect_unrecoverable; + state->expect_recoverable = expect_recoverable; f = fopen(path, "rt"); if (!f) { @@ -124,6 +131,13 @@ void state_config(struct snapraid_state* state, const char* path, int verbose, i exit(EXIT_FAILURE); } pathcpy(state->parity, sizeof(state->parity), s); + } else if (strcmp(tag, "q-parity") == 0) { + if (*state->qarity) { + fprintf(stderr, "Multiple 'q-parity' specification in '%s' at line %u\n", path, line); + exit(EXIT_FAILURE); + } + pathcpy(state->qarity, sizeof(state->qarity), s); + state->level = 2; } else if (strcmp(tag, "content") == 0) { if (*state->content) { fprintf(stderr, "Multiple 'content' specification in '%s' at line %u\n", path, line); @@ -186,6 +200,10 @@ void state_read(struct snapraid_state* state) printf("Loading state...\n"); + /* start with a MD5 default. */ + /* it's for compatibility with version 1.0 where MD5 was implicit. */ + state->hash = HASH_MD5; + disk = 0; file = 0; line = 0; @@ -379,6 +397,15 @@ void state_read(struct snapraid_state* state) fprintf(stderr, "Please restore the 'block_size' value in the configuration file to '%u'\n", blksize / 1024); exit(EXIT_FAILURE); } + } else if (strcmp(tag, "checksum") == 0) { + if (strcmp(s, "md5") == 0) { + state->hash = HASH_MD5; + } else if (strcmp(s, "murmur3") == 0) { + state->hash = HASH_MURMUR3; + } else { + fprintf(stderr, "Invalid 'checksum' specification '%s' in '%s' at line %u\n", s, path, line); + exit(EXIT_FAILURE); + } } else { fprintf(stderr, "Unknown tag '%s' in '%s' at line%u\n", tag, path, line); exit(EXIT_FAILURE); @@ -425,6 +452,12 @@ void state_write(struct snapraid_state* state) exit(EXIT_FAILURE); } + ret = fprintf(f, "checksum %s\n", state->hash == HASH_MD5 ? "md5" : "murmur3"); + if (ret < 0) { + fprintf(stderr, "Error writing the content file '%s' in fprintf(). %s.\n", path, strerror(errno)); + exit(EXIT_FAILURE); + } + /* for each disk */ for(i=0;idiskarr);++i) { tommy_node* j; @@ -594,7 +627,6 @@ int state_progress(time_t* start, time_t* last, block_off_t countpos, block_off_ *last = now; } - /* stop if requested */ if (global_interrupt) { printf("\n"); diff --git a/state.h b/state.h index 8faad4c7..5c78f7ce 100644 --- a/state.h +++ b/state.h @@ -26,14 +26,22 @@ */ extern volatile int global_interrupt; +#define HASH_MURMUR3 0 +#define HASH_MD5 1 + struct snapraid_state { int verbose; /**< Verbose output. */ int force_zero; /**< Forced dangerous operations of synching file now with zero size. */ int force_empty; /**< Forced dangerous operations of synching disk now empty. */ + int expect_unrecoverable; /**< Expect presence of unrecoverable error in checking or fixing. */ + int expect_recoverable; /**< Expect presence of recoverable error in checking. */ int need_write; /**< If the state is changed. */ uint32_t block_size; /**< Block size in bytes. */ char content[PATH_MAX]; /**< Path of the content file. */ char parity[PATH_MAX]; /**< Path of the parity file. */ + char qarity[PATH_MAX]; /**< Path of the qarity file. */ + unsigned level; /**< Number of parity levels. 1 for RAID5, 2 for RAID6. */ + unsigned hash; /**< Hash kind used. */ tommy_array diskarr; /**< Disk array. */ tommy_list excludelist; /**< List of exclusion. */ }; @@ -51,7 +59,7 @@ void state_done(struct snapraid_state* state); /** * Read the configuration file. */ -void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty); +void state_config(struct snapraid_state* state, const char* path, int verbose, int force_zero, int force_empty, int expect_unrecoverable, int expect_recoverable); /** * Read the state. diff --git a/sync.c b/sync.c index 1716daa5..6878043b 100644 --- a/sync.c +++ b/sync.c @@ -22,18 +22,21 @@ #include "state.h" #include "parity.h" #include "handle.h" +#include "raid.h" /****************************************************************************/ /* sync */ -static int state_sync_process(struct snapraid_state* state, int parity_f, block_off_t blockstart, block_off_t blockmax) +static int state_sync_process(struct snapraid_state* state, int parity_f, int qarity_f, block_off_t blockstart, block_off_t blockmax) { struct snapraid_handle* handle; unsigned diskmax = tommy_array_size(&state->diskarr); block_off_t i; unsigned j; - unsigned char* block_buffer; - unsigned char* xor_buffer; + void* buffer_alloc; + unsigned char* buffer_aligned; + unsigned char** buffer; + unsigned buffermax; data_off_t countsize; block_off_t countpos; block_off_t countmax; @@ -42,8 +45,14 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ int ret; unsigned unrecoverable_error; - block_buffer = malloc_nofail(state->block_size); - xor_buffer = malloc_nofail(state->block_size); + /* we need disk + 1 for each parity level buffers */ + buffermax = diskmax + state->level; + + buffer_aligned = malloc_nofail_align(buffermax * state->block_size, &buffer_alloc); + buffer = malloc_nofail(buffermax * sizeof(void*)); + for(i=0;iblock_size; + } handle = malloc_nofail(diskmax * sizeof(struct snapraid_handle)); for(i=0;iblock_size); - /* for each disk, process the block */ for(j=0;jblock_size); continue; + } ret = handle_close_if_different(&handle[j], block->file); if (ret == -1) { @@ -150,7 +159,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ goto bail; } - read_size = handle_read(&handle[j], block, block_buffer, state->block_size); + read_size = handle_read(&handle[j], block, buffer[j], state->block_size); if (read_size == -1) { fprintf(stderr, "DANGER! Unexpected read error in a data disk, it isn't possible to sync.\n"); fprintf(stderr, "Stopping to allow recovery. Try with 'snapraid check'\n"); @@ -159,7 +168,7 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ } /* now compute the hash */ - memhash(hash, block_buffer, read_size); + memhash(state->hash, hash, buffer[j], read_size); if (bit_has(block->flag, BLOCK_HAS_HASH)) { /* compare the hash */ @@ -176,21 +185,36 @@ static int state_sync_process(struct snapraid_state* state, int parity_f, block_ memcpy(block->hash, hash, HASH_SIZE); } - /* compute the parity */ - memxor(xor_buffer, block_buffer, read_size); - countsize += read_size; } + /* compute the parity */ + if (state->level == 1) { + raid5_gen(buffer, diskmax + state->level, state->block_size); + } else { + raid6_gen(buffer, diskmax + state->level, state->block_size); + } + /* write the parity */ - ret = parity_write(state->parity, parity_f, i, xor_buffer, state->block_size); + ret = parity_write(state->parity, parity_f, i, buffer[diskmax], state->block_size); if (ret == -1) { - fprintf(stderr, "DANGER! Write error in the parity disk, it isn't possible to sync.\n"); + fprintf(stderr, "DANGER! Write error in the Parity disk, it isn't possible to sync.\n"); fprintf(stderr, "Stopping at block %u\n", i); ++unrecoverable_error; goto bail; } + /* write the qarity */ + if (state->level >= 2) { + ret = parity_write(state->qarity, qarity_f, i, buffer[diskmax+1], state->block_size); + if (ret == -1) { + fprintf(stderr, "DANGER! Write error in the Q-Parity disk, it isn't possible to sync.\n"); + fprintf(stderr, "Stopping at block %u\n", i); + ++unrecoverable_error; + goto bail; + } + } + /* for each disk, mark the blocks as processed */ for(j=0;jparity); - f = parity_create(path, size); - if (f == -1) { - fprintf(stderr, "WARNING! Without an accessible parity file, it isn't possible to sync.\n"); + pathcpy(parity_path, sizeof(parity_path), state->parity); + parity_f = parity_create(parity_path, size); + if (parity_f == -1) { + fprintf(stderr, "WARNING! Without an accessible Parity file, it isn't possible to sync.\n"); exit(EXIT_FAILURE); } + if (state->level >= 2) { + pathcpy(qarity_path, sizeof(qarity_path), state->qarity); + qarity_f = parity_create(qarity_path, size); + if (qarity_f == -1) { + fprintf(stderr, "WARNING! Without an accessible Q-Parity file, it isn't possible to sync.\n"); + exit(EXIT_FAILURE); + } + } else { + qarity_f = -1; + } + unrecoverable_error = 0; /* skip degenerated cases of empty parity, or skipping all */ if (blockstart < blockmax) { - ret = state_sync_process(state, f, blockstart, blockmax); + ret = state_sync_process(state, parity_f, qarity_f, blockstart, blockmax); if (ret == -1) { ++unrecoverable_error; /* continue, as we are already exiting */ } } - ret = parity_sync(path, f); + ret = parity_sync(parity_path, parity_f); if (ret == -1) { - fprintf(stderr, "DANGER! Unexpected sync error in parity disk.\n"); + fprintf(stderr, "DANGER! Unexpected sync error in Parity disk.\n"); ++unrecoverable_error; /* continue, as we are already exiting */ } - ret = parity_close(path, f); + ret = parity_close(parity_path, parity_f); if (ret == -1) { - fprintf(stderr, "DANGER! Unexpected close error in parity disk.\n"); + fprintf(stderr, "DANGER! Unexpected close error in Parity disk.\n"); ++unrecoverable_error; /* continue, as we are already exiting */ } + if (state->level >= 2) { + ret = parity_sync(qarity_path, qarity_f); + if (ret == -1) { + fprintf(stderr, "DANGER! Unexpected sync error in Q-Parity disk.\n"); + ++unrecoverable_error; + /* continue, as we are already exiting */ + } + + ret = parity_close(qarity_path, qarity_f); + if (ret == -1) { + fprintf(stderr, "DANGER! Unexpected close error in Q-Parity disk.\n"); + ++unrecoverable_error; + /* continue, as we are already exiting */ + } + } + /* abort if required */ if (unrecoverable_error != 0) return -1; diff --git a/util.c b/util.c index 51106eee..eafb0bf8 100644 --- a/util.c +++ b/util.c @@ -223,6 +223,24 @@ void* malloc_nofail(size_t size) return ptr; } +#define ALIGN 256 + +void* malloc_nofail_align(size_t size, void** freeptr) +{ + unsigned char* ptr = malloc_nofail(size + ALIGN); + uintptr_t offset; + + *freeptr = ptr; + + offset = ((uintptr_t)ptr) % ALIGN; + + if (offset != 0) { + ptr += ALIGN - offset; + } + + return ptr; +} + void memxor(unsigned char* xor, const unsigned char* block, unsigned size) { while (size >= 16) { @@ -263,15 +281,25 @@ void memxor(unsigned char* xor, const unsigned char* block, unsigned size) #include "md5.c" #endif -void memhash(void* digest, const void* src, unsigned size) +#include "murmurhash3.c" + +void memhash(unsigned kind, void* digest, const void* src, unsigned size) { + if (kind == HASH_MURMUR3) { + MurmurHash3_x86_128(src, size, 0, digest); + return; + } + + if (kind == HASH_MD5) { #if HAVE_LIBCRYPTO - MD5((void*)src, size, digest); + MD5((void*)src, size, digest); #else - struct md5_t md5; - md5_init(&md5); - md5_update(&md5, src, size); - md5_final(&md5, digest); + struct md5_t md5; + md5_init(&md5); + md5_update(&md5, src, size); + md5_final(&md5, digest); #endif + return; + } } diff --git a/util.h b/util.h index a20d8ddc..703952c6 100644 --- a/util.h +++ b/util.h @@ -121,6 +121,12 @@ static inline unsigned bit_clear(unsigned bit, unsigned mask) */ void* malloc_nofail(size_t size); +/** + * Safe aligned malloc. + * If no memory is available, it aborts. + */ +void* malloc_nofail_align(size_t size, void** freeptr); + /** * Xor two memory blocks. */ @@ -133,9 +139,15 @@ void memxor(unsigned char* xor, const unsigned char* block, unsigned size); #define HASH_SIZE 16 /** - * Computes the hash of a memory block. + * Hash kinds. */ -void memhash(void* digest, const void* src, unsigned size); +#define HASH_MURMUR3 0 +#define HASH_MD5 1 + +/** + * Computes the HASH of a memory block. + */ +void memhash(unsigned kind, void* digest, const void* src, unsigned size); #endif