diff --git a/check.c b/check.c index 6a4f36a2..1047314f 100644 --- a/check.c +++ b/check.c @@ -93,7 +93,7 @@ static int blockcmp(struct snapraid_state* state, int rehash, struct snapraid_bl /** * Checks if the hash of at least one failed block is now matching. */ -static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char* buffer_zero) +static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void* buffer_zero) { unsigned j; int hash_checked; @@ -122,7 +122,7 @@ static int is_hash_matching(struct snapraid_state* state, int rehash, unsigned d /** * Checks if specified parity is now matching with a recomputed one. */ -static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, unsigned i, unsigned char** buffer, unsigned char** buffer_recov) +static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, unsigned i, void** buffer, void** buffer_recov) { /* recompute parity, note that we don't need parity over i */ raid_par(i + 1, diskmax, state->block_size, buffer); @@ -142,7 +142,7 @@ static int is_parity_matching(struct snapraid_state* state, unsigned diskmax, un * Returns <0 if failure for missing strategy, >0 if data is wrong and we cannot rebuild correctly, 0 on success. * If success, the parity are computed in the buffer variable. */ -static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char** buffer_recov, unsigned char* buffer_zero) +static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero) { unsigned i, n; int error; @@ -199,7 +199,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+ip[i]], buffer_recov[ip[i]], state->block_size); /* recover using one less parity, the ip[r-1] one */ - raid_rec(r-1, id, ip, diskmax, state->block_size, buffer, buffer_zero); + raid_rec(r-1, id, ip, diskmax, state->block_size, buffer); /* use the remaining ip[r-1] parity to check the result */ if (is_parity_matching(state, diskmax, ip[r-1], buffer, buffer_recov)) @@ -239,7 +239,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+ip[i]], buffer_recov[ip[i]], state->block_size); /* recover */ - raid_rec(r, id, ip, diskmax, state->block_size, buffer, buffer_zero); + raid_rec(r, id, ip, diskmax, state->block_size, buffer); /* use the hash to check the result */ if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero)) @@ -264,7 +264,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u return -1; } -static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, unsigned char** buffer, unsigned char** buffer_recov, unsigned char* buffer_zero) +static int repair(struct snapraid_state* state, int rehash, unsigned pos, unsigned diskmax, struct failed_struct* failed, unsigned* failed_map, unsigned failed_count, void** buffer, void** buffer_recov, void* buffer_zero) { int ret; int error; @@ -468,7 +468,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, block_off_t i; unsigned j; void* buffer_alloc; - unsigned char** buffer; + void** buffer; unsigned buffermax; int ret; data_off_t countsize; @@ -492,6 +492,7 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* fill up the zero buffer */ memset(buffer[buffermax-1], 0, state->block_size); + raid_zero(buffer[buffermax-1]); failed = malloc_nofail(diskmax * sizeof(struct failed_struct)); failed_map = malloc_nofail(diskmax * sizeof(unsigned)); @@ -768,8 +769,8 @@ static int state_check_process(struct snapraid_state* state, int check, int fix, /* now read and check the parity if requested */ if (check) { - unsigned char* buffer_recov[LEV_MAX]; - unsigned char* buffer_zero; + void* buffer_recov[LEV_MAX]; + void* buffer_zero; /* buffers for parity read and not computed */ for(l=0;llevel;++l) diff --git a/cpu.h b/cpu.h index b529b742..4d8c2079 100644 --- a/cpu.h +++ b/cpu.h @@ -15,12 +15,12 @@ * along with this program. If not, see . */ -#ifndef __CPU_H -#define __CPU_H +#ifndef __RAID_CPU_H +#define __RAID_CPU_H #if defined(__i386__) || defined(__x86_64__) -static inline void cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t* reg) +static __always_inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t* reg) { asm volatile( #if defined(__i386__) && defined(__PIC__) @@ -40,19 +40,19 @@ static inline void cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t* reg) #define CPU_VENDOR_MAX 13 -static inline void cpu_info(char* vendor, unsigned* family, unsigned* model) +static __always_inline void raid_cpu_info(char* vendor, unsigned* family, unsigned* model) { uint32_t reg[4]; unsigned f, ef, m, em; - cpuid(0, 0, reg); + raid_cpuid(0, 0, reg); ((uint32_t*)vendor)[0] = reg[1]; ((uint32_t*)vendor)[1] = reg[3]; ((uint32_t*)vendor)[2] = reg[2]; vendor[12] = 0; - cpuid(1, 0, reg); + raid_cpuid(1, 0, reg); f = (reg[0] >> 8) & 0xF; ef = (reg[0] >> 20) & 0xFF; @@ -73,56 +73,56 @@ static inline void cpu_info(char* vendor, unsigned* family, unsigned* model) } } -static inline int cpu_has_mmx(void) +static __always_inline int raid_cpu_has_mmx(void) { uint32_t reg[4]; - cpuid(1, 0, reg); + raid_cpuid(1, 0, reg); return (reg[3] >> 23) & 1; } -static inline int cpu_has_sse2(void) +static __always_inline int raid_cpu_has_sse2(void) { uint32_t reg[4]; - cpuid(1, 0, reg); + raid_cpuid(1, 0, reg); return (reg[3] >> 26) & 1; } -static inline int cpu_has_ssse3(void) +static __always_inline int raid_cpu_has_ssse3(void) { uint32_t reg[4]; - cpuid(1, 0, reg); + raid_cpuid(1, 0, reg); return (reg[2] >> 9) & 1; } -static inline int cpu_has_sse42(void) +static __always_inline int raid_cpu_has_sse42(void) { uint32_t reg[4]; - cpuid(1, 0, reg); + raid_cpuid(1, 0, reg); return (reg[2] >> 20) & 1; } -static inline int cpu_has_avx(void) +static __always_inline int raid_cpu_has_avx(void) { uint32_t reg[4]; - cpuid(1, 0, reg); + raid_cpuid(1, 0, reg); return (reg[2] >> 28) & 1; } -static inline int cpu_has_avx2(void) +static __always_inline int raid_cpu_has_avx2(void) { uint32_t reg[4]; - cpuid(7, 0, reg); + raid_cpuid(7, 0, reg); return (reg[1] >> 5) & 1; } @@ -131,13 +131,13 @@ static inline int cpu_has_avx2(void) * Check if the processor has a slow MULT implementation. * If yes, it's better to use a hash not based on multiplication. */ -static inline int cpu_has_slowmult(void) +static __always_inline int raid_cpu_has_slowmult(void) { char vendor[CPU_VENDOR_MAX]; unsigned family; unsigned model; - cpu_info(vendor, &family, &model); + raid_cpu_info(vendor, &family, &model); if (strcmp(vendor, "GenuineIntel") == 0) { /* Intel(R) Atom(TM) CPU D525 @ 1.80GHz (info from user) @@ -169,13 +169,13 @@ static inline int cpu_has_slowmult(void) * Check if the processor has a slow PSHUFB implementation. * If yes, it's better to not use it. */ -static inline int cpu_has_slowpshufb(void) +static __always_inline int raid_cpu_has_slowpshufb(void) { char vendor[CPU_VENDOR_MAX]; unsigned family; unsigned model; - cpu_info(vendor, &family, &model); + raid_cpu_info(vendor, &family, &model); if (strcmp(vendor, "GenuineIntel") == 0) { /* Intel ATOM (info from x264 -> full family/model 6/28) */ @@ -233,13 +233,13 @@ static inline int cpu_has_slowpshufb(void) * Check if the processor has a slow extended set of SSE registers. * If yes, it's better to unroll without using the second part of registers. */ -static inline int cpu_has_slowextendedreg(void) +static __always_inline int raid_cpu_has_slowextendedreg(void) { char vendor[CPU_VENDOR_MAX]; unsigned family; unsigned model; - cpu_info(vendor, &family, &model); + raid_cpu_info(vendor, &family, &model); if (strcmp(vendor, "AuthenticAMD") == 0) { /* AMD Bulldozer (from user) diff --git a/mktables.c b/mktables.c index e6683cd5..2f726d5d 100644 --- a/mktables.c +++ b/mktables.c @@ -16,13 +16,14 @@ */ #include +#include /** * Multiplication in GF(2^8). */ -unsigned char gfmul(unsigned char a, unsigned char b) +static uint8_t gfmul(uint8_t a, uint8_t b) { - unsigned char v; + uint8_t v; v = 0; while (b) { @@ -45,7 +46,7 @@ unsigned char gfmul(unsigned char a, unsigned char b) /** * Inversion table in GF(2^8). */ -unsigned char gfinv[256]; +uint8_t gfinv[256]; /** * Number of parity. @@ -62,10 +63,10 @@ unsigned char gfinv[256]; /** * Setup the Cauchy matrix used to generate the parity. */ -void set_cauchy(unsigned char* matrix) +static void set_cauchy(uint8_t* matrix) { int i, j; - unsigned char inv_x, y; + uint8_t inv_x, y; /* * First row is formed by all 1. @@ -110,7 +111,7 @@ void set_cauchy(unsigned char* matrix) for(j=0;j> 1; @@ -178,9 +179,9 @@ unsigned np(unsigned v) int main(void) { - unsigned char v; + uint8_t v; int i, j, k, p; - unsigned char matrix[PARITY * 256]; + uint8_t matrix[PARITY * 256]; printf("/*\n"); printf(" * Copyright (C) 2013 Andrea Mazzoleni\n"); @@ -201,7 +202,7 @@ int main(void) printf("\n"); /* a*b */ - printf("const unsigned char __attribute__((aligned(256))) gfmul[256][256] =\n"); + printf("const uint8_t __attribute__((aligned(256))) gfmul[256][256] =\n"); printf("{\n"); for(i=0;i<256;++i) { printf("\t{\n"); @@ -222,7 +223,7 @@ int main(void) printf("};\n\n"); /* 2^a */ - printf("const unsigned char __attribute__((aligned(256))) gfexp2[256] =\n"); + printf("const uint8_t __attribute__((aligned(256))) gfexp[256] =\n"); printf("{\n"); v = 1; for(i=0;i<256;++i) { @@ -238,7 +239,7 @@ int main(void) printf("};\n\n"); /* 1/a */ - printf("const unsigned char __attribute__((aligned(256))) gfinv[256] =\n"); + printf("const uint8_t __attribute__((aligned(256))) gfinv[256] =\n"); printf("{\n"); printf("\t/* note that the first element is not significative */\n"); for(i=0;i<256;++i) { @@ -271,7 +272,7 @@ int main(void) printf("\n"); } printf(" */\n"); - printf("const unsigned char __attribute__((aligned(256))) gfvandermonde[%u][256] =\n", 3); + printf("const uint8_t __attribute__((aligned(256))) gfvandermonde[%u][256] =\n", 3); printf("{\n"); for(p=0;p<3;++p) { printf("\t{\n"); @@ -303,7 +304,7 @@ int main(void) printf("\n"); } printf(" */\n"); - printf("const unsigned char __attribute__((aligned(256))) gfcauchy[%u][256] =\n", PARITY); + printf("const uint8_t __attribute__((aligned(256))) gfcauchy[%u][256] =\n", PARITY); printf("{\n"); for(p=0;p=0;--d) { - p0 ^= v_32(vbuf[d][i]); - p1 ^= v_32(vbuf[d][i+4]); + p0 ^= v_32(v[d][i]); + p1 ^= v_32(v[d][i+4]); } v_32(p[i]) = p0; v_32(p[i+4]) = p1; @@ -246,26 +252,27 @@ void raid_par1_int32(unsigned nd, unsigned size, unsigned char** vbuf) /* * PAR1 (RAID5 with xor) 64bit C implementation */ -void raid_par1_int64(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par1_int64(int nd, size_t size, void** vv) { - unsigned char* p; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; int d, l; - unsigned i; + size_t i; uint64_t p0; uint64_t p1; l = nd - 1; - p = vbuf[nd]; + p = v[nd]; for(i=0;i=0;--d) { - p0 ^= v_64(vbuf[d][i]); - p1 ^= v_64(vbuf[d][i+8]); + p0 ^= v_64(v[d][i]); + p1 ^= v_64(v[d][i+8]); } v_64(p[i]) = p0; v_64(p[i+8]) = p1; @@ -281,27 +288,28 @@ void raid_par1_int64(unsigned nd, unsigned size, unsigned char** vbuf) * the typical cache block, and processing 128 bytes doesn't increase * performance. */ -void raid_par1_sse2(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par1_sse2(int nd, size_t size, void** vv) { - unsigned char* p; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; + p = v[nd]; for(i=0;i=0;--d) { - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); - asm volatile("movdqa %0,%%xmm5" : : "m" (vbuf[d][i+16])); - asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][i+32])); - asm volatile("movdqa %0,%%xmm7" : : "m" (vbuf[d][i+48])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile("movdqa %0,%%xmm5" : : "m" (v[d][i+16])); + asm volatile("movdqa %0,%%xmm6" : : "m" (v[d][i+32])); + asm volatile("movdqa %0,%%xmm7" : : "m" (v[d][i+48])); asm volatile("pxor %xmm4,%xmm0"); asm volatile("pxor %xmm5,%xmm1"); asm volatile("pxor %xmm6,%xmm2"); @@ -320,8 +328,7 @@ void raid_par1_sse2(unsigned nd, unsigned size, unsigned char** vbuf) /** * Multiply each byte of a uint32 by 2 in the GF(2^8). */ -__attribute__((always_inline)) -static inline uint32_t x2_32(uint32_t v) +static __always_inline uint32_t x2_32(uint32_t v) { uint32_t mask = v & 0x80808080U; mask = (mask << 1) - (mask >> 7); @@ -333,8 +340,7 @@ static inline uint32_t x2_32(uint32_t v) /** * Multiply each byte of a uint64 by 2 in the GF(2^8). */ -__attribute__((always_inline)) -static inline uint64_t x2_64(uint64_t v) +static __always_inline uint64_t x2_64(uint64_t v) { uint64_t mask = v & 0x8080808080808080ULL; mask = (mask << 1) - (mask >> 7); @@ -346,8 +352,7 @@ static inline uint64_t x2_64(uint64_t v) /** * Divide each byte of a uint32 by 2 in the GF(2^8). */ -__attribute__((always_inline)) -static inline uint32_t d2_32(uint32_t v) +static __always_inline uint32_t d2_32(uint32_t v) { uint32_t mask = v & 0x01010101U; mask = (mask << 8) - mask; @@ -359,8 +364,7 @@ static inline uint32_t d2_32(uint32_t v) /** * Divide each byte of a uint64 by 2 in the GF(2^8). */ -__attribute__((always_inline)) -static inline uint64_t d2_64(uint64_t v) +static __always_inline uint64_t d2_64(uint64_t v) { uint64_t mask = v & 0x0101010101010101ULL; mask = (mask << 8) - mask; @@ -372,26 +376,27 @@ static inline uint64_t d2_64(uint64_t v) /* * PAR2 (RAID6 with powers of 2) 32bit C implementation */ -void raid_par2_int32(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par2_int32(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; int d, l; - unsigned i; + size_t i; uint32_t d0, q0, p0; uint32_t d1, q1, p1; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; + p = v[nd]; + q = v[nd+1]; for(i=0;i=0;--d) { - d0 = v_32(vbuf[d][i]); - d1 = v_32(vbuf[d][i+4]); + d0 = v_32(v[d][i]); + d1 = v_32(v[d][i+4]); p0 ^= d0; p1 ^= d1; @@ -412,26 +417,27 @@ void raid_par2_int32(unsigned nd, unsigned size, unsigned char** vbuf) /* * PAR2 (RAID6 with powers of 2) 64bit C implementation */ -void raid_par2_int64(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par2_int64(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; int d, l; - unsigned i; + size_t i; uint64_t d0, q0, p0; uint64_t d1, q1, p1; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; + p = v[nd]; + q = v[nd+1]; for(i=0;i=0;--d) { - d0 = v_64(vbuf[d][i]); - d1 = v_64(vbuf[d][i+8]); + d0 = v_64(v[d][i]); + d1 = v_64(v[d][i+8]); p0 ^= d0; p1 ^= d1; @@ -451,10 +457,10 @@ void raid_par2_int64(unsigned nd, unsigned size, unsigned char** vbuf) #if defined(__i386__) || defined(__x86_64__) static const struct gfconst16 { - unsigned char poly[16]; - unsigned char low4[16]; - unsigned char half[16]; - unsigned char low7[16]; + uint8_t poly[16]; + uint8_t low4[16]; + uint8_t half[16]; + uint8_t low7[16]; } gfconst16 __attribute__((aligned(64))) = { { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f }, @@ -465,22 +471,23 @@ static const struct gfconst16 { /* * PAR2 (RAID6 with powers of 2) SSE2 implementation */ -void raid_par2_sse2(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par2_sse2(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; + p = v[nd]; + q = v[nd+1]; asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); for(i=0;i=0;--d) { @@ -495,8 +502,8 @@ void raid_par2_sse2(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pxor %xmm4,%xmm2"); asm volatile("pxor %xmm5,%xmm3"); - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); - asm volatile("movdqa %0,%%xmm5" : : "m" (vbuf[d][i+16])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile("movdqa %0,%%xmm5" : : "m" (v[d][i+16])); asm volatile("pxor %xmm4,%xmm0"); asm volatile("pxor %xmm5,%xmm1"); asm volatile("pxor %xmm4,%xmm2"); @@ -518,24 +525,25 @@ void raid_par2_sse2(unsigned nd, unsigned size, unsigned char** vbuf) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par2_sse2ext(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par2_sse2ext(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; + p = v[nd]; + q = v[nd+1]; asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); for(i=0;i=0;--d) { - d0 = v_32(vbuf[d][i]); - d1 = v_32(vbuf[d][i+4]); + d0 = v_32(v[d][i]); + d1 = v_32(v[d][i+4]); p0 ^= d0; p1 ^= d1; @@ -642,28 +651,29 @@ void raid_parz_int32(unsigned nd, unsigned size, unsigned char** vbuf) /* * PARz (triple parity with powers of 2^-1) 64bit C implementation */ -void raid_parz_int64(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_parz_int64(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; int d, l; - unsigned i; + size_t i; uint64_t d0, r0, q0, p0; uint64_t d1, r1, q1, p1; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; for(i=0;i=0;--d) { - d0 = v_64(vbuf[d][i]); - d1 = v_64(vbuf[d][i+8]); + d0 = v_64(v[d][i]); + d1 = v_64(v[d][i+8]); p0 ^= d0; p1 ^= d1; @@ -693,25 +703,26 @@ void raid_parz_int64(unsigned nd, unsigned size, unsigned char** vbuf) /* * PARz (triple parity with powers of 2^-1) SSE2 implementation */ -void raid_parz_sse2(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_parz_sse2(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); asm volatile("movdqa %0,%%xmm6" : : "m" (gfconst16.low7[0])); for(i=0;i=0;--d) { @@ -730,7 +741,7 @@ void raid_parz_sse2(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm3,%xmm5"); asm volatile("pxor %xmm5,%xmm2"); - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("pxor %xmm4,%xmm0"); asm volatile("pxor %xmm4,%xmm1"); asm volatile("pxor %xmm4,%xmm2"); @@ -750,26 +761,27 @@ void raid_parz_sse2(unsigned nd, unsigned size, unsigned char** vbuf) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_parz_sse2ext(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_parz_sse2ext(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); asm volatile("movdqa %0,%%xmm11" : : "m" (gfconst16.low7[0])); for(i=0;i0;--d) { - d0 = v_8(vbuf[d][i]); + d0 = v_8(v[d][i]); p0 ^= d0; q0 ^= gfmul[d0][gfcauchy[1][d]]; @@ -857,7 +870,7 @@ void raid_par3_int8(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - d0 = v_8(vbuf[0][i]); + d0 = v_8(v[0][i]); p0 ^= d0; q0 ^= d0; @@ -873,23 +886,24 @@ void raid_par3_int8(unsigned nd, unsigned size, unsigned char** vbuf) /* * PAR3 (triple parity with Cauchy matrix) SSSE3 implementation */ -void raid_par3_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par3_ssse3(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<3;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } @@ -899,7 +913,7 @@ void raid_par3_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) for(i=0;i0;--d) { - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm1,%xmm5"); @@ -933,16 +947,16 @@ void raid_par3_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm7,%xmm4"); asm volatile("pand %xmm7,%xmm5"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][0][0][0])); asm volatile("pshufb %xmm4,%xmm6"); asm volatile("pxor %xmm6,%xmm2"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("pshufb %xmm5,%xmm6"); asm volatile("pxor %xmm6,%xmm2"); } /* first disk with all coefficients at 1 */ - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[0][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm1,%xmm5"); @@ -969,23 +983,24 @@ void raid_par3_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par3_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par3_ssse3ext(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<3;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } @@ -995,8 +1010,8 @@ void raid_par3_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) for(i=0;i0;--d) { - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); - asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][i+16])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile("movdqa %0,%%xmm12" : : "m" (v[d][i+16])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm13,%xmm13"); @@ -1053,8 +1068,8 @@ void raid_par3_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm11,%xmm5"); asm volatile("pand %xmm11,%xmm13"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("movdqa %xmm6,%xmm14"); asm volatile("movdqa %xmm7,%xmm15"); asm volatile("pshufb %xmm4,%xmm6"); @@ -1068,8 +1083,8 @@ void raid_par3_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[0][i])); - asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[0][i+16])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile("movdqa %0,%%xmm12" : : "m" (v[0][i+16])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm13,%xmm13"); @@ -1109,27 +1124,28 @@ void raid_par3_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par4_int8(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par4_int8(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; int d, l; - unsigned i; + size_t i; uint8_t d0, s0, r0, q0, p0; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; for(i=0;i0;--d) { - d0 = v_8(vbuf[d][i]); + d0 = v_8(v[d][i]); p0 ^= d0; q0 ^= gfmul[d0][gfcauchy[1][d]]; @@ -1138,7 +1154,7 @@ void raid_par4_int8(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - d0 = v_8(vbuf[0][i]); + d0 = v_8(v[0][i]); p0 ^= d0; q0 ^= d0; @@ -1156,25 +1172,26 @@ void raid_par4_int8(unsigned nd, unsigned size, unsigned char** vbuf) /* * PAR4 (quad parity with Cauchy matrix) SSSE3 implementation */ -void raid_par4_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par4_ssse3(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<4;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } @@ -1182,7 +1199,7 @@ void raid_par4_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) for(i=0;i0;--d) { asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm1,%xmm5"); @@ -1225,15 +1242,15 @@ void raid_par4_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm7,%xmm4"); asm volatile("pand %xmm7,%xmm5"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("pshufb %xmm4,%xmm6"); asm volatile("pshufb %xmm5,%xmm7"); asm volatile("pxor %xmm6,%xmm2"); asm volatile("pxor %xmm7,%xmm2"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][1][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][1][1][0])); asm volatile("pshufb %xmm4,%xmm6"); asm volatile("pshufb %xmm5,%xmm7"); asm volatile("pxor %xmm6,%xmm3"); @@ -1242,7 +1259,7 @@ void raid_par4_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) /* first disk with all coefficients at 1 */ asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[0][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm1,%xmm5"); @@ -1271,25 +1288,26 @@ void raid_par4_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par4_ssse3ext(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<4;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } @@ -1297,8 +1315,8 @@ void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) for(i=0;i0;--d) { asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); - asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][i+16])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile("movdqa %0,%%xmm12" : : "m" (v[d][i+16])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm13,%xmm13"); @@ -1368,8 +1386,8 @@ void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm15,%xmm5"); asm volatile("pand %xmm15,%xmm13"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("movdqa %xmm6,%xmm14"); asm volatile("movdqa %xmm7,%xmm15"); asm volatile("pshufb %xmm4,%xmm6"); @@ -1381,8 +1399,8 @@ void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pxor %xmm7,%xmm2"); asm volatile("pxor %xmm15,%xmm10"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][1][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][1][1][0])); asm volatile("movdqa %xmm6,%xmm14"); asm volatile("movdqa %xmm7,%xmm15"); asm volatile("pshufb %xmm4,%xmm6"); @@ -1398,8 +1416,8 @@ void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) /* first disk with all coefficients at 1 */ asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[0][i])); - asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[0][i+16])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[0][i])); + asm volatile("movdqa %0,%%xmm12" : : "m" (v[0][i+16])); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm13,%xmm13"); @@ -1443,29 +1461,30 @@ void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par5_int8(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par5_int8(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; - unsigned char* t; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; int d, l; - unsigned i; + size_t i; uint8_t d0, t0, s0, r0, q0, p0; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; - t = vbuf[nd+4]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; for(i=0;i0;--d) { - d0 = v_8(vbuf[d][i]); + d0 = v_8(v[d][i]); p0 ^= d0; q0 ^= gfmul[d0][gfcauchy[1][d]]; @@ -1475,7 +1494,7 @@ void raid_par5_int8(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - d0 = v_8(vbuf[0][i]); + d0 = v_8(v[0][i]); p0 ^= d0; q0 ^= d0; @@ -1499,36 +1518,37 @@ void raid_par5_int8(unsigned nd, unsigned size, unsigned char** vbuf) /* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ __attribute__((force_align_arg_pointer)) #endif -void raid_par5_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par5_ssse3(int nd, size_t size, void** vv) /* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; - unsigned char* t; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; int d, l; - unsigned i; - unsigned char p0[16] __attribute__((aligned(16))); + size_t i; + uint8_t p0[16] __attribute__((aligned(16))); l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; - t = vbuf[nd+4]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<5;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } /* generic case with at least two data disks */ for(i=0;i0;--d) { - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("movdqa %0,%%xmm6" : : "m" (p0[0])); asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); @@ -1579,22 +1599,22 @@ void raid_par5_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm7,%xmm4"); asm volatile("pand %xmm7,%xmm5"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("pshufb %xmm4,%xmm6"); asm volatile("pshufb %xmm5,%xmm7"); asm volatile("pxor %xmm6,%xmm1"); asm volatile("pxor %xmm7,%xmm1"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][1][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][1][1][0])); asm volatile("pshufb %xmm4,%xmm6"); asm volatile("pshufb %xmm5,%xmm7"); asm volatile("pxor %xmm6,%xmm2"); asm volatile("pxor %xmm7,%xmm2"); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile("movdqa %0,%%xmm7" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfcauchypshufb[d][2][0][0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfcauchypshufb[d][2][1][0])); asm volatile("pshufb %xmm4,%xmm6"); asm volatile("pshufb %xmm5,%xmm7"); asm volatile("pxor %xmm6,%xmm3"); @@ -1602,7 +1622,7 @@ void raid_par5_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[0][i])); + asm volatile("movdqa %0,%%xmm4" : : "m" (v[0][i])); asm volatile("movdqa %0,%%xmm6" : : "m" (p0[0])); asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); @@ -1635,27 +1655,28 @@ void raid_par5_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par5_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par5_ssse3ext(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; - unsigned char* t; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; int d, l; - unsigned i; + size_t i; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; - t = vbuf[nd+4]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<5;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } @@ -1665,7 +1686,7 @@ void raid_par5_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) for(i=0;i0;--d) { - asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[d][i])); + asm volatile("movdqa %0,%%xmm10" : : "m" (v[d][i])); asm volatile("pxor %xmm11,%xmm11"); asm volatile("pcmpgtb %xmm1,%xmm11"); @@ -1711,22 +1732,22 @@ void raid_par5_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm15,%xmm10"); asm volatile("pand %xmm15,%xmm11"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm2"); asm volatile("pxor %xmm13,%xmm2"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][1][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][1][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm3"); asm volatile("pxor %xmm13,%xmm3"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][2][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][2][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm4"); @@ -1734,7 +1755,7 @@ void raid_par5_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[0][i])); + asm volatile("movdqa %0,%%xmm10" : : "m" (v[0][i])); asm volatile("pxor %xmm11,%xmm11"); asm volatile("pcmpgtb %xmm1,%xmm11"); @@ -1767,31 +1788,32 @@ void raid_par5_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par6_int8(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par6_int8(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; - unsigned char* t; - unsigned char* u; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; + uint8_t* u; int d, l; - unsigned i; + size_t i; uint8_t d0, u0, t0, s0, r0, q0, p0; l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; - t = vbuf[nd+4]; - u = vbuf[nd+5]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; + u = v[nd+5]; for(i=0;i0;--d) { - d0 = v_8(vbuf[d][i]); + d0 = v_8(v[d][i]); p0 ^= d0; q0 ^= gfmul[d0][gfcauchy[1][d]]; @@ -1802,7 +1824,7 @@ void raid_par6_int8(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - d0 = v_8(vbuf[0][i]); + d0 = v_8(v[0][i]); p0 ^= d0; q0 ^= d0; @@ -1810,7 +1832,7 @@ void raid_par6_int8(unsigned nd, unsigned size, unsigned char** vbuf) s0 ^= d0; t0 ^= d0; u0 ^= d0; - + v_8(p[i]) = p0; v_8(q[i]) = q0; v_8(r[i]) = r0; @@ -1828,38 +1850,39 @@ void raid_par6_int8(unsigned nd, unsigned size, unsigned char** vbuf) /* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ __attribute__((force_align_arg_pointer)) #endif -void raid_par6_ssse3(unsigned nd, unsigned size, unsigned char** vbuf) +void raid_par6_ssse3(int nd, size_t size, void** vv) { - unsigned char* p; - unsigned char* q; - unsigned char* r; - unsigned char* s; - unsigned char* t; - unsigned char* u; + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; + uint8_t* u; int d, l; - unsigned i; - unsigned char p0[16] __attribute__((aligned(16))); - unsigned char q0[16] __attribute__((aligned(16))); + size_t i; + uint8_t p0[16] __attribute__((aligned(16))); + uint8_t q0[16] __attribute__((aligned(16))); l = nd - 1; - p = vbuf[nd]; - q = vbuf[nd+1]; - r = vbuf[nd+2]; - s = vbuf[nd+3]; - t = vbuf[nd+4]; - u = vbuf[nd+5]; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; + u = v[nd+5]; /* special case with only one data disk */ if (l == 0) { for(i=0;i<6;++i) - memcpy(vbuf[1+i], vbuf[0], size); + memcpy(v[1+i], v[0], size); return; } /* generic case with at least two data disks */ for(i=0;i0;--d) { - asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[d][i])); + asm volatile("movdqa %0,%%xmm10" : : "m" (v[d][i])); asm volatile("pxor %xmm11,%xmm11"); asm volatile("pcmpgtb %xmm1,%xmm11"); @@ -2069,29 +2093,29 @@ void raid_par6_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) asm volatile("pand %xmm15,%xmm10"); asm volatile("pand %xmm15,%xmm11"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][0][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][0][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][0][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][0][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm2"); asm volatile("pxor %xmm13,%xmm2"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][1][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][1][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][1][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][1][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm3"); asm volatile("pxor %xmm13,%xmm3"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][2][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][2][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][2][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][2][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm4"); asm volatile("pxor %xmm13,%xmm4"); - asm volatile("movdqa %0,%%xmm12" : : "m" (gfgenpshufb[d][3][0][0])); - asm volatile("movdqa %0,%%xmm13" : : "m" (gfgenpshufb[d][3][1][0])); + asm volatile("movdqa %0,%%xmm12" : : "m" (gfcauchypshufb[d][3][0][0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (gfcauchypshufb[d][3][1][0])); asm volatile("pshufb %xmm10,%xmm12"); asm volatile("pshufb %xmm11,%xmm13"); asm volatile("pxor %xmm12,%xmm5"); @@ -2099,7 +2123,7 @@ void raid_par6_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) } /* first disk with all coefficients at 1 */ - asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[0][i])); + asm volatile("movdqa %0,%%xmm10" : : "m" (v[0][i])); asm volatile("pxor %xmm11,%xmm11"); asm volatile("pcmpgtb %xmm1,%xmm11"); @@ -2126,56 +2150,57 @@ void raid_par6_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf) } #endif -/****************************************************************************/ -/* mode */ +/* internal forwarder */ +static void (*raid_par_ptr[RAID_PARITY_MAX])(int nd, size_t size, void** vv); +static void (*raid_par3_ptr)(int nd, size_t size, void** vv); +static void (*raid_parz_ptr)(int nd, size_t size, void** vv); -static unsigned raid_mode = RAID_MODE_CAUCHY; /**< RAID mode. */ - -void raid_set(unsigned mode) +void raid_par(int np, int nd, size_t size, void** v) { - raid_mode = mode; + bug_on(np < 1 || np > RAID_PARITY_MAX); + + raid_par_ptr[np - 1](nd, size, v); } /****************************************************************************/ -/* parity generation */ +/* mode */ -/* internal forwarder */ -static void (*raid_par1_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); -static void (*raid_par2_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); -static void (*raid_parz_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); -static void (*raid_par3_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); -static void (*raid_par4_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); -static void (*raid_par5_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); -static void (*raid_par6_ptr)(unsigned nd, unsigned size, unsigned char** vbuf); +/** + * Generator matrix currently used. + */ +static const uint8_t (*raid_gen)[256]; -void raid_par(unsigned np, unsigned nd, unsigned size, unsigned char** vbuf) +void raid_mode(int mode) { - switch (np) { - case 1 : raid_par1_ptr(nd, size, vbuf); break; - case 2 : raid_par2_ptr(nd, size, vbuf); break; - case 3 : - if (raid_mode == RAID_MODE_CAUCHY) - raid_par3_ptr(nd, size, vbuf); - else - raid_parz_ptr(nd, size, vbuf); - break; - case 4 : raid_par4_ptr(nd, size, vbuf); break; - case 5 : raid_par5_ptr(nd, size, vbuf); break; - case 6 : raid_par6_ptr(nd, size, vbuf); break; - default: - fprintf(stderr, "Invalid raid gen level\n"); - exit(EXIT_FAILURE); + if (mode == RAID_MODE_VANDERMONDE) { + raid_par_ptr[2] = raid_parz_ptr; + raid_gen = gfvandermonde; + } else { + raid_par_ptr[2] = raid_par3_ptr; + raid_gen = gfcauchy; } } +/****************************************************************************/ +/* zero */ + +/** + * Buffer filled with 0 used in recovering. + */ +static void* raid_empty_zero_page; + +void raid_zero(void* zero) +{ + raid_empty_zero_page = zero; +} + /****************************************************************************/ /* recovering */ /** * GF a*b. */ -__attribute__((always_inline)) -static inline unsigned char mul(unsigned char a, unsigned char b) +static __always_inline uint8_t mul(uint8_t a, uint8_t b) { return gfmul[a][b]; } @@ -2184,13 +2209,9 @@ static inline unsigned char mul(unsigned char a, unsigned char b) * GF 1/a. * Not defined for a == 0. */ -__attribute__((always_inline)) -static inline unsigned char inv(unsigned char v) +static __always_inline uint8_t inv(uint8_t v) { - if (v == 0) { - fprintf(stderr, "GF division by zero\n"); - exit(EXIT_FAILURE); - } + bug_on(v == 0); /* division by zero */ return gfinv[v]; } @@ -2198,22 +2219,17 @@ static inline unsigned char inv(unsigned char v) /** * GF 2^a. */ -__attribute__((always_inline)) -static inline unsigned char pow2(int v) +static __always_inline uint8_t pow2(int v) { - if (v < 0 || v > 254) { - fprintf(stderr, "GF invalid exponent\n"); - exit(EXIT_FAILURE); - } + bug_on(v < 0 || v > 254); /* invalid exponent */ - return gfexp2[v]; + return gfexp[v]; } /** * Gets the multiplication table for a specified value. */ -__attribute__((always_inline)) -static inline const unsigned char* table(unsigned char v) +static __always_inline const uint8_t* table(uint8_t v) { return gfmul[v]; } @@ -2228,43 +2244,38 @@ static inline const unsigned char* table(unsigned char v) * Note that all the other parities not in the ip[] vector * are destroyed. */ -static inline void raid_delta_gen(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero) +static void raid_delta_gen(int nr, const int* id, const int* ip, int nd, size_t size, void** v) { - unsigned char* p[RAID_PARITY_CAUCHY_MAX]; - unsigned char* pa[RAID_PARITY_CAUCHY_MAX]; - unsigned i; - - assert(nr > 0 && nr <= RAID_PARITY_CAUCHY_MAX); + void* p[RAID_PARITY_MAX]; + void* pa[RAID_PARITY_MAX]; + int i; for(i=0;i nd); + + /* if failed data is present */ + if (nr != 0) { + raid_rec_ptr[nr - 1](nr, id, ip, nd, size, v); + } +} + +void raid_recpar(int nrd, const int* id, int nrp, int* ip, int np, int nd, size_t size, void** v) +{ + bug_on(nrd > nd); + bug_on(nrd + nrp > np); + + /* if failed data is present */ + if (nrd != 0) { + int iu[RAID_PARITY_MAX]; + int i, j, k; + + /* setup the vector of parities to use */ + for(i=0,j=0,k=0;i<(int)np;++i) { + if (ip[j] == i) { + ++j; + } else { + iu[k] = i; + ++k; + } + } + + /* recover the data */ + raid_rec_ptr[nrd - 1](nrd, id, iu, nd, size, v); + } + + /* recompute all the parities up to the last bad one */ + if (nrp != 0) { + raid_par(ip[nrp - 1] + 1, nd, size, v); } } @@ -2882,58 +2902,67 @@ void raid_rec(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned s void raid_init(void) { raid_par3_ptr = raid_par3_int8; - raid_par4_ptr = raid_par4_int8; - raid_par5_ptr = raid_par5_int8; - raid_par6_ptr = raid_par6_int8; + raid_par_ptr[3] = raid_par4_int8; + raid_par_ptr[4] = raid_par5_int8; + raid_par_ptr[5] = raid_par6_int8; if (sizeof(void*) == 4) { - raid_par1_ptr = raid_par1_int32; - raid_par2_ptr = raid_par2_int32; + raid_par_ptr[0] = raid_par1_int32; + raid_par_ptr[1] = raid_par2_int32; raid_parz_ptr = raid_parz_int32; } else { - raid_par1_ptr = raid_par1_int64; - raid_par2_ptr = raid_par2_int64; + raid_par_ptr[0] = raid_par1_int64; + raid_par_ptr[1] = raid_par2_int64; raid_parz_ptr = raid_parz_int64; } - raid_rec1_ptr = raid_rec1_int8; - raid_rec2_ptr = raid_rec2_int8; - raid_recX_ptr = raid_recX_int8; + raid_rec_ptr[0] = raid_rec1_int8; + raid_rec_ptr[1] = raid_rec2_int8; + raid_rec_ptr[2] = raid_recX_int8; + raid_rec_ptr[3] = raid_recX_int8; + raid_rec_ptr[4] = raid_recX_int8; + raid_rec_ptr[5] = raid_recX_int8; #if defined(__i386__) || defined(__x86_64__) - if (cpu_has_sse2()) { - raid_par1_ptr = raid_par1_sse2; + if (raid_cpu_has_sse2()) { + raid_par_ptr[0] = raid_par1_sse2; #if defined(__x86_64__) - if (cpu_has_slowextendedreg()) { - raid_par2_ptr = raid_par2_sse2; + if (raid_cpu_has_slowextendedreg()) { + raid_par_ptr[1] = raid_par2_sse2; raid_parz_ptr = raid_parz_sse2; } else { - raid_par2_ptr = raid_par2_sse2ext; + raid_par_ptr[1] = raid_par2_sse2ext; raid_parz_ptr = raid_parz_sse2ext; } #else - raid_par2_ptr = raid_par2_sse2; + raid_par_ptr[1] = raid_par2_sse2; raid_parz_ptr = raid_parz_sse2; #endif } - if (cpu_has_ssse3()) { + if (raid_cpu_has_ssse3()) { #if defined(__x86_64__) raid_par3_ptr = raid_par3_ssse3ext; - raid_par4_ptr = raid_par4_ssse3ext; - raid_par5_ptr = raid_par5_ssse3ext; - raid_par6_ptr = raid_par6_ssse3ext; + raid_par_ptr[3] = raid_par4_ssse3ext; + raid_par_ptr[4] = raid_par5_ssse3ext; + raid_par_ptr[5] = raid_par6_ssse3ext; #else raid_par3_ptr = raid_par3_ssse3; - raid_par4_ptr = raid_par4_ssse3; - raid_par5_ptr = raid_par5_ssse3; - raid_par6_ptr = raid_par6_ssse3; + raid_par_ptr[3] = raid_par4_ssse3; + raid_par_ptr[4] = raid_par5_ssse3; + raid_par_ptr[5] = raid_par6_ssse3; #endif - raid_rec1_ptr = raid_rec1_ssse3; - raid_rec2_ptr = raid_rec2_ssse3; - raid_recX_ptr = raid_recX_ssse3; + raid_rec_ptr[0] = raid_rec1_ssse3; + raid_rec_ptr[1] = raid_rec2_ssse3; + raid_rec_ptr[2] = raid_recX_ssse3; + raid_rec_ptr[3] = raid_recX_ssse3; + raid_rec_ptr[4] = raid_recX_ssse3; + raid_rec_ptr[5] = raid_recX_ssse3; } #endif + + /* set the default mode */ + raid_mode(RAID_MODE_CAUCHY); } static struct raid_func { @@ -2991,12 +3020,12 @@ static const char* raid_tag(void* func) const char* raid_par1_tag(void) { - return raid_tag(raid_par1_ptr); + return raid_tag(raid_par_ptr[0]); } const char* raid_par2_tag(void) { - return raid_tag(raid_par2_ptr); + return raid_tag(raid_par_ptr[1]); } const char* raid_parz_tag(void) @@ -3006,36 +3035,36 @@ const char* raid_parz_tag(void) const char* raid_par3_tag(void) { - return raid_tag(raid_par3_ptr); + return raid_tag(raid_par_ptr[2]); } const char* raid_par4_tag(void) { - return raid_tag(raid_par4_ptr); + return raid_tag(raid_par_ptr[3]); } const char* raid_par5_tag(void) { - return raid_tag(raid_par5_ptr); + return raid_tag(raid_par_ptr[4]); } const char* raid_par6_tag(void) { - return raid_tag(raid_par6_ptr); + return raid_tag(raid_par_ptr[5]); } const char* raid_rec1_tag(void) { - return raid_tag(raid_rec1_ptr); + return raid_tag(raid_rec_ptr[0]); } const char* raid_rec2_tag(void) { - return raid_tag(raid_rec2_ptr); + return raid_tag(raid_rec_ptr[1]); } const char* raid_recX_tag(void) { - return raid_tag(raid_recX_ptr); + return raid_tag(raid_rec_ptr[2]); } diff --git a/raid.h b/raid.h index 1301a612..78b7bee3 100644 --- a/raid.h +++ b/raid.h @@ -41,14 +41,9 @@ #define RAID_MODE_VANDERMONDE 1 /** - * Max level of parity supported for RAID_MODE_CAUCHY. + * Max level of parity supported in the default mode RAID_MODE_CAUCHY. */ -#define RAID_PARITY_CAUCHY_MAX 6 - -/** - * Max level of parity supported for RAID_MODE_VANDERMONDE. - */ -#define RAID_PARITY_VANDERMONDE_MAX 3 +#define RAID_PARITY_MAX 6 /** * Maximum number of data disks. @@ -62,39 +57,74 @@ void raid_init(void); /** - * Set the RAID mode to use. One of RAID_MODE_*. + * Sets the mode to use. One of RAID_MODE_*. * - * You can change mode at any time, and it will affect next calls to raid_par() and raid_rec(). + * You can change mode at any time, and it will affect next calls to raid_par(), + * raid_rec() and raid_recpar(). * - * The two modes are compatible for the first two levels of parity. The third one is different. + * The two modes are compatible for the first two levels of parity. The third one + * is different. */ -void raid_set(unsigned mode); +void raid_mode(int mode); + +/** + * Sets the zero buffer to use in recovering. + * + * Before calling raid_rec() and raid_recpar() you must provide a memory + * buffer filled with zero with the same size of the blocks to recover. + */ +void raid_zero(void* zero); /** * Computes the parity. * * \param np Number of parities to compute. * \param nd Number of data disks. - * \param size Size of the blocks pointed by vbuf. It must be a power of 2, and at least 64. - * \param vbuf Vector of pointers to the blocks for disks and parities. - * It has (::nd + ::np) elements. Each element points to a buffer of ::size bytes. + * \param size Size of the blocks pointed by vv. It must be a power of 2, and at least 64. + * \param vv Vector of pointers to the blocks for disks and parities. + * It has (::nd + ::np) elements. Each element points to a buffer of ::size bytes. */ -void raid_par(unsigned np, unsigned nd, unsigned size, unsigned char** vbuf); +void raid_par(int np, int nd, size_t size, void** vv); /** * Recovers failures of data disks using the specified parities. * - * Note that only the data blocks are recovered. The parity is destroyed. - * \param nr Number of failed disks to recover. + * Only the data blocks are recovered. The parity buffers are destroyed. + * If you want also to recompute parities, you can use raid_recpar(). + * + * \param nr Number of failed data disks to recover. * \param id[] Vector of ::nr indexes of the data disks to recover. Starting from 0. - * \param ip[] Vector of ::nr indexes of the parity to use for the recovering. Starting from 0. + * \param ip[] Vector of ::nr indexes of the parity disks to use for recovering. Starting from 0. * \param nd Number of data disks. - * \param size Size of the blocks pointed by vbuf. It must be a power of 2, and at least 64. - * \param vbuf Vector of pointers to the blocks for disks and parities. - * It has (::nd + max(::ip[] + 1)) elements. Each element points to a buffer of ::size bytes. - * \param zero Buffer filled with 0 of ::size bytes. This buffer is not modified. + * \param size Size of the blocks pointed by vv. It must be a power of 2, and at least 64. + * \param vv Vector of pointers to the blocks for disks and parities. + * It has (::nd + ::ip[::nr - 1] + 1) elements. Each element points to a buffer of ::size bytes. */ -void raid_rec(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); +void raid_rec(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); + +/** + * Recovers failures of data and parity disks. + * + * It's similar at raid_rec() but it also recovers parities. + * + * In case there are more parities available than needed, ones to use are + * automatically selected, excluding the ones specified in the ::ip vector. + * + * Ensure to have ::nrd + ::nrp <= ::np, otherwise recovering is not possible. + * + * \param nrd Number of failed data disks to recover. + * \param id[] Vector of ::nrd indexes of the data disks to recover. Starting from 0. + * \param nrp Number of failed parity disks to recover. + * \param ip[] Vector of ::nrp indexes of the parity disks to recover. Starting from 0. + * Note that as difference than raid_rec() these are the failed parities. + * All the parities not specified here are assumed correct, and they are not recomputed. + * \param np Number of parity disks. + * \param nd Number of data disks. + * \param size Size of the blocks pointed by vv. It must be a power of 2, and at least 64. + * \param vv Vector of pointers to the blocks for disks and parities. + * It has (::nd + ::np) elements. Each element points to a buffer of ::size bytes. + */ +void raid_recpar(int nrd, const int* id, int nrp, int* ip, int np, int nd, size_t size, void** vv); /** * Gets the name of the selected function to compute the parity. @@ -115,34 +145,34 @@ const char* raid_recX_tag(void); * Specialized parity computation. * Only for testing. */ -void raid_par1_int32(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par1_int64(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par1_sse2(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par2_int32(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par2_int64(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par2_sse2(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par2_sse2ext(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_parz_int32(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_parz_int64(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_parz_sse2(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_parz_sse2ext(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par3_int8(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par3_ssse3(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par3_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par4_int8(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par4_ssse3(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par4_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par5_int8(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par5_ssse3(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par5_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par6_int8(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par6_ssse3(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_par6_ssse3ext(unsigned nd, unsigned size, unsigned char** vbuf); -void raid_rec1_int8(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); -void raid_rec2_int8(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); -void raid_recX_int8(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); -void raid_rec1_ssse3(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); -void raid_rec2_ssse3(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); -void raid_recX_ssse3(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); +void raid_par1_int32(int nd, size_t size, void** vv); +void raid_par1_int64(int nd, size_t size, void** vv); +void raid_par1_sse2(int nd, size_t size, void** vv); +void raid_par2_int32(int nd, size_t size, void** vv); +void raid_par2_int64(int nd, size_t size, void** vv); +void raid_par2_sse2(int nd, size_t size, void** vv); +void raid_par2_sse2ext(int nd, size_t size, void** vv); +void raid_parz_int32(int nd, size_t size, void** vv); +void raid_parz_int64(int nd, size_t size, void** vv); +void raid_parz_sse2(int nd, size_t size, void** vv); +void raid_parz_sse2ext(int nd, size_t size, void** vv); +void raid_par3_int8(int nd, size_t size, void** vv); +void raid_par3_ssse3(int nd, size_t size, void** vv); +void raid_par3_ssse3ext(int nd, size_t size, void** vv); +void raid_par4_int8(int nd, size_t size, void** vv); +void raid_par4_ssse3(int nd, size_t size, void** vv); +void raid_par4_ssse3ext(int nd, size_t size, void** vv); +void raid_par5_int8(int nd, size_t size, void** vv); +void raid_par5_ssse3(int nd, size_t size, void** vv); +void raid_par5_ssse3ext(int nd, size_t size, void** vv); +void raid_par6_int8(int nd, size_t size, void** vv); +void raid_par6_ssse3(int nd, size_t size, void** vv); +void raid_par6_ssse3ext(int nd, size_t size, void** vv); +void raid_rec1_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_rec2_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_recX_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_rec1_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_rec2_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_recX_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); #endif diff --git a/scrub.c b/scrub.c index dbcd1d2c..646dae39 100644 --- a/scrub.c +++ b/scrub.c @@ -44,7 +44,7 @@ static int state_scrub_process(struct snapraid_state* state, struct snapraid_par block_off_t i; unsigned j; void* buffer_alloc; - unsigned char** buffer; + void** buffer; unsigned buffermax; data_off_t countsize; block_off_t countpos; diff --git a/selftest.c b/selftest.c index 76134307..ab375f41 100644 --- a/selftest.c +++ b/selftest.c @@ -83,29 +83,29 @@ static void combotest(void) static void recovtest(unsigned mode, unsigned nd, unsigned block_size) { void* buffer_alloc; - unsigned char** buffer; - unsigned char** data; - unsigned char** parity; - unsigned char** test; - unsigned char* data_save[LEV_MAX]; - unsigned char* parity_save[LEV_MAX]; - unsigned char* zero; - unsigned char* waste; + void** buffer; + void** data; + void** parity; + void** test; + void* data_save[LEV_MAX]; + void* parity_save[LEV_MAX]; + void* zero; + void* waste; unsigned buffermax; int id[LEV_MAX]; int ip[LEV_MAX]; unsigned i; unsigned j; unsigned nr; - void (*map[LEV_MAX][4])(unsigned nr, const int* id, const int* ip, unsigned nd, unsigned size, unsigned char** vbuf, unsigned char* zero); + void (*map[LEV_MAX][4])(int nr, const int* id, const int* ip, int nd, size_t size, void** vbuf); unsigned mac[LEV_MAX]; unsigned np; - raid_set(mode); + raid_mode(mode); if (mode == RAID_MODE_CAUCHY) - np = RAID_PARITY_CAUCHY_MAX; + np = RAID_PARITY_MAX; else - np = RAID_PARITY_VANDERMONDE_MAX; + np = 3; buffermax = nd + np * 2 + 2; @@ -121,14 +121,12 @@ static void recovtest(unsigned mode, unsigned nd, unsigned block_size) zero = buffer[buffermax-2]; memset(zero, 0, block_size); + raid_zero(zero); waste = buffer[buffermax-1]; /* fill data disk with random */ - for(i=0;iraid_mode == RAID_MODE_VANDERMONDE) { - if (state->level > RAID_PARITY_VANDERMONDE_MAX) { - fprintf(stderr, "If you use the z-parity you cannot have more than %u parities.\n", RAID_PARITY_VANDERMONDE_MAX); + if (state->level > 3) { + fprintf(stderr, "If you use the z-parity you cannot have more than 3 parities.\n"); exit(EXIT_FAILURE); } } @@ -283,7 +283,7 @@ static void state_config_check(struct snapraid_state* state, const char* path, t /* check for speed */ #if defined(__i386__) || defined(__x86_64__) - if (!cpu_has_ssse3()) + if (!raid_cpu_has_ssse3()) #endif if (state->raid_mode == RAID_MODE_CAUCHY) { if (state->level == 3) { @@ -704,7 +704,7 @@ void state_config(struct snapraid_state* state, const char* path, const char* co state->besthash = HASH_SPOOKY2; } else { #if defined(__i386__) || defined(__x86_64__) - if (sizeof(void*) == 4 && !cpu_has_slowmult()) + if (sizeof(void*) == 4 && !raid_cpu_has_slowmult()) state->besthash = HASH_MURMUR3; else state->besthash = HASH_SPOOKY2; diff --git a/sync.c b/sync.c index a3af97ca..9634326f 100644 --- a/sync.c +++ b/sync.c @@ -44,7 +44,7 @@ static int state_sync_process(struct snapraid_state* state, struct snapraid_pari block_off_t i; unsigned j; void* buffer_alloc; - unsigned char** buffer; + void** buffer; unsigned buffermax; data_off_t countsize; block_off_t countpos; diff --git a/tables.h b/tables.h index 7d74157c..68011a55 100644 --- a/tables.h +++ b/tables.h @@ -15,7 +15,7 @@ * along with this program. If not, see . */ -const unsigned char __attribute__((aligned(256))) gfmul[256][256] = +const uint8_t __attribute__((aligned(256))) gfmul[256][256] = { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -8723,7 +8723,7 @@ const unsigned char __attribute__((aligned(256))) gfmul[256][256] = }, }; -const unsigned char __attribute__((aligned(256))) gfexp2[256] = +const uint8_t __attribute__((aligned(256))) gfexp[256] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1d, 0x3a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, @@ -8759,7 +8759,7 @@ const unsigned char __attribute__((aligned(256))) gfexp2[256] = 0x1b, 0x36, 0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01, }; -const unsigned char __attribute__((aligned(256))) gfinv[256] = +const uint8_t __attribute__((aligned(256))) gfinv[256] = { /* note that the first element is not significative */ 0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, @@ -8804,7 +8804,7 @@ const unsigned char __attribute__((aligned(256))) gfinv[256] = * 01 02 04 08 10 20 40 80 1d 3a 74 e8 cd 87 13 26 4c 98 2d 5a b4 75 ea c9 8f 03 06 0c 18 30 60 c0 9d 27 4e 9c 25 4a 94 35 6a d4 b5 77 ee c1 9f 23 46 8c 05 0a 14 28 50 a0 5d ba 69 d2 b9 6f de a1 5f be 61 c2 99 2f 5e bc 65 ca 89 0f 1e 3c 78 f0 fd e7 d3 bb 6b d6 b1 7f fe e1 df a3 5b b6 71 e2 d9 af 43 86 11 22 44 88 0d 1a 34 68 d0 bd 67 ce 81 1f 3e 7c f8 ed c7 93 3b 76 ec c5 97 33 66 cc 85 17 2e 5c b8 6d da a9 4f 9e 21 42 84 15 2a 54 a8 4d 9a 29 52 a4 55 aa 49 92 39 72 e4 d5 b7 73 e6 d1 bf 63 c6 91 3f 7e fc e5 d7 b3 7b f6 f1 ff e3 db ab 4b 96 31 62 c4 95 37 6e dc a5 57 ae 41 82 19 32 64 c8 8d 07 0e 1c 38 70 e0 dd a7 53 a6 51 a2 59 b2 79 f2 f9 ef c3 9b 2b 56 ac 45 8a 09 12 24 48 90 3d 7a f4 f5 f7 f3 fb eb cb 8b 0b 16 2c 58 b0 7d fa e9 cf 83 1b 36 6c * 01 8e 47 ad d8 6c 36 1b 83 cf e9 fa 7d b0 58 2c 16 0b 8b cb eb fb f3 f7 f5 f4 7a 3d 90 48 24 12 09 8a 45 ac 56 2b 9b c3 ef f9 f2 79 b2 59 a2 51 a6 53 a7 dd e0 70 38 1c 0e 07 8d c8 64 32 19 82 41 ae 57 a5 dc 6e 37 95 c4 62 31 96 4b ab db e3 ff f1 f6 7b b3 d7 e5 fc 7e 3f 91 c6 63 bf d1 e6 73 b7 d5 e4 72 39 92 49 aa 55 a4 52 29 9a 4d a8 54 2a 15 84 42 21 9e 4f a9 da 6d b8 5c 2e 17 85 cc 66 33 97 c5 ec 76 3b 93 c7 ed f8 7c 3e 1f 81 ce 67 bd d0 68 34 1a 0d 88 44 22 11 86 43 af d9 e2 71 b6 5b a3 df e1 fe 7f b1 d6 6b bb d3 e7 fd f0 78 3c 1e 0f 89 ca 65 bc 5e 2f 99 c2 61 be 5f a1 de 6f b9 d2 69 ba 5d a0 50 28 14 0a 05 8c 46 23 9f c1 ee 77 b5 d4 6a 35 94 4a 25 9c 4e 27 9d c0 60 30 18 0c 06 03 8f c9 ea 75 b4 5a 2d 98 4c 26 13 87 cd e8 74 3a 1d 80 40 20 */ -const unsigned char __attribute__((aligned(256))) gfvandermonde[3][256] = +const uint8_t __attribute__((aligned(256))) gfvandermonde[3][256] = { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, @@ -8921,7 +8921,7 @@ const unsigned char __attribute__((aligned(256))) gfvandermonde[3][256] = * 01 97 7f 9c 7c 18 bd a2 58 1a da 74 70 a3 e5 47 29 07 f5 80 23 e9 fa 46 54 a0 99 95 53 9b 0b c7 09 c0 78 89 92 e3 0d b0 2a 8c fb 17 3f 26 65 87 27 5c 66 61 79 4d 32 b3 8d 52 e2 82 3d f9 c5 02 bc 4c 73 48 62 af ba 41 d9 c4 2f b1 33 b8 15 7d cf 3a a9 5f 84 6d 34 1b 44 94 72 81 42 be cc 4b 0a 6f 5a 22 36 b5 3c 9d 13 7e 08 dd d6 5e 04 fc 5b ec ef f1 6e 1e 77 24 e6 c6 aa cb fd 51 67 06 6a 4a 88 db b2 c2 5d 43 40 f7 50 a8 f2 7a 71 a4 d2 bf 31 90 19 9a 8e f6 c3 a6 e7 60 12 ee 2d de 38 e8 b7 98 c1 28 f3 05 96 63 d1 b9 14 9f 1d 83 68 75 ed 16 03 ce e4 df e0 10 ae 69 55 91 2e 4e fe 21 1f 9e e1 d5 cd ca f0 8b 2b c9 8a 93 bb 57 20 86 1c a1 4f 3e 25 d4 6c a5 6b a7 37 ff 39 35 0c f8 ea 56 45 8f 2c 59 ab 85 eb 49 0f dc d8 76 b6 f4 0e 11 b4 d0 30 d3 3b ad d7 * 01 2b 3f cf 73 2c d6 ed cb 74 15 78 8a c1 17 c9 89 68 21 ab 76 3b 4b 5a 6e 0e b9 d3 b6 3e 36 86 bf a2 a7 30 14 eb c7 2d 96 67 20 b5 9a e0 a8 c6 80 04 8d fe 75 5e 23 ca 8f 48 99 0d df 8e b8 70 29 9c 44 69 3d a5 c2 90 d2 1c 9b 02 1d 98 93 ec 84 e8 64 4c 3a 8b 97 f3 e5 c0 7d 26 c8 08 a0 62 82 55 f7 33 f6 51 63 4d 77 da fd c3 38 6d ee 09 47 a3 05 de a6 f1 22 25 6a 0c 81 b2 6b 58 d5 b3 fc fb 28 7f 07 dc 7a 9e d0 37 b4 e1 1a 24 03 ae 94 ba 88 2f ea 2e 8c 5b bb 79 d1 11 ff a4 19 3c 2a 4e 52 e3 95 bd 31 5d 35 4a 41 c4 db 42 c5 0b 49 1b 7c e4 b0 9d 45 f0 a9 61 57 06 d4 40 91 56 13 fa 87 ac 27 54 dd 59 1f 71 39 43 6c f9 be 4f f4 1e 32 cd e9 7e 7b 66 5f ef e7 6f 0a 60 d7 b7 83 92 e2 af 72 f8 b1 50 10 ce 18 53 a1 cc ad 12 34 0f f5 aa 16 e6 f2 d8 85 9f bc */ -const unsigned char __attribute__((aligned(256))) gfcauchy[6][256] = +const uint8_t __attribute__((aligned(256))) gfcauchy[6][256] = { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, @@ -9136,7 +9136,7 @@ const unsigned char __attribute__((aligned(256))) gfcauchy[6][256] = * Indexes are [DISK][PARITY - 2][LH]. * Where DISK is from 0 to 250, PARITY from 2 to 5, LH from 0 to 1. */ -const unsigned char __attribute__((aligned(256))) gfgenpshufb[251][4][2][16] = +const uint8_t __attribute__((aligned(256))) gfcauchypshufb[251][4][2][16] = { { { @@ -13666,7 +13666,7 @@ const unsigned char __attribute__((aligned(256))) gfgenpshufb[251][4][2][16] = * Indexes are [MULTIPLER][LH]. * Where MULTIPLER is from 0 to 255, LH from 0 to 1. */ -const unsigned char __attribute__((aligned(256))) gfmulpshufb[256][2][16] = +const uint8_t __attribute__((aligned(256))) gfmulpshufb[256][2][16] = { { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, diff --git a/util.c b/util.c index 8b31e547..f0efa796 100644 --- a/util.c +++ b/util.c @@ -1133,7 +1133,7 @@ void crc32c_init(void) { crc32c = crc32c_gen; #if HAVE_CRC32B - if (cpu_has_sse42()) { + if (raid_cpu_has_sse42()) { crc32c = crc32c_x86; } #endif @@ -1391,9 +1391,9 @@ void* malloc_nofail_align(size_t size, void** freeptr) */ #define MALLOC_DISPLACEMENT (7*256) -unsigned char** malloc_nofail_vector_align(size_t reverse, size_t count, size_t size, void** freeptr) +void** malloc_nofail_vector_align(size_t reverse, size_t count, size_t size, void** freeptr) { - unsigned char** buffer; + void** buffer; unsigned char* buffer_aligned; size_t i; @@ -1416,8 +1416,9 @@ unsigned char** malloc_nofail_vector_align(size_t reverse, size_t count, size_t return buffer; } -void mtest_vector(unsigned char** buf, size_t count, size_t size) +void mtest_vector(void** vv, size_t count, size_t size) { + unsigned char** v = (unsigned char**)vv; size_t i; size_t j; unsigned k; @@ -1428,7 +1429,7 @@ void mtest_vector(unsigned char** buf, size_t count, size_t size) d = 0; for(i=0;i0;--j) { - if (buf[i][j-1] != p) + if (v[i][j-1] != p) goto fail; - buf[i][j-1] = d; + v[i][j-1] = d; } } } @@ -1466,6 +1467,18 @@ fail: exit(EXIT_FAILURE); } +void mrand_vector(void** vv, size_t count, size_t size) +{ + unsigned char** v = (unsigned char**)vv; + size_t i; + size_t j; + + for(i=0;i