diff --git a/Makefile.am b/Makefile.am index 93d55fff..e5adddf0 100644 --- a/Makefile.am +++ b/Makefile.am @@ -3,6 +3,13 @@ check_PROGRAMS = mktest EXTRA_PROGRAMS = mktables snapraid_SOURCES = \ + raid/raid.c \ + raid/module.c \ + raid/tables.c \ + raid/int.c \ + raid/x86.c \ + raid/intz.c \ + raid/x86z.c \ snapraid.c \ tommy.c \ util.c \ @@ -21,7 +28,6 @@ snapraid_SOURCES = \ parity.c \ handle.c \ fnmatch.c \ - raid.c \ selftest.c \ speed.c \ import.c \ @@ -29,9 +35,13 @@ snapraid_SOURCES = \ unix.c mktables_SOURCES = \ - mktables.c + raid/mktables.c noinst_HEADERS = \ + raid/raid.h \ + raid/internal.h \ + raid/cpu.h \ + raid/gf.h \ portable.h \ snapraid.h \ util.h \ @@ -58,10 +68,7 @@ noinst_HEADERS = \ tommyhashdyn.c \ tommylist.c \ fnmatch.h \ - cpu.h \ - raid.h \ combo.h \ - tables.h \ import.h \ mingw.h \ unix.h diff --git a/check.c b/check.c index 1047314f..ad4c62a2 100644 --- a/check.c +++ b/check.c @@ -23,7 +23,7 @@ #include "state.h" #include "parity.h" #include "handle.h" -#include "raid.h" +#include "raid/raid.h" #include "combo.h" /****************************************************************************/ diff --git a/cpu.h b/raid/cpu.h similarity index 99% rename from cpu.h rename to raid/cpu.h index 4d8c2079..62504820 100644 --- a/cpu.h +++ b/raid/cpu.h @@ -1,5 +1,5 @@ /* - * Copyright (C) 2011,2013 Andrea Mazzoleni + * Copyright (C) 2013 Andrea Mazzoleni * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/raid/gf.h b/raid/gf.h new file mode 100644 index 00000000..2eaedfe5 --- /dev/null +++ b/raid/gf.h @@ -0,0 +1,144 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#ifndef __RAID_GF_H +#define __RAID_GF_H + +/* + * Galois field operations. + * + * Basic range checks are implemented using BUG_ON(). + */ + +/* + * Generic assert() with inverted meaning. + */ +#define BUG_ON(a) assert(!(a)) + + +/* + * GF a*b. + */ +static __always_inline uint8_t mul(uint8_t a, uint8_t b) +{ + return gfmul[a][b]; +} + +/* + * GF 1/a. + * Not defined for a == 0. + */ +static __always_inline uint8_t inv(uint8_t v) +{ + BUG_ON(v == 0); /* division by zero */ + + return gfinv[v]; +} + +/* + * GF 2^a. + */ +static __always_inline uint8_t pow2(int v) +{ + BUG_ON(v < 0 || v > 254); /* invalid exponent */ + + return gfexp[v]; +} + +/* + * Gets the multiplication table for a specified value. + */ +static __always_inline const uint8_t* table(uint8_t v) +{ + return gfmul[v]; +} + +extern const uint8_t (*raid_gen)[256]; + +/* + * Gets the generator matrix coefficient for parity 'p' and disk 'd'. + */ +static __always_inline uint8_t A(int p, int d) +{ + return raid_gen[p][d]; +} + +/* + * Dereference as uint8_t + */ +#define v_8(p) (*(uint8_t*)&(p)) + +/* + * Dereference as uint32_t + */ +#define v_32(p) (*(uint32_t*)&(p)) + +/* + * Dereference as uint64_t + */ +#define v_64(p) (*(uint64_t*)&(p)) + +/* + * Multiply each byte of a uint32 by 2 in the GF(2^8). + */ +static __always_inline uint32_t x2_32(uint32_t v) +{ + uint32_t mask = v & 0x80808080U; + mask = (mask << 1) - (mask >> 7); + v = (v << 1) & 0xfefefefeU; + v ^= mask & 0x1d1d1d1dU; + return v; +} + +/* + * Multiply each byte of a uint64 by 2 in the GF(2^8). + */ +static __always_inline uint64_t x2_64(uint64_t v) +{ + uint64_t mask = v & 0x8080808080808080ULL; + mask = (mask << 1) - (mask >> 7); + v = (v << 1) & 0xfefefefefefefefeULL; + v ^= mask & 0x1d1d1d1d1d1d1d1dULL; + return v; +} + +/* + * Divide each byte of a uint32 by 2 in the GF(2^8). + */ +static __always_inline uint32_t d2_32(uint32_t v) +{ + uint32_t mask = v & 0x01010101U; + mask = (mask << 8) - mask; + v = (v >> 1) & 0x7f7f7f7fU; + v ^= mask & 0x8e8e8e8eU; + return v; +} + +/* + * Divide each byte of a uint64 by 2 in the GF(2^8). + */ +static __always_inline uint64_t d2_64(uint64_t v) +{ + uint64_t mask = v & 0x0101010101010101ULL; + mask = (mask << 8) - mask; + v = (v >> 1) & 0x7f7f7f7f7f7f7f7fULL; + v ^= mask & 0x8e8e8e8e8e8e8e8eULL; + return v; +} + +#endif + diff --git a/raid/int.c b/raid/int.c new file mode 100644 index 00000000..21417021 --- /dev/null +++ b/raid/int.c @@ -0,0 +1,570 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "internal.h" +#include "gf.h" + +/* + * PAR1 (RAID5 with xor) 32bit C implementation + */ +void raid_par1_int32(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + int d, l; + size_t i; + + uint32_t p0; + uint32_t p1; + + l = nd - 1; + p = v[nd]; + + for(i=0;i=0;--d) { + p0 ^= v_32(v[d][i]); + p1 ^= v_32(v[d][i+4]); + } + v_32(p[i]) = p0; + v_32(p[i+4]) = p1; + } +} + +/* + * PAR1 (RAID5 with xor) 64bit C implementation + */ +void raid_par1_int64(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + int d, l; + size_t i; + + uint64_t p0; + uint64_t p1; + + l = nd - 1; + p = v[nd]; + + for(i=0;i=0;--d) { + p0 ^= v_64(v[d][i]); + p1 ^= v_64(v[d][i+8]); + } + v_64(p[i]) = p0; + v_64(p[i+8]) = p1; + } +} + +/* + * PAR2 (RAID6 with powers of 2) 32bit C implementation + */ +void raid_par2_int32(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + int d, l; + size_t i; + + uint32_t d0, q0, p0; + uint32_t d1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + + for(i=0;i=0;--d) { + d0 = v_32(v[d][i]); + d1 = v_32(v[d][i+4]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_32(q0); + q1 = x2_32(q1); + + q0 ^= d0; + q1 ^= d1; + } + v_32(p[i]) = p0; + v_32(p[i+4]) = p1; + v_32(q[i]) = q0; + v_32(q[i+4]) = q1; + } +} + +/* + * PAR2 (RAID6 with powers of 2) 64bit C implementation + */ +void raid_par2_int64(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + int d, l; + size_t i; + + uint64_t d0, q0, p0; + uint64_t d1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + + for(i=0;i=0;--d) { + d0 = v_64(v[d][i]); + d1 = v_64(v[d][i+8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0); + q1 = x2_64(q1); + + q0 ^= d0; + q1 ^= d1; + } + v_64(p[i]) = p0; + v_64(p[i+8]) = p1; + v_64(q[i]) = q0; + v_64(q[i+8]) = q1; + } +} + +/* + * PAR3 (triple parity with Cauchy matrix) 8bit C implementation + * + * Note that instead of a generic multiplicationt table, likely resulting + * in multiple cache misses, a precomputed table could be used. + * But this is only a kind of reference function, and we are not really + * interested in speed. + */ +void raid_par3_int8(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + int d, l; + size_t i; + + uint8_t d0, r0, q0, p0; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + + for(i=0;i0;--d) { + d0 = v_8(v[d][i]); + + p0 ^= d0; + q0 ^= gfmul[d0][gfcauchy[1][d]]; + r0 ^= gfmul[d0][gfcauchy[2][d]]; + } + + /* first disk with all coefficients at 1 */ + d0 = v_8(v[0][i]); + + p0 ^= d0; + q0 ^= d0; + r0 ^= d0; + + v_8(p[i]) = p0; + v_8(q[i]) = q0; + v_8(r[i]) = r0; + } +} + +/* + * PAR4 (quad parity with Cauchy matrix) 8bit C implementation + * + * Note that instead of a generic multiplicationt table, likely resulting + * in multiple cache misses, a precomputed table could be used. + * But this is only a kind of reference function, and we are not really + * interested in speed. + */ +void raid_par4_int8(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + int d, l; + size_t i; + + uint8_t d0, s0, r0, q0, p0; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + + for(i=0;i0;--d) { + d0 = v_8(v[d][i]); + + p0 ^= d0; + q0 ^= gfmul[d0][gfcauchy[1][d]]; + r0 ^= gfmul[d0][gfcauchy[2][d]]; + s0 ^= gfmul[d0][gfcauchy[3][d]]; + } + + /* first disk with all coefficients at 1 */ + d0 = v_8(v[0][i]); + + p0 ^= d0; + q0 ^= d0; + r0 ^= d0; + s0 ^= d0; + + v_8(p[i]) = p0; + v_8(q[i]) = q0; + v_8(r[i]) = r0; + v_8(s[i]) = s0; + } +} + +/* + * PAR5 (penta parity with Cauchy matrix) 8bit C implementation + * + * Note that instead of a generic multiplicationt table, likely resulting + * in multiple cache misses, a precomputed table could be used. + * But this is only a kind of reference function, and we are not really + * interested in speed. + */ +void raid_par5_int8(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; + int d, l; + size_t i; + + uint8_t d0, t0, s0, r0, q0, p0; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; + + for(i=0;i0;--d) { + d0 = v_8(v[d][i]); + + p0 ^= d0; + q0 ^= gfmul[d0][gfcauchy[1][d]]; + r0 ^= gfmul[d0][gfcauchy[2][d]]; + s0 ^= gfmul[d0][gfcauchy[3][d]]; + t0 ^= gfmul[d0][gfcauchy[4][d]]; + } + + /* first disk with all coefficients at 1 */ + d0 = v_8(v[0][i]); + + p0 ^= d0; + q0 ^= d0; + r0 ^= d0; + s0 ^= d0; + t0 ^= d0; + + v_8(p[i]) = p0; + v_8(q[i]) = q0; + v_8(r[i]) = r0; + v_8(s[i]) = s0; + v_8(t[i]) = t0; + } +} + +/* + * PAR6 (hexa parity with Cauchy matrix) 8bit C implementation + * + * Note that instead of a generic multiplicationt table, likely resulting + * in multiple cache misses, a precomputed table could be used. + * But this is only a kind of reference function, and we are not really + * interested in speed. + */ +void raid_par6_int8(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + uint8_t* s; + uint8_t* t; + uint8_t* u; + int d, l; + size_t i; + + uint8_t d0, u0, t0, s0, r0, q0, p0; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + s = v[nd+3]; + t = v[nd+4]; + u = v[nd+5]; + + for(i=0;i0;--d) { + d0 = v_8(v[d][i]); + + p0 ^= d0; + q0 ^= gfmul[d0][gfcauchy[1][d]]; + r0 ^= gfmul[d0][gfcauchy[2][d]]; + s0 ^= gfmul[d0][gfcauchy[3][d]]; + t0 ^= gfmul[d0][gfcauchy[4][d]]; + u0 ^= gfmul[d0][gfcauchy[5][d]]; + } + + /* first disk with all coefficients at 1 */ + d0 = v_8(v[0][i]); + + p0 ^= d0; + q0 ^= d0; + r0 ^= d0; + s0 ^= d0; + t0 ^= d0; + u0 ^= d0; + + v_8(p[i]) = p0; + v_8(q[i]) = q0; + v_8(r[i]) = r0; + v_8(s[i]) = s0; + v_8(t[i]) = t0; + v_8(u[i]) = u0; + } +} + +/* + * Recover failure of one data block at index id[0] using parity at index + * ip[0] for any RAID level. + * + * Starting from the equation: + * + * Pd = A[ip[0],id[0]] * Dx + * + * and solving we get: + * + * Dx = A[ip[0],id[0]]^-1 * Pd + */ +void raid_rec1_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* pa; + const uint8_t* T; + uint8_t G; + uint8_t V; + size_t i; + + (void)nr; /* unused, it's always 1 */ + + /* if it's RAID5 uses the dedicated and faster function */ + if (ip[0] == 0) { + raid_rec1_par1(id, nd, size, vv); + return; + } + + /* setup the coefficients matrix */ + G = A(ip[0],id[0]); + + /* invert it to solve the system of linear equations */ + V = inv(G); + + /* get multiplication tables */ + T = table(V); + + /* compute delta parity */ + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd+ip[0]]; + pa = v[id[0]]; + + for(i=0;i. + */ + +#ifndef __RAID_INTERNAL_H +#define __RAID_INTERNAL_H + +#include "../portable.h" + +#include "raid.h" + +/* + * Internal functions. + * + * These are intented to provide access for testing. + */ +void raid_invert(uint8_t* M, uint8_t* V, int n); +void raid_delta_gen(int nr, const int* id, const int* ip, int nd, size_t size, void** v); +void raid_rec1_par1(const int* id, int nd, size_t size, void** v); +void raid_rec2_par2(const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_par1_int32(int nd, size_t size, void** vv); +void raid_par1_int64(int nd, size_t size, void** vv); +void raid_par1_sse2(int nd, size_t size, void** vv); +void raid_par2_int32(int nd, size_t size, void** vv); +void raid_par2_int64(int nd, size_t size, void** vv); +void raid_par2_sse2(int nd, size_t size, void** vv); +void raid_par2_sse2ext(int nd, size_t size, void** vv); +void raid_parz_int32(int nd, size_t size, void** vv); +void raid_parz_int64(int nd, size_t size, void** vv); +void raid_parz_sse2(int nd, size_t size, void** vv); +void raid_parz_sse2ext(int nd, size_t size, void** vv); +void raid_par3_int8(int nd, size_t size, void** vv); +void raid_par3_ssse3(int nd, size_t size, void** vv); +void raid_par3_ssse3ext(int nd, size_t size, void** vv); +void raid_par4_int8(int nd, size_t size, void** vv); +void raid_par4_ssse3(int nd, size_t size, void** vv); +void raid_par4_ssse3ext(int nd, size_t size, void** vv); +void raid_par5_int8(int nd, size_t size, void** vv); +void raid_par5_ssse3(int nd, size_t size, void** vv); +void raid_par5_ssse3ext(int nd, size_t size, void** vv); +void raid_par6_int8(int nd, size_t size, void** vv); +void raid_par6_ssse3(int nd, size_t size, void** vv); +void raid_par6_ssse3ext(int nd, size_t size, void** vv); +void raid_rec1_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_rec2_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_recX_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_rec1_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_rec2_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); +void raid_recX_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); + +/* + * Internal naming. + * + * These are intented to provide access for testing. + */ +const char* raid_par1_tag(void); +const char* raid_par2_tag(void); +const char* raid_parz_tag(void); +const char* raid_par3_tag(void); +const char* raid_par4_tag(void); +const char* raid_par5_tag(void); +const char* raid_par6_tag(void); +const char* raid_rec1_tag(void); +const char* raid_rec2_tag(void); +const char* raid_recX_tag(void); + +/* + * Internal forwarders. + */ +extern void (*raid_par_ptr[RAID_PARITY_MAX])(int nd, size_t size, void** vv); +extern void (*raid_par3_ptr)(int nd, size_t size, void** vv); +extern void (*raid_parz_ptr)(int nd, size_t size, void** vv); +extern void (*raid_rec_ptr[RAID_PARITY_MAX])(int nr, const int* id, const int* ip, int nd, size_t size, void** vv); + +/* + * Tables. + */ +extern const uint8_t raid_gfmul[256][256] __attribute__((aligned(256))); +extern const uint8_t raid_gfexp[256] __attribute__((aligned(256))); +extern const uint8_t raid_gfinv[256] __attribute__((aligned(256))); +extern const uint8_t raid_gfvandermonde[3][256] __attribute__((aligned(256))); +extern const uint8_t raid_gfcauchy[6][256] __attribute__((aligned(256))); +extern const uint8_t raid_gfcauchypshufb[251][4][2][16] __attribute__((aligned(256))); +extern const uint8_t raid_gfmulpshufb[256][2][16] __attribute__((aligned(256))); +#define gfmul raid_gfmul +#define gfexp raid_gfexp +#define gfinv raid_gfinv +#define gfvandermonde raid_gfvandermonde +#define gfcauchy raid_gfcauchy +#define gfcauchypshufb raid_gfcauchypshufb +#define gfmulpshufb raid_gfmulpshufb + +#endif + diff --git a/raid/intz.c b/raid/intz.c new file mode 100644 index 00000000..f412f001 --- /dev/null +++ b/raid/intz.c @@ -0,0 +1,122 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "internal.h" +#include "gf.h" + +/* + * PARz (triple parity with powers of 2^-1) 32bit C implementation + */ +void raid_parz_int32(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + int d, l; + size_t i; + + uint32_t d0, r0, q0, p0; + uint32_t d1, r1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + + for(i=0;i=0;--d) { + d0 = v_32(v[d][i]); + d1 = v_32(v[d][i+4]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_32(q0); + q1 = x2_32(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = d2_32(r0); + r1 = d2_32(r1); + + r0 ^= d0; + r1 ^= d1; + } + v_32(p[i]) = p0; + v_32(p[i+4]) = p1; + v_32(q[i]) = q0; + v_32(q[i+4]) = q1; + v_32(r[i]) = r0; + v_32(r[i+4]) = r1; + } +} + +/* + * PARz (triple parity with powers of 2^-1) 64bit C implementation + */ +void raid_parz_int64(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + int d, l; + size_t i; + + uint64_t d0, r0, q0, p0; + uint64_t d1, r1, q1, p1; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + + for(i=0;i=0;--d) { + d0 = v_64(v[d][i]); + d1 = v_64(v[d][i+8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0); + q1 = x2_64(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = d2_64(r0); + r1 = d2_64(r1); + + r0 ^= d0; + r1 ^= d1; + } + v_64(p[i]) = p0; + v_64(p[i+8]) = p1; + v_64(q[i]) = q0; + v_64(q[i+8]) = q1; + v_64(r[i]) = r0; + v_64(r[i+8]) = r1; + } +} + diff --git a/mktables.c b/raid/mktables.c similarity index 92% rename from mktables.c rename to raid/mktables.c index 2f726d5d..932ba1fb 100644 --- a/mktables.c +++ b/raid/mktables.c @@ -201,8 +201,11 @@ int main(void) printf(" */\n"); printf("\n"); + printf("#include \"internal.h\"\n"); + printf("\n"); + /* a*b */ - printf("const uint8_t __attribute__((aligned(256))) gfmul[256][256] =\n"); + printf("const uint8_t __attribute__((aligned(256))) raid_gfmul[256][256] =\n"); printf("{\n"); for(i=0;i<256;++i) { printf("\t{\n"); @@ -223,7 +226,7 @@ int main(void) printf("};\n\n"); /* 2^a */ - printf("const uint8_t __attribute__((aligned(256))) gfexp[256] =\n"); + printf("const uint8_t __attribute__((aligned(256))) raid_gfexp[256] =\n"); printf("{\n"); v = 1; for(i=0;i<256;++i) { @@ -239,7 +242,7 @@ int main(void) printf("};\n\n"); /* 1/a */ - printf("const uint8_t __attribute__((aligned(256))) gfinv[256] =\n"); + printf("const uint8_t __attribute__((aligned(256))) raid_gfinv[256] =\n"); printf("{\n"); printf("\t/* note that the first element is not significative */\n"); for(i=0;i<256;++i) { @@ -272,7 +275,7 @@ int main(void) printf("\n"); } printf(" */\n"); - printf("const uint8_t __attribute__((aligned(256))) gfvandermonde[%u][256] =\n", 3); + printf("const uint8_t __attribute__((aligned(256))) raid_gfvandermonde[%u][256] =\n", 3); printf("{\n"); for(p=0;p<3;++p) { printf("\t{\n"); @@ -304,7 +307,7 @@ int main(void) printf("\n"); } printf(" */\n"); - printf("const uint8_t __attribute__((aligned(256))) gfcauchy[%u][256] =\n", PARITY); + printf("const uint8_t __attribute__((aligned(256))) raid_gfcauchy[%u][256] =\n", PARITY); printf("{\n"); for(p=0;p. + */ + +#include "internal.h" +#include "cpu.h" + +void raid_init(void) +{ + raid_par3_ptr = raid_par3_int8; + raid_par_ptr[3] = raid_par4_int8; + raid_par_ptr[4] = raid_par5_int8; + raid_par_ptr[5] = raid_par6_int8; + + if (sizeof(void*) == 4) { + raid_par_ptr[0] = raid_par1_int32; + raid_par_ptr[1] = raid_par2_int32; + raid_parz_ptr = raid_parz_int32; + } else { + raid_par_ptr[0] = raid_par1_int64; + raid_par_ptr[1] = raid_par2_int64; + raid_parz_ptr = raid_parz_int64; + } + + raid_rec_ptr[0] = raid_rec1_int8; + raid_rec_ptr[1] = raid_rec2_int8; + raid_rec_ptr[2] = raid_recX_int8; + raid_rec_ptr[3] = raid_recX_int8; + raid_rec_ptr[4] = raid_recX_int8; + raid_rec_ptr[5] = raid_recX_int8; + +#if defined(__i386__) || defined(__x86_64__) + if (raid_cpu_has_sse2()) { + raid_par_ptr[0] = raid_par1_sse2; +#if defined(__x86_64__) + if (raid_cpu_has_slowextendedreg()) { + raid_par_ptr[1] = raid_par2_sse2; + raid_parz_ptr = raid_parz_sse2; + } else { + raid_par_ptr[1] = raid_par2_sse2ext; + raid_parz_ptr = raid_parz_sse2ext; + } +#else + raid_par_ptr[1] = raid_par2_sse2; + raid_parz_ptr = raid_parz_sse2; +#endif + } + + if (raid_cpu_has_ssse3()) { +#if defined(__x86_64__) + raid_par3_ptr = raid_par3_ssse3ext; + raid_par_ptr[3] = raid_par4_ssse3ext; + raid_par_ptr[4] = raid_par5_ssse3ext; + raid_par_ptr[5] = raid_par6_ssse3ext; +#else + raid_par3_ptr = raid_par3_ssse3; + raid_par_ptr[3] = raid_par4_ssse3; + raid_par_ptr[4] = raid_par5_ssse3; + raid_par_ptr[5] = raid_par6_ssse3; +#endif + raid_rec_ptr[0] = raid_rec1_ssse3; + raid_rec_ptr[1] = raid_rec2_ssse3; + raid_rec_ptr[2] = raid_recX_ssse3; + raid_rec_ptr[3] = raid_recX_ssse3; + raid_rec_ptr[4] = raid_recX_ssse3; + raid_rec_ptr[5] = raid_recX_ssse3; + } +#endif + + /* set the default mode */ + raid_mode(RAID_MODE_CAUCHY); +} + +static struct raid_func { + const char* name; + void* p; +} RAID_FUNC[] = { + { "int8", raid_par3_int8 }, + { "int8", raid_par4_int8 }, + { "int8", raid_par5_int8 }, + { "int8", raid_par6_int8 }, + { "int32", raid_par1_int32 }, + { "int64", raid_par1_int64 }, + { "int32", raid_par2_int32 }, + { "int64", raid_par2_int64 }, + { "int32", raid_parz_int32 }, + { "int64", raid_parz_int64 }, + { "int8", raid_rec1_int8 }, + { "int8", raid_rec2_int8 }, + { "int8", raid_recX_int8 }, + +#if defined(__i386__) || defined(__x86_64__) + { "sse2", raid_par1_sse2 }, + { "sse2", raid_par2_sse2 }, + { "sse2", raid_parz_sse2 }, + { "ssse3", raid_par3_ssse3 }, + { "ssse3", raid_par4_ssse3 }, + { "ssse3", raid_par5_ssse3 }, + { "ssse3", raid_par6_ssse3 }, + { "ssse3", raid_rec1_ssse3 }, + { "ssse3", raid_rec2_ssse3 }, + { "ssse3", raid_recX_ssse3 }, +#endif + +#if defined(__x86_64__) + { "sse2e", raid_par2_sse2ext }, + { "sse2e", raid_parz_sse2ext }, + { "ssse3e", raid_par3_ssse3ext }, + { "ssse3e", raid_par4_ssse3ext }, + { "ssse3e", raid_par5_ssse3ext }, + { "ssse3e", raid_par6_ssse3ext }, +#endif + { 0, 0 } +}; + +static const char* raid_tag(void* func) +{ + struct raid_func* i = RAID_FUNC; + while (i->name != 0) { + if (i->p == func) + return i->name; + ++i; + } + return "unknown"; +} + +const char* raid_par1_tag(void) +{ + return raid_tag(raid_par_ptr[0]); +} + +const char* raid_par2_tag(void) +{ + return raid_tag(raid_par_ptr[1]); +} + +const char* raid_parz_tag(void) +{ + return raid_tag(raid_parz_ptr); +} + +const char* raid_par3_tag(void) +{ + return raid_tag(raid_par_ptr[2]); +} + +const char* raid_par4_tag(void) +{ + return raid_tag(raid_par_ptr[3]); +} + +const char* raid_par5_tag(void) +{ + return raid_tag(raid_par_ptr[4]); +} + +const char* raid_par6_tag(void) +{ + return raid_tag(raid_par_ptr[5]); +} + +const char* raid_rec1_tag(void) +{ + return raid_tag(raid_rec_ptr[0]); +} + +const char* raid_rec2_tag(void) +{ + return raid_tag(raid_rec_ptr[1]); +} + +const char* raid_recX_tag(void) +{ + return raid_tag(raid_rec_ptr[2]); +} + diff --git a/raid/raid.c b/raid/raid.c new file mode 100644 index 00000000..bdad2a26 --- /dev/null +++ b/raid/raid.c @@ -0,0 +1,458 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +/* + * Copyright notes: + * + * The RAID5 and RAID6 support was originally derived from the + * H. Peter Anvin paper "The mathematics of RAID-6" [1] and the + * libraid6 library, also by H. Peter Anvin, released with license + * "GPLv2 or any later version" inside the Linux Kernel 2.6.38. + * + * This support was later completely rewritten (many times), but + * the H. Peter Anvin's Copyright may still apply. + * + * The others RAID levels and the recovering based on matrix + * inversion is original work implemented from scratch. + */ + +/* + * The RAID5 and RAID6 support is implemented using the Galois Field + * GF(2^8) with the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 + * (285 decimal). + * + * The parity P and Q of a set of N disk Di with 0<=i RAID_PARITY_MAX); + + raid_par_ptr[np - 1](nd, size, v); +} + +/** + * Inverts the square matrix M of size nxn into V. + * We use Gauss elimination to invert. + */ +void raid_invert(uint8_t* M, uint8_t* V, int n) +{ + int i,j,k; + + /* set the identity matrix in V */ + for(i=0;i nd); + + /* if failed data is present */ + if (nr != 0) { + raid_rec_ptr[nr - 1](nr, id, ip, nd, size, v); + } +} + +void raid_recpar(int nrd, const int* id, int nrp, int* ip, int np, int nd, size_t size, void** v) +{ + BUG_ON(nrd > nd); + BUG_ON(nrd + nrp > np); + + /* if failed data is present */ + if (nrd != 0) { + int iu[RAID_PARITY_MAX]; + int i, j, k; + + /* setup the vector of parities to use */ + for(i=0,j=0,k=0;i. */ -const uint8_t __attribute__((aligned(256))) gfmul[256][256] = +#include "internal.h" + +const uint8_t __attribute__((aligned(256))) raid_gfmul[256][256] = { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -8723,7 +8725,7 @@ const uint8_t __attribute__((aligned(256))) gfmul[256][256] = }, }; -const uint8_t __attribute__((aligned(256))) gfexp[256] = +const uint8_t __attribute__((aligned(256))) raid_gfexp[256] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, 0x1d, 0x3a, 0x74, 0xe8, 0xcd, 0x87, 0x13, 0x26, @@ -8759,7 +8761,7 @@ const uint8_t __attribute__((aligned(256))) gfexp[256] = 0x1b, 0x36, 0x6c, 0xd8, 0xad, 0x47, 0x8e, 0x01, }; -const uint8_t __attribute__((aligned(256))) gfinv[256] = +const uint8_t __attribute__((aligned(256))) raid_gfinv[256] = { /* note that the first element is not significative */ 0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, @@ -8804,7 +8806,7 @@ const uint8_t __attribute__((aligned(256))) gfinv[256] = * 01 02 04 08 10 20 40 80 1d 3a 74 e8 cd 87 13 26 4c 98 2d 5a b4 75 ea c9 8f 03 06 0c 18 30 60 c0 9d 27 4e 9c 25 4a 94 35 6a d4 b5 77 ee c1 9f 23 46 8c 05 0a 14 28 50 a0 5d ba 69 d2 b9 6f de a1 5f be 61 c2 99 2f 5e bc 65 ca 89 0f 1e 3c 78 f0 fd e7 d3 bb 6b d6 b1 7f fe e1 df a3 5b b6 71 e2 d9 af 43 86 11 22 44 88 0d 1a 34 68 d0 bd 67 ce 81 1f 3e 7c f8 ed c7 93 3b 76 ec c5 97 33 66 cc 85 17 2e 5c b8 6d da a9 4f 9e 21 42 84 15 2a 54 a8 4d 9a 29 52 a4 55 aa 49 92 39 72 e4 d5 b7 73 e6 d1 bf 63 c6 91 3f 7e fc e5 d7 b3 7b f6 f1 ff e3 db ab 4b 96 31 62 c4 95 37 6e dc a5 57 ae 41 82 19 32 64 c8 8d 07 0e 1c 38 70 e0 dd a7 53 a6 51 a2 59 b2 79 f2 f9 ef c3 9b 2b 56 ac 45 8a 09 12 24 48 90 3d 7a f4 f5 f7 f3 fb eb cb 8b 0b 16 2c 58 b0 7d fa e9 cf 83 1b 36 6c * 01 8e 47 ad d8 6c 36 1b 83 cf e9 fa 7d b0 58 2c 16 0b 8b cb eb fb f3 f7 f5 f4 7a 3d 90 48 24 12 09 8a 45 ac 56 2b 9b c3 ef f9 f2 79 b2 59 a2 51 a6 53 a7 dd e0 70 38 1c 0e 07 8d c8 64 32 19 82 41 ae 57 a5 dc 6e 37 95 c4 62 31 96 4b ab db e3 ff f1 f6 7b b3 d7 e5 fc 7e 3f 91 c6 63 bf d1 e6 73 b7 d5 e4 72 39 92 49 aa 55 a4 52 29 9a 4d a8 54 2a 15 84 42 21 9e 4f a9 da 6d b8 5c 2e 17 85 cc 66 33 97 c5 ec 76 3b 93 c7 ed f8 7c 3e 1f 81 ce 67 bd d0 68 34 1a 0d 88 44 22 11 86 43 af d9 e2 71 b6 5b a3 df e1 fe 7f b1 d6 6b bb d3 e7 fd f0 78 3c 1e 0f 89 ca 65 bc 5e 2f 99 c2 61 be 5f a1 de 6f b9 d2 69 ba 5d a0 50 28 14 0a 05 8c 46 23 9f c1 ee 77 b5 d4 6a 35 94 4a 25 9c 4e 27 9d c0 60 30 18 0c 06 03 8f c9 ea 75 b4 5a 2d 98 4c 26 13 87 cd e8 74 3a 1d 80 40 20 */ -const uint8_t __attribute__((aligned(256))) gfvandermonde[3][256] = +const uint8_t __attribute__((aligned(256))) raid_gfvandermonde[3][256] = { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, @@ -8921,7 +8923,7 @@ const uint8_t __attribute__((aligned(256))) gfvandermonde[3][256] = * 01 97 7f 9c 7c 18 bd a2 58 1a da 74 70 a3 e5 47 29 07 f5 80 23 e9 fa 46 54 a0 99 95 53 9b 0b c7 09 c0 78 89 92 e3 0d b0 2a 8c fb 17 3f 26 65 87 27 5c 66 61 79 4d 32 b3 8d 52 e2 82 3d f9 c5 02 bc 4c 73 48 62 af ba 41 d9 c4 2f b1 33 b8 15 7d cf 3a a9 5f 84 6d 34 1b 44 94 72 81 42 be cc 4b 0a 6f 5a 22 36 b5 3c 9d 13 7e 08 dd d6 5e 04 fc 5b ec ef f1 6e 1e 77 24 e6 c6 aa cb fd 51 67 06 6a 4a 88 db b2 c2 5d 43 40 f7 50 a8 f2 7a 71 a4 d2 bf 31 90 19 9a 8e f6 c3 a6 e7 60 12 ee 2d de 38 e8 b7 98 c1 28 f3 05 96 63 d1 b9 14 9f 1d 83 68 75 ed 16 03 ce e4 df e0 10 ae 69 55 91 2e 4e fe 21 1f 9e e1 d5 cd ca f0 8b 2b c9 8a 93 bb 57 20 86 1c a1 4f 3e 25 d4 6c a5 6b a7 37 ff 39 35 0c f8 ea 56 45 8f 2c 59 ab 85 eb 49 0f dc d8 76 b6 f4 0e 11 b4 d0 30 d3 3b ad d7 * 01 2b 3f cf 73 2c d6 ed cb 74 15 78 8a c1 17 c9 89 68 21 ab 76 3b 4b 5a 6e 0e b9 d3 b6 3e 36 86 bf a2 a7 30 14 eb c7 2d 96 67 20 b5 9a e0 a8 c6 80 04 8d fe 75 5e 23 ca 8f 48 99 0d df 8e b8 70 29 9c 44 69 3d a5 c2 90 d2 1c 9b 02 1d 98 93 ec 84 e8 64 4c 3a 8b 97 f3 e5 c0 7d 26 c8 08 a0 62 82 55 f7 33 f6 51 63 4d 77 da fd c3 38 6d ee 09 47 a3 05 de a6 f1 22 25 6a 0c 81 b2 6b 58 d5 b3 fc fb 28 7f 07 dc 7a 9e d0 37 b4 e1 1a 24 03 ae 94 ba 88 2f ea 2e 8c 5b bb 79 d1 11 ff a4 19 3c 2a 4e 52 e3 95 bd 31 5d 35 4a 41 c4 db 42 c5 0b 49 1b 7c e4 b0 9d 45 f0 a9 61 57 06 d4 40 91 56 13 fa 87 ac 27 54 dd 59 1f 71 39 43 6c f9 be 4f f4 1e 32 cd e9 7e 7b 66 5f ef e7 6f 0a 60 d7 b7 83 92 e2 af 72 f8 b1 50 10 ce 18 53 a1 cc ad 12 34 0f f5 aa 16 e6 f2 d8 85 9f bc */ -const uint8_t __attribute__((aligned(256))) gfcauchy[6][256] = +const uint8_t __attribute__((aligned(256))) raid_gfcauchy[6][256] = { { 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, @@ -9136,7 +9138,7 @@ const uint8_t __attribute__((aligned(256))) gfcauchy[6][256] = * Indexes are [DISK][PARITY - 2][LH]. * Where DISK is from 0 to 250, PARITY from 2 to 5, LH from 0 to 1. */ -const uint8_t __attribute__((aligned(256))) gfcauchypshufb[251][4][2][16] = +const uint8_t __attribute__((aligned(256))) raid_gfcauchypshufb[251][4][2][16] = { { { @@ -13666,7 +13668,7 @@ const uint8_t __attribute__((aligned(256))) gfcauchypshufb[251][4][2][16] = * Indexes are [MULTIPLER][LH]. * Where MULTIPLER is from 0 to 255, LH from 0 to 1. */ -const uint8_t __attribute__((aligned(256))) gfmulpshufb[256][2][16] = +const uint8_t __attribute__((aligned(256))) raid_gfmulpshufb[256][2][16] = { { { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, diff --git a/raid.c b/raid/x86.c similarity index 58% rename from raid.c rename to raid/x86.c index 07f5f32b..29f1922d 100644 --- a/raid.c +++ b/raid/x86.c @@ -15,269 +15,8 @@ * along with this program. If not, see . */ -/* - * Copyright notes: - * - * The RAID5 and RAID6 support was originally derived from the - * H. Peter Anvin paper "The mathematics of RAID-6" [1] and the - * libraid6 library, also by H. Peter Anvin, released with license - * "GPLv2 or any later version" inside the Linux Kernel 2.6.38. - * - * This support was later completely rewritten (many times), but - * the H. Peter Anvin's Copyright may still apply. - * - * The others RAID levels and the recovering based on matrix - * inversion is original work implemented from scratch. - */ - -/* - * The RAID5 and RAID6 support is implemented using the Galois Field - * GF(2^8) with the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 - * (285 decimal). - * - * The parity P and Q of a set of N disk Di with 0<=i=0;--d) { - p0 ^= v_32(v[d][i]); - p1 ^= v_32(v[d][i+4]); - } - v_32(p[i]) = p0; - v_32(p[i+4]) = p1; - } -} - -/* - * PAR1 (RAID5 with xor) 64bit C implementation - */ -void raid_par1_int64(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - int d, l; - size_t i; - - uint64_t p0; - uint64_t p1; - - l = nd - 1; - p = v[nd]; - - for(i=0;i=0;--d) { - p0 ^= v_64(v[d][i]); - p1 ^= v_64(v[d][i+8]); - } - v_64(p[i]) = p0; - v_64(p[i+8]) = p1; - } -} +#include "internal.h" +#include "gf.h" #if defined(__i386__) || defined(__x86_64__) /* @@ -325,147 +64,13 @@ void raid_par1_sse2(int nd, size_t size, void** vv) } #endif -/** - * Multiply each byte of a uint32 by 2 in the GF(2^8). - */ -static __always_inline uint32_t x2_32(uint32_t v) -{ - uint32_t mask = v & 0x80808080U; - mask = (mask << 1) - (mask >> 7); - v = (v << 1) & 0xfefefefeU; - v ^= mask & 0x1d1d1d1dU; - return v; -} - -/** - * Multiply each byte of a uint64 by 2 in the GF(2^8). - */ -static __always_inline uint64_t x2_64(uint64_t v) -{ - uint64_t mask = v & 0x8080808080808080ULL; - mask = (mask << 1) - (mask >> 7); - v = (v << 1) & 0xfefefefefefefefeULL; - v ^= mask & 0x1d1d1d1d1d1d1d1dULL; - return v; -} - -/** - * Divide each byte of a uint32 by 2 in the GF(2^8). - */ -static __always_inline uint32_t d2_32(uint32_t v) -{ - uint32_t mask = v & 0x01010101U; - mask = (mask << 8) - mask; - v = (v >> 1) & 0x7f7f7f7fU; - v ^= mask & 0x8e8e8e8eU; - return v; -} - -/** - * Divide each byte of a uint64 by 2 in the GF(2^8). - */ -static __always_inline uint64_t d2_64(uint64_t v) -{ - uint64_t mask = v & 0x0101010101010101ULL; - mask = (mask << 8) - mask; - v = (v >> 1) & 0x7f7f7f7f7f7f7f7fULL; - v ^= mask & 0x8e8e8e8e8e8e8e8eULL; - return v; -} - -/* - * PAR2 (RAID6 with powers of 2) 32bit C implementation - */ -void raid_par2_int32(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - int d, l; - size_t i; - - uint32_t d0, q0, p0; - uint32_t d1, q1, p1; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - - for(i=0;i=0;--d) { - d0 = v_32(v[d][i]); - d1 = v_32(v[d][i+4]); - - p0 ^= d0; - p1 ^= d1; - - q0 = x2_32(q0); - q1 = x2_32(q1); - - q0 ^= d0; - q1 ^= d1; - } - v_32(p[i]) = p0; - v_32(p[i+4]) = p1; - v_32(q[i]) = q0; - v_32(q[i+4]) = q1; - } -} - -/* - * PAR2 (RAID6 with powers of 2) 64bit C implementation - */ -void raid_par2_int64(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - int d, l; - size_t i; - - uint64_t d0, q0, p0; - uint64_t d1, q1, p1; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - - for(i=0;i=0;--d) { - d0 = v_64(v[d][i]); - d1 = v_64(v[d][i+8]); - - p0 ^= d0; - p1 ^= d1; - - q0 = x2_64(q0); - q1 = x2_64(q1); - - q0 ^= d0; - q1 ^= d1; - } - v_64(p[i]) = p0; - v_64(p[i+8]) = p1; - v_64(q[i]) = q0; - v_64(q[i+8]) = q1; - } -} - #if defined(__i386__) || defined(__x86_64__) static const struct gfconst16 { uint8_t poly[16]; uint8_t low4[16]; - uint8_t half[16]; - uint8_t low7[16]; } gfconst16 __attribute__((aligned(64))) = { { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f }, - { 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e }, - { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f } }; /* @@ -597,291 +202,6 @@ void raid_par2_sse2ext(int nd, size_t size, void** vv) } #endif -/* - * PARz (triple parity with powers of 2^-1) 32bit C implementation - */ -void raid_parz_int32(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - int d, l; - size_t i; - - uint32_t d0, r0, q0, p0; - uint32_t d1, r1, q1, p1; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - - for(i=0;i=0;--d) { - d0 = v_32(v[d][i]); - d1 = v_32(v[d][i+4]); - - p0 ^= d0; - p1 ^= d1; - - q0 = x2_32(q0); - q1 = x2_32(q1); - - q0 ^= d0; - q1 ^= d1; - - r0 = d2_32(r0); - r1 = d2_32(r1); - - r0 ^= d0; - r1 ^= d1; - } - v_32(p[i]) = p0; - v_32(p[i+4]) = p1; - v_32(q[i]) = q0; - v_32(q[i+4]) = q1; - v_32(r[i]) = r0; - v_32(r[i+4]) = r1; - } -} - -/* - * PARz (triple parity with powers of 2^-1) 64bit C implementation - */ -void raid_parz_int64(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - int d, l; - size_t i; - - uint64_t d0, r0, q0, p0; - uint64_t d1, r1, q1, p1; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - - for(i=0;i=0;--d) { - d0 = v_64(v[d][i]); - d1 = v_64(v[d][i+8]); - - p0 ^= d0; - p1 ^= d1; - - q0 = x2_64(q0); - q1 = x2_64(q1); - - q0 ^= d0; - q1 ^= d1; - - r0 = d2_64(r0); - r1 = d2_64(r1); - - r0 ^= d0; - r1 ^= d1; - } - v_64(p[i]) = p0; - v_64(p[i+8]) = p1; - v_64(q[i]) = q0; - v_64(q[i+8]) = q1; - v_64(r[i]) = r0; - v_64(r[i+8]) = r1; - } -} - -#if defined(__i386__) || defined(__x86_64__) -/* - * PARz (triple parity with powers of 2^-1) SSE2 implementation - */ -void raid_parz_sse2(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - - asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); - asm volatile("movdqa %0,%%xmm6" : : "m" (gfconst16.low7[0])); - - for(i=0;i=0;--d) { - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pcmpgtb %xmm1,%xmm4"); - asm volatile("paddb %xmm1,%xmm1"); - asm volatile("pand %xmm7,%xmm4"); - asm volatile("pxor %xmm4,%xmm1"); - - asm volatile("movdqa %xmm2,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("psllw $7,%xmm4"); - asm volatile("psrlw $1,%xmm2"); - asm volatile("pcmpgtb %xmm4,%xmm5"); - asm volatile("pand %xmm6,%xmm2"); - asm volatile("pand %xmm3,%xmm5"); - asm volatile("pxor %xmm5,%xmm2"); - - asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile("pxor %xmm4,%xmm0"); - asm volatile("pxor %xmm4,%xmm1"); - asm volatile("pxor %xmm4,%xmm2"); - } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); - } - - asm volatile("sfence" : : : "memory"); -} -#endif - -#if defined(__x86_64__) -/* - * PARz (triple parity with powers of 2^-1) SSE2 implementation - * - * Note that it uses 16 registers, meaning that x64 is required. - */ -void raid_parz_sse2ext(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - int d, l; - size_t i; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - - asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - asm volatile("movdqa %0,%%xmm3" : : "m" (gfconst16.half[0])); - asm volatile("movdqa %0,%%xmm11" : : "m" (gfconst16.low7[0])); - - for(i=0;i=0;--d) { - asm volatile("movdqa %xmm2,%xmm6"); - asm volatile("movdqa %xmm10,%xmm14"); - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm12,%xmm12"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pxor %xmm13,%xmm13"); - asm volatile("psllw $7,%xmm6"); - asm volatile("psllw $7,%xmm14"); - asm volatile("psrlw $1,%xmm2"); - asm volatile("psrlw $1,%xmm10"); - asm volatile("pcmpgtb %xmm1,%xmm4"); - asm volatile("pcmpgtb %xmm9,%xmm12"); - asm volatile("pcmpgtb %xmm6,%xmm5"); - asm volatile("pcmpgtb %xmm14,%xmm13"); - asm volatile("paddb %xmm1,%xmm1"); - asm volatile("paddb %xmm9,%xmm9"); - asm volatile("pand %xmm11,%xmm2"); - asm volatile("pand %xmm11,%xmm10"); - asm volatile("pand %xmm7,%xmm4"); - asm volatile("pand %xmm7,%xmm12"); - asm volatile("pand %xmm3,%xmm5"); - asm volatile("pand %xmm3,%xmm13"); - asm volatile("pxor %xmm4,%xmm1"); - asm volatile("pxor %xmm12,%xmm9"); - asm volatile("pxor %xmm5,%xmm2"); - asm volatile("pxor %xmm13,%xmm10"); - - asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); - asm volatile("movdqa %0,%%xmm12" : : "m" (v[d][i+16])); - asm volatile("pxor %xmm4,%xmm0"); - asm volatile("pxor %xmm4,%xmm1"); - asm volatile("pxor %xmm4,%xmm2"); - asm volatile("pxor %xmm12,%xmm8"); - asm volatile("pxor %xmm12,%xmm9"); - asm volatile("pxor %xmm12,%xmm10"); - } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[i])); - asm volatile("movntdq %%xmm8,%0" : "=m" (p[i+16])); - asm volatile("movntdq %%xmm1,%0" : "=m" (q[i])); - asm volatile("movntdq %%xmm9,%0" : "=m" (q[i+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); - asm volatile("movntdq %%xmm10,%0" : "=m" (r[i+16])); - } - - asm volatile("sfence" : : : "memory"); -} -#endif - -/* - * PAR3 (triple parity with Cauchy matrix) 8bit C implementation - * - * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. - * But this is only a kind of reference function, and we are not really - * interested in speed. - */ -void raid_par3_int8(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - int d, l; - size_t i; - - uint8_t d0, r0, q0, p0; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - - for(i=0;i0;--d) { - d0 = v_8(v[d][i]); - - p0 ^= d0; - q0 ^= gfmul[d0][gfcauchy[1][d]]; - r0 ^= gfmul[d0][gfcauchy[2][d]]; - } - - /* first disk with all coefficients at 1 */ - d0 = v_8(v[0][i]); - - p0 ^= d0; - q0 ^= d0; - r0 ^= d0; - - v_8(p[i]) = p0; - v_8(q[i]) = q0; - v_8(r[i]) = r0; - } -} - #if defined(__i386__) || defined(__x86_64__) /* * PAR3 (triple parity with Cauchy matrix) SSSE3 implementation @@ -1116,58 +436,6 @@ void raid_par3_ssse3ext(int nd, size_t size, void** vv) } #endif -/* - * PAR4 (quad parity with Cauchy matrix) 8bit C implementation - * - * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. - * But this is only a kind of reference function, and we are not really - * interested in speed. - */ -void raid_par4_int8(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - int d, l; - size_t i; - - uint8_t d0, s0, r0, q0, p0; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - s = v[nd+3]; - - for(i=0;i0;--d) { - d0 = v_8(v[d][i]); - - p0 ^= d0; - q0 ^= gfmul[d0][gfcauchy[1][d]]; - r0 ^= gfmul[d0][gfcauchy[2][d]]; - s0 ^= gfmul[d0][gfcauchy[3][d]]; - } - - /* first disk with all coefficients at 1 */ - d0 = v_8(v[0][i]); - - p0 ^= d0; - q0 ^= d0; - r0 ^= d0; - s0 ^= d0; - - v_8(p[i]) = p0; - v_8(q[i]) = q0; - v_8(r[i]) = r0; - v_8(s[i]) = s0; - } -} - #if defined(__i386__) || defined(__x86_64__) /* * PAR4 (quad parity with Cauchy matrix) SSSE3 implementation @@ -1453,63 +721,6 @@ void raid_par4_ssse3ext(int nd, size_t size, void** vv) } #endif -/* - * PAR5 (penta parity with Cauchy matrix) 8bit C implementation - * - * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. - * But this is only a kind of reference function, and we are not really - * interested in speed. - */ -void raid_par5_int8(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; - int d, l; - size_t i; - - uint8_t d0, t0, s0, r0, q0, p0; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - s = v[nd+3]; - t = v[nd+4]; - - for(i=0;i0;--d) { - d0 = v_8(v[d][i]); - - p0 ^= d0; - q0 ^= gfmul[d0][gfcauchy[1][d]]; - r0 ^= gfmul[d0][gfcauchy[2][d]]; - s0 ^= gfmul[d0][gfcauchy[3][d]]; - t0 ^= gfmul[d0][gfcauchy[4][d]]; - } - - /* first disk with all coefficients at 1 */ - d0 = v_8(v[0][i]); - - p0 ^= d0; - q0 ^= d0; - r0 ^= d0; - s0 ^= d0; - t0 ^= d0; - - v_8(p[i]) = p0; - v_8(q[i]) = q0; - v_8(r[i]) = r0; - v_8(s[i]) = s0; - v_8(t[i]) = t0; - } -} - #if defined(__i386__) || defined(__x86_64__) /* * PAR5 (penta parity with Cauchy matrix) SSSE3 implementation @@ -1780,68 +991,6 @@ void raid_par5_ssse3ext(int nd, size_t size, void** vv) } #endif -/* - * PAR6 (hexa parity with Cauchy matrix) 8bit C implementation - * - * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. - * But this is only a kind of reference function, and we are not really - * interested in speed. - */ -void raid_par6_int8(int nd, size_t size, void** vv) -{ - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; - uint8_t* u; - int d, l; - size_t i; - - uint8_t d0, u0, t0, s0, r0, q0, p0; - - l = nd - 1; - p = v[nd]; - q = v[nd+1]; - r = v[nd+2]; - s = v[nd+3]; - t = v[nd+4]; - u = v[nd+5]; - - for(i=0;i0;--d) { - d0 = v_8(v[d][i]); - - p0 ^= d0; - q0 ^= gfmul[d0][gfcauchy[1][d]]; - r0 ^= gfmul[d0][gfcauchy[2][d]]; - s0 ^= gfmul[d0][gfcauchy[3][d]]; - t0 ^= gfmul[d0][gfcauchy[4][d]]; - u0 ^= gfmul[d0][gfcauchy[5][d]]; - } - - /* first disk with all coefficients at 1 */ - d0 = v_8(v[0][i]); - - p0 ^= d0; - q0 ^= d0; - r0 ^= d0; - s0 ^= d0; - t0 ^= d0; - u0 ^= d0; - - v_8(p[i]) = p0; - v_8(q[i]) = q0; - v_8(r[i]) = r0; - v_8(s[i]) = s0; - v_8(t[i]) = t0; - v_8(u[i]) = u0; - } -} - #if defined(__i386__) || defined(__x86_64__) /* * PAR6 (hexa parity with Cauchy matrix) SSSE3 implementation @@ -2150,469 +1299,6 @@ void raid_par6_ssse3ext(int nd, size_t size, void** vv) } #endif -/* internal forwarder */ -static void (*raid_par_ptr[RAID_PARITY_MAX])(int nd, size_t size, void** vv); -static void (*raid_par3_ptr)(int nd, size_t size, void** vv); -static void (*raid_parz_ptr)(int nd, size_t size, void** vv); - -void raid_par(int np, int nd, size_t size, void** v) -{ - bug_on(np < 1 || np > RAID_PARITY_MAX); - - raid_par_ptr[np - 1](nd, size, v); -} - -/****************************************************************************/ -/* mode */ - -/** - * Generator matrix currently used. - */ -static const uint8_t (*raid_gen)[256]; - -void raid_mode(int mode) -{ - if (mode == RAID_MODE_VANDERMONDE) { - raid_par_ptr[2] = raid_parz_ptr; - raid_gen = gfvandermonde; - } else { - raid_par_ptr[2] = raid_par3_ptr; - raid_gen = gfcauchy; - } -} - -/****************************************************************************/ -/* zero */ - -/** - * Buffer filled with 0 used in recovering. - */ -static void* raid_empty_zero_page; - -void raid_zero(void* zero) -{ - raid_empty_zero_page = zero; -} - -/****************************************************************************/ -/* recovering */ - -/** - * GF a*b. - */ -static __always_inline uint8_t mul(uint8_t a, uint8_t b) -{ - return gfmul[a][b]; -} - -/** - * GF 1/a. - * Not defined for a == 0. - */ -static __always_inline uint8_t inv(uint8_t v) -{ - bug_on(v == 0); /* division by zero */ - - return gfinv[v]; -} - -/** - * GF 2^a. - */ -static __always_inline uint8_t pow2(int v) -{ - bug_on(v < 0 || v > 254); /* invalid exponent */ - - return gfexp[v]; -} - -/** - * Gets the multiplication table for a specified value. - */ -static __always_inline const uint8_t* table(uint8_t v) -{ - return gfmul[v]; -} - -/** - * Computes the parity without the missing data blocks - * and store it in the buffers of such data blocks. - * - * This is the parity expressed as Pa,Qa,Ra,Sa,Ta,Ua - * in the equations. - * - * Note that all the other parities not in the ip[] vector - * are destroyed. - */ -static void raid_delta_gen(int nr, const int* id, const int* ip, int nd, size_t size, void** v) -{ - void* p[RAID_PARITY_MAX]; - void* pa[RAID_PARITY_MAX]; - int i; - - for(i=0;i nd); - - /* if failed data is present */ - if (nr != 0) { - raid_rec_ptr[nr - 1](nr, id, ip, nd, size, v); - } -} - -void raid_recpar(int nrd, const int* id, int nrp, int* ip, int np, int nd, size_t size, void** v) -{ - bug_on(nrd > nd); - bug_on(nrd + nrp > np); - - /* if failed data is present */ - if (nrd != 0) { - int iu[RAID_PARITY_MAX]; - int i, j, k; - - /* setup the vector of parities to use */ - for(i=0,j=0,k=0;i<(int)np;++i) { - if (ip[j] == i) { - ++j; - } else { - iu[k] = i; - ++k; - } - } - - /* recover the data */ - raid_rec_ptr[nrd - 1](nrd, id, iu, nd, size, v); - } - - /* recompute all the parities up to the last bad one */ - if (nrp != 0) { - raid_par(ip[nrp - 1] + 1, nd, size, v); - } -} - -/****************************************************************************/ -/* init/done */ - -void raid_init(void) -{ - raid_par3_ptr = raid_par3_int8; - raid_par_ptr[3] = raid_par4_int8; - raid_par_ptr[4] = raid_par5_int8; - raid_par_ptr[5] = raid_par6_int8; - - if (sizeof(void*) == 4) { - raid_par_ptr[0] = raid_par1_int32; - raid_par_ptr[1] = raid_par2_int32; - raid_parz_ptr = raid_parz_int32; - } else { - raid_par_ptr[0] = raid_par1_int64; - raid_par_ptr[1] = raid_par2_int64; - raid_parz_ptr = raid_parz_int64; - } - - raid_rec_ptr[0] = raid_rec1_int8; - raid_rec_ptr[1] = raid_rec2_int8; - raid_rec_ptr[2] = raid_recX_int8; - raid_rec_ptr[3] = raid_recX_int8; - raid_rec_ptr[4] = raid_recX_int8; - raid_rec_ptr[5] = raid_recX_int8; - -#if defined(__i386__) || defined(__x86_64__) - if (raid_cpu_has_sse2()) { - raid_par_ptr[0] = raid_par1_sse2; -#if defined(__x86_64__) - if (raid_cpu_has_slowextendedreg()) { - raid_par_ptr[1] = raid_par2_sse2; - raid_parz_ptr = raid_parz_sse2; - } else { - raid_par_ptr[1] = raid_par2_sse2ext; - raid_parz_ptr = raid_parz_sse2ext; - } -#else - raid_par_ptr[1] = raid_par2_sse2; - raid_parz_ptr = raid_parz_sse2; -#endif - } - - if (raid_cpu_has_ssse3()) { -#if defined(__x86_64__) - raid_par3_ptr = raid_par3_ssse3ext; - raid_par_ptr[3] = raid_par4_ssse3ext; - raid_par_ptr[4] = raid_par5_ssse3ext; - raid_par_ptr[5] = raid_par6_ssse3ext; -#else - raid_par3_ptr = raid_par3_ssse3; - raid_par_ptr[3] = raid_par4_ssse3; - raid_par_ptr[4] = raid_par5_ssse3; - raid_par_ptr[5] = raid_par6_ssse3; -#endif - raid_rec_ptr[0] = raid_rec1_ssse3; - raid_rec_ptr[1] = raid_rec2_ssse3; - raid_rec_ptr[2] = raid_recX_ssse3; - raid_rec_ptr[3] = raid_recX_ssse3; - raid_rec_ptr[4] = raid_recX_ssse3; - raid_rec_ptr[5] = raid_recX_ssse3; - } -#endif - - /* set the default mode */ - raid_mode(RAID_MODE_CAUCHY); -} - -static struct raid_func { - const char* name; - void* p; -} RAID_FUNC[] = { - { "int8", raid_par3_int8 }, - { "int8", raid_par4_int8 }, - { "int8", raid_par5_int8 }, - { "int8", raid_par6_int8 }, - { "int32", raid_par1_int32 }, - { "int64", raid_par1_int64 }, - { "int32", raid_par2_int32 }, - { "int64", raid_par2_int64 }, - { "int32", raid_parz_int32 }, - { "int64", raid_parz_int64 }, - { "int8", raid_rec1_int8 }, - { "int8", raid_rec2_int8 }, - { "int8", raid_recX_int8 }, - -#if defined(__i386__) || defined(__x86_64__) - { "sse2", raid_par1_sse2 }, - { "sse2", raid_par2_sse2 }, - { "sse2", raid_parz_sse2 }, - { "ssse3", raid_par3_ssse3 }, - { "ssse3", raid_par4_ssse3 }, - { "ssse3", raid_par5_ssse3 }, - { "ssse3", raid_par6_ssse3 }, - { "ssse3", raid_rec1_ssse3 }, - { "ssse3", raid_rec2_ssse3 }, - { "ssse3", raid_recX_ssse3 }, -#endif - -#if defined(__x86_64__) - { "sse2e", raid_par2_sse2ext }, - { "sse2e", raid_parz_sse2ext }, - { "ssse3e", raid_par3_ssse3ext }, - { "ssse3e", raid_par4_ssse3ext }, - { "ssse3e", raid_par5_ssse3ext }, - { "ssse3e", raid_par6_ssse3ext }, -#endif - { 0, 0 } -}; - -static const char* raid_tag(void* func) -{ - struct raid_func* i = RAID_FUNC; - while (i->name != 0) { - if (i->p == func) - return i->name; - ++i; - } - return "unknown"; -} - -const char* raid_par1_tag(void) -{ - return raid_tag(raid_par_ptr[0]); -} - -const char* raid_par2_tag(void) -{ - return raid_tag(raid_par_ptr[1]); -} - -const char* raid_parz_tag(void) -{ - return raid_tag(raid_parz_ptr); -} - -const char* raid_par3_tag(void) -{ - return raid_tag(raid_par_ptr[2]); -} - -const char* raid_par4_tag(void) -{ - return raid_tag(raid_par_ptr[3]); -} - -const char* raid_par5_tag(void) -{ - return raid_tag(raid_par_ptr[4]); -} - -const char* raid_par6_tag(void) -{ - return raid_tag(raid_par_ptr[5]); -} - -const char* raid_rec1_tag(void) -{ - return raid_tag(raid_rec_ptr[0]); -} - -const char* raid_rec2_tag(void) -{ - return raid_tag(raid_rec_ptr[1]); -} - -const char* raid_recX_tag(void) -{ - return raid_tag(raid_rec_ptr[2]); -} - diff --git a/raid/x86z.c b/raid/x86z.c new file mode 100644 index 00000000..bb96eb60 --- /dev/null +++ b/raid/x86z.c @@ -0,0 +1,165 @@ +/* + * Copyright (C) 2013 Andrea Mazzoleni + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +#include "internal.h" + +#if defined(__i386__) || defined(__x86_64__) +static const struct gfzconst16 { + uint8_t poly[16]; + uint8_t half[16]; + uint8_t low7[16]; +} gfzconst16 __attribute__((aligned(64))) = { + { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, + { 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e }, + { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f } +}; + +/* + * PARz (triple parity with powers of 2^-1) SSE2 implementation + */ +void raid_parz_sse2(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + + asm volatile("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); + asm volatile("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (gfzconst16.low7[0])); + + for(i=0;i=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pxor %xmm4,%xmm1"); + + asm volatile("movdqa %xmm2,%xmm4"); + asm volatile("pxor %xmm5,%xmm5"); + asm volatile("psllw $7,%xmm4"); + asm volatile("psrlw $1,%xmm2"); + asm volatile("pcmpgtb %xmm4,%xmm5"); + asm volatile("pand %xmm6,%xmm2"); + asm volatile("pand %xmm3,%xmm5"); + asm volatile("pxor %xmm5,%xmm2"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); + } + + asm volatile("sfence" : : : "memory"); +} +#endif + +#if defined(__x86_64__) +/* + * PARz (triple parity with powers of 2^-1) SSE2 implementation + * + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raid_parz_sse2ext(int nd, size_t size, void** vv) +{ + uint8_t** v = (uint8_t**)vv; + uint8_t* p; + uint8_t* q; + uint8_t* r; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + r = v[nd+2]; + + asm volatile("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); + asm volatile("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); + asm volatile("movdqa %0,%%xmm11" : : "m" (gfzconst16.low7[0])); + + for(i=0;i=0;--d) { + asm volatile("movdqa %xmm2,%xmm6"); + asm volatile("movdqa %xmm10,%xmm14"); + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm12,%xmm12"); + asm volatile("pxor %xmm5,%xmm5"); + asm volatile("pxor %xmm13,%xmm13"); + asm volatile("psllw $7,%xmm6"); + asm volatile("psllw $7,%xmm14"); + asm volatile("psrlw $1,%xmm2"); + asm volatile("psrlw $1,%xmm10"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("pcmpgtb %xmm9,%xmm12"); + asm volatile("pcmpgtb %xmm6,%xmm5"); + asm volatile("pcmpgtb %xmm14,%xmm13"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("paddb %xmm9,%xmm9"); + asm volatile("pand %xmm11,%xmm2"); + asm volatile("pand %xmm11,%xmm10"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pand %xmm7,%xmm12"); + asm volatile("pand %xmm3,%xmm5"); + asm volatile("pand %xmm3,%xmm13"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm12,%xmm9"); + asm volatile("pxor %xmm5,%xmm2"); + asm volatile("pxor %xmm13,%xmm10"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); + asm volatile("movdqa %0,%%xmm12" : : "m" (v[d][i+16])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm12,%xmm8"); + asm volatile("pxor %xmm12,%xmm9"); + asm volatile("pxor %xmm12,%xmm10"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[i])); + asm volatile("movntdq %%xmm8,%0" : "=m" (p[i+16])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[i])); + asm volatile("movntdq %%xmm9,%0" : "=m" (q[i+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); + asm volatile("movntdq %%xmm10,%0" : "=m" (r[i+16])); + } + + asm volatile("sfence" : : : "memory"); +} +#endif + diff --git a/rehash.c b/rehash.c index 261d7e2b..199d19cc 100644 --- a/rehash.c +++ b/rehash.c @@ -23,7 +23,7 @@ #include "state.h" #include "parity.h" #include "handle.h" -#include "raid.h" +#include "raid/raid.h" /****************************************************************************/ /* rehash */ diff --git a/scrub.c b/scrub.c index 646dae39..b067d9a7 100644 --- a/scrub.c +++ b/scrub.c @@ -22,7 +22,7 @@ #include "state.h" #include "parity.h" #include "handle.h" -#include "raid.h" +#include "raid/raid.h" /****************************************************************************/ /* scrub */ diff --git a/selftest.c b/selftest.c index aea71089..d509fd60 100644 --- a/selftest.c +++ b/selftest.c @@ -19,8 +19,9 @@ #include "snapraid.h" #include "util.h" -#include "raid.h" -#include "cpu.h" +#include "raid/raid.h" +#include "raid/cpu.h" +#include "raid/internal.h" #include "elem.h" #include "state.h" #include "combo.h" diff --git a/snapraid.c b/snapraid.c index 8d3f3ca0..e3174086 100644 --- a/snapraid.c +++ b/snapraid.c @@ -22,7 +22,7 @@ #include "elem.h" #include "import.h" #include "state.h" -#include "raid.h" +#include "raid/raid.h" /****************************************************************************/ /* main */ diff --git a/speed.c b/speed.c index 1faf73fd..839ae53e 100644 --- a/speed.c +++ b/speed.c @@ -19,8 +19,9 @@ #include "snapraid.h" #include "util.h" -#include "raid.h" -#include "cpu.h" +#include "raid/raid.h" +#include "raid/cpu.h" +#include "raid/internal.h" #include "state.h" /** diff --git a/state.c b/state.c index 89d327d7..dee75b03 100644 --- a/state.c +++ b/state.c @@ -22,8 +22,8 @@ #include "state.h" #include "util.h" #include "parity.h" -#include "cpu.h" -#include "raid.h" +#include "raid/raid.h" +#include "raid/cpu.h" const char* lev_name(unsigned l) { diff --git a/status.c b/status.c index 6d65ddcc..06ec1f71 100644 --- a/status.c +++ b/status.c @@ -22,7 +22,7 @@ #include "state.h" #include "parity.h" #include "handle.h" -#include "raid.h" +#include "raid/raid.h" /****************************************************************************/ /* status */ diff --git a/sync.c b/sync.c index 9634326f..32e32151 100644 --- a/sync.c +++ b/sync.c @@ -22,7 +22,7 @@ #include "state.h" #include "parity.h" #include "handle.h" -#include "raid.h" +#include "raid/raid.h" /****************************************************************************/ /* sync */ diff --git a/util.c b/util.c index f0efa796..00e08ae3 100644 --- a/util.c +++ b/util.c @@ -18,7 +18,7 @@ #include "portable.h" #include "util.h" -#include "cpu.h" +#include "raid/cpu.h" /****************************************************************************/ /* hex conversion table */