diff --git a/check.c b/check.c index a2797b0a..fa0a5683 100644 --- a/check.c +++ b/check.c @@ -228,7 +228,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO1[i].a], buffer_recov[COMBO1[i].a], state->block_size); /* recover data */ - raid_recov_1data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, COMBO1[i].a, buffer_zero); + raid_recov_1data(failed[failed_map[0]].index, COMBO1[i].a, buffer, diskmax, buffer_zero, state->block_size); if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero)) return 0; @@ -249,7 +249,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO2[i].a], buffer_recov[COMBO2[i].a], state->block_size); /* recover data */ - raid_recov_1data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, COMBO2[i].a, buffer_zero); + raid_recov_1data(failed[failed_map[0]].index, COMBO2[i].a, buffer, diskmax, buffer_zero, state->block_size); if (is_parity_matching(state, diskmax, COMBO2[i].b, buffer, buffer_recov)) return 0; @@ -271,7 +271,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO2[i].b], buffer_recov[COMBO2[i].b], state->block_size); /* recover data */ - raid_recov_2data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO2[i].a, COMBO2[i].b, buffer_zero); + raid_recov_2data(failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO2[i].a, COMBO2[i].b, buffer, diskmax, buffer_zero, state->block_size); if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero)) return 0; @@ -293,7 +293,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO3[i].b], buffer_recov[COMBO3[i].b], state->block_size); /* recover data */ - raid_recov_2data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO3[i].a, COMBO3[i].b, buffer_zero); + raid_recov_2data(failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO3[i].a, COMBO3[i].b, buffer, diskmax, buffer_zero, state->block_size); if (is_parity_matching(state, diskmax, COMBO3[i].c, buffer, buffer_recov)) return 0; @@ -316,7 +316,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO3[i].c], buffer_recov[COMBO3[i].c], state->block_size); /* recover data */ - raid_recov_3data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO3[i].a, COMBO3[i].b, COMBO3[i].c, buffer_zero); + raid_recov_3data(failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO3[i].a, COMBO3[i].b, COMBO3[i].c, buffer, diskmax, buffer_zero, state->block_size); if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero)) return 0; @@ -339,7 +339,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO4[i].c], buffer_recov[COMBO4[i].c], state->block_size); /* recover data */ - raid_recov_3data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, buffer_zero); + raid_recov_3data(failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, buffer, diskmax, buffer_zero, state->block_size); if (is_parity_matching(state, diskmax, COMBO4[i].d, buffer, buffer_recov)) return 0; @@ -363,7 +363,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u memcpy(buffer[diskmax+COMBO4[i].d], buffer_recov[COMBO4[i].d], state->block_size); /* recover data */ - raid_recov_4data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, failed[failed_map[3]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, COMBO4[i].d, buffer_zero); + raid_recov_4data(failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, failed[failed_map[3]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, COMBO4[i].d, buffer, diskmax, buffer_zero, state->block_size); if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero)) return 0; diff --git a/raid.c b/raid.c index 02f159d9..25c6fc99 100644 --- a/raid.c +++ b/raid.c @@ -15,17 +15,18 @@ * along with this program. If not, see . */ -/* -*- linux-c -*- ------------------------------------------------------- * +/* + * The RAID5/RAID6 support was originally derived by the libraid6 library + * by H. Peter Anvin released with license "GPL2 or any later version" + * used by the Linux Kernel 2.6.38. * - * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved + * This support was later completely rewritten (many times), but the + * H. Peter Anvin's Copyright may still apply. * - * This program is free software; you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, Inc., 53 Temple Place Ste 330, - * Boston MA 02111-1307, USA; either version 2 of the License, or - * (at your option) any later version; incorporated herein by reference. - * - * ----------------------------------------------------------------------- */ + * The RAIDTP/RAIDQP support is original work implemented from scratch. + * For some notes about the method used, see the "recovering" section + * in this file. + */ #include "portable.h" @@ -34,159 +35,140 @@ #include "tables.h" /****************************************************************************/ -/* internal computation */ +/* specialized parity generation */ + +/** + * Dereference as uint32_t + */ +#define v_32(p) (*(uint32_t*)&(p)) + +/** + * Dereference as uint64_t + */ +#define v_64(p) (*(uint64_t*)&(p)) /* * RAID5 unrolled-by-2 C implementation */ -void raid5_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid5_int32r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; - int z, z0; - unsigned d; + int d, l; + unsigned o; - uint32_t wp0; - uint32_t wp1; + uint32_t p0; + uint32_t p1; - z0 = diskmax - 1; - p = buffer[diskmax]; + l = data - 1; + p = vbuf[data]; - for(d=0;d=0;--z) { - wp0 ^= *(uint32_t *)&buffer[z][d+0*4]; - wp1 ^= *(uint32_t *)&buffer[z][d+1*4]; + for(o=0;o=0;--d) { + p0 ^= v_32(vbuf[d][o]); + p1 ^= v_32(vbuf[d][o+4]); } - *(uint32_t *)&p[d+4*0] = wp0; - *(uint32_t *)&p[d+4*1] = wp1; + v_32(p[o]) = p0; + v_32(p[o+4]) = p1; + } +} + +/* + * RAID5 unrolled-by-2 C implementation + */ +void raid5_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + int d, l; + unsigned o; + + uint64_t p0; + uint64_t p1; + + l = data - 1; + p = vbuf[data]; + + for(o=0;o=0;--d) { + p0 ^= v_64(vbuf[d][o]); + p1 ^= v_64(vbuf[d][o+8]); + } + v_64(p[o]) = p0; + v_64(p[o+8]) = p1; } } #if defined(__i386__) || defined(__x86_64__) -/* - * RAID5 unrolled-by-2 MMX implementation - */ -void raid5_mmxr2(unsigned char** buffer, unsigned diskmax, unsigned size) -{ - unsigned char* p; - int z, z0; - unsigned d; - - z0 = diskmax - 1; - p = buffer[diskmax]; - - for(d=0;d=0;--z) { - asm volatile("movq %0,%%mm4" : : "m" (buffer[z][d])); - asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d+8])); - asm volatile("pxor %mm4,%mm0"); - asm volatile("pxor %mm5,%mm1"); - } - asm volatile("movq %%mm0,%0" : "=m" (p[d])); - asm volatile("movq %%mm1,%0" : "=m" (p[d+8])); - } - - asm volatile("emms" : : : "memory"); -} - /* * RAID5 unrolled-by-4 MMX implementation */ -void raid5_mmxr4(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid5_mmxr4(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; + l = data - 1; + p = vbuf[data]; - for(d=0;d=0;--z) { - asm volatile("movq %0,%%mm4" : : "m" (buffer[z][d])); - asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d+8])); - asm volatile("movq %0,%%mm6" : : "m" (buffer[z][d+16])); - asm volatile("movq %0,%%mm7" : : "m" (buffer[z][d+24])); + for(o=0;o=0;--d) { + asm volatile("movq %0,%%mm4" : : "m" (vbuf[d][o])); + asm volatile("movq %0,%%mm5" : : "m" (vbuf[d][o+8])); + asm volatile("movq %0,%%mm6" : : "m" (vbuf[d][o+16])); + asm volatile("movq %0,%%mm7" : : "m" (vbuf[d][o+24])); asm volatile("pxor %mm4,%mm0"); asm volatile("pxor %mm5,%mm1"); asm volatile("pxor %mm6,%mm2"); asm volatile("pxor %mm7,%mm3"); } - asm volatile("movq %%mm0,%0" : "=m" (p[d])); - asm volatile("movq %%mm1,%0" : "=m" (p[d+8])); - asm volatile("movq %%mm2,%0" : "=m" (p[d+16])); - asm volatile("movq %%mm3,%0" : "=m" (p[d+24])); + asm volatile("movq %%mm0,%0" : "=m" (p[o])); + asm volatile("movq %%mm1,%0" : "=m" (p[o+8])); + asm volatile("movq %%mm2,%0" : "=m" (p[o+16])); + asm volatile("movq %%mm3,%0" : "=m" (p[o+24])); } asm volatile("emms" : : : "memory"); } -/* - * RAID5 unrolled-by-2 SSE2 implementation - */ -void raid5_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) -{ - unsigned char* p; - int z, z0; - unsigned d; - - z0 = diskmax - 1; - p = buffer[diskmax]; - - for(d=0;d=0;--z) { - asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16])); - asm volatile("pxor %xmm4,%xmm0"); - asm volatile("pxor %xmm5,%xmm1"); - } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); - } - - asm volatile("sfence" : : : "memory"); -} - /* * RAID5 unrolled-by-4 SSE2 implementation */ -void raid5_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid5_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; + l = data - 1; + p = vbuf[data]; - for(d=0;d=0;--z) { - asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16])); - asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d+32])); - asm volatile("movdqa %0,%%xmm7" : : "m" (buffer[z][d+48])); + for(o=0;o=0;--d) { + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm5" : : "m" (vbuf[d][o+16])); + asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o+32])); + asm volatile("movdqa %0,%%xmm7" : : "m" (vbuf[d][o+48])); asm volatile("pxor %xmm4,%xmm0"); asm volatile("pxor %xmm5,%xmm1"); asm volatile("pxor %xmm6,%xmm2"); asm volatile("pxor %xmm7,%xmm3"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32])); - asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (p[o+32])); + asm volatile("movntdq %%xmm3,%0" : "=m" (p[o+48])); } asm volatile("sfence" : : : "memory"); @@ -196,35 +178,35 @@ void raid5_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) #if defined(__x86_64__) /* * RAID5 unrolled-by-8 SSE2 implementation - * Note that it uses all the 16 registers, meaning that x64 is required. + * Note that it uses 16 registers, meaning that x64 is required. */ -void raid5_sse2r8(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid5_sse2r8(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; + l = data - 1; + p = vbuf[data]; - for(d=0;d=0;--z) { - asm volatile("movdqa %0,%%xmm8" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm9" : : "m" (buffer[z][d+16])); - asm volatile("movdqa %0,%%xmm10" : : "m" (buffer[z][d+32])); - asm volatile("movdqa %0,%%xmm11" : : "m" (buffer[z][d+48])); - asm volatile("movdqa %0,%%xmm12" : : "m" (buffer[z][d+64])); - asm volatile("movdqa %0,%%xmm13" : : "m" (buffer[z][d+80])); - asm volatile("movdqa %0,%%xmm14" : : "m" (buffer[z][d+96])); - asm volatile("movdqa %0,%%xmm15" : : "m" (buffer[z][d+112])); + for(o=0;o=0;--d) { + asm volatile("movdqa %0,%%xmm8" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm9" : : "m" (vbuf[d][o+16])); + asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[d][o+32])); + asm volatile("movdqa %0,%%xmm11" : : "m" (vbuf[d][o+48])); + asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][o+64])); + asm volatile("movdqa %0,%%xmm13" : : "m" (vbuf[d][o+80])); + asm volatile("movdqa %0,%%xmm14" : : "m" (vbuf[d][o+96])); + asm volatile("movdqa %0,%%xmm15" : : "m" (vbuf[d][o+112])); asm volatile("pxor %xmm8,%xmm0"); asm volatile("pxor %xmm9,%xmm1"); asm volatile("pxor %xmm10,%xmm2"); @@ -234,185 +216,213 @@ void raid5_sse2r8(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm14,%xmm6"); asm volatile("pxor %xmm15,%xmm7"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32])); - asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48])); - asm volatile("movntdq %%xmm4,%0" : "=m" (p[d+64])); - asm volatile("movntdq %%xmm5,%0" : "=m" (p[d+80])); - asm volatile("movntdq %%xmm6,%0" : "=m" (p[d+96])); - asm volatile("movntdq %%xmm7,%0" : "=m" (p[d+112])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (p[o+32])); + asm volatile("movntdq %%xmm3,%0" : "=m" (p[o+48])); + asm volatile("movntdq %%xmm4,%0" : "=m" (p[o+64])); + asm volatile("movntdq %%xmm5,%0" : "=m" (p[o+80])); + asm volatile("movntdq %%xmm6,%0" : "=m" (p[o+96])); + asm volatile("movntdq %%xmm7,%0" : "=m" (p[o+112])); } asm volatile("sfence" : : : "memory"); } #endif -/* - * Repeats an 8 bit nibble to a full 32 bits value. +/** + * Multiply each byte of a uint32 by 2 in the GF(2^8). */ -#define NBYTES(x) ((x) * 0x01010101U) - -/* - * These sub-operations are separate inlines since they can sometimes be - * specially optimized using architecture-specific hacks. - */ - -/* - * The SHLBYTE() operation shifts each byte left by 1, *not* - * rolling over into the next byte. - */ -static inline uint32_t SHLBYTE(uint32_t v) +__attribute__((always_inline)) static inline uint32_t x2_32(uint32_t v) { - uint32_t vv; - - vv = (v << 1) & NBYTES(0xfe); - return vv; + uint32_t mask = v & 0x80808080U; + mask = (mask << 1) - (mask >> 7); + v = (v << 1) & 0xfefefefeU; + v ^= mask & 0x1d1d1d1dU; + return v; } -/* - * The MASK() operation returns 0xFF in any byte for which the high - * bit is 1, 0x00 for any byte for which the high bit is 0. +/** + * Multiply each byte of a uint64 by 2 in the GF(2^8). */ -static inline uint32_t MASK(uint32_t v) +__attribute__((always_inline)) static inline uint64_t x2_64(uint64_t v) { - uint32_t vv; - - vv = v & NBYTES(0x80); - vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */ - return vv; + uint64_t mask = v & 0x8080808080808080ULL; + mask = (mask << 1) - (mask >> 7); + v = (v << 1) & 0xfefefefefefefefeULL; + v ^= mask & 0x1d1d1d1d1d1d1d1dULL; + return v; } /* * RAID6 unrolled-by-2 C implementation */ -void raid6_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid6_int32r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; - int z, z0; - unsigned d; + int d, l; + unsigned o; - uint32_t wd0, wq0, wp0, w10, w20; - uint32_t wd1, wq1, wp1, w11, w21; + uint32_t d0, q0, p0; + uint32_t d1, q1, p1; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; - for(d=0;d= 0;--z) { - wd0 = *(uint32_t *)&buffer[z][d+0*4]; - wd1 = *(uint32_t *)&buffer[z][d+1*4]; - wp0 ^= wd0; - wp1 ^= wd1; - w20 = MASK(wq0); - w21 = MASK(wq1); - w10 = SHLBYTE(wq0); - w11 = SHLBYTE(wq1); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; - wq0 = w10 ^ wd0; - wq1 = w11 ^ wd1; + for(o=0;o=0;--d) { + d0 = v_32(vbuf[d][o]); + d1 = v_32(vbuf[d][o+4]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_32(q0); + q1 = x2_32(q1); + + q0 ^= d0; + q1 ^= d1; } - *(uint32_t *)&p[d+4*0] = wp0; - *(uint32_t *)&p[d+4*1] = wp1; - *(uint32_t *)&q[d+4*0] = wq0; - *(uint32_t *)&q[d+4*1] = wq1; + v_32(p[o]) = p0; + v_32(p[o+4]) = p1; + v_32(q[o]) = q0; + v_32(q[o+4]) = q1; + } +} + +/* + * RAID6 unrolled-by-2 C implementation + */ +void raid6_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + int d, l; + unsigned o; + + uint64_t d0, q0, p0; + uint64_t d1, q1, p1; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + + for(o=0;o=0;--d) { + d0 = v_64(vbuf[d][o]); + d1 = v_64(vbuf[d][o+8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0); + q1 = x2_64(q1); + + q0 ^= d0; + q1 ^= d1; + } + v_64(p[o]) = p0; + v_64(p[o+8]) = p1; + v_64(q[o]) = q0; + v_64(q[o+8]) = q1; } } #if defined(__i386__) || defined(__x86_64__) -static const struct raid6_mmx_constants { - uint64_t x1d; -} raid6_mmx_constants = { +static const struct raid_mmx_poly { + uint64_t poly; +} raid_mmx_poly = { 0x1d1d1d1d1d1d1d1dULL, }; /* * RAID6 unrolled-by-2 MMX implementation */ -void raid6_mmxr2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; - asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d)); + asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly)); + asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); - asm volatile("pxor %mm7,%mm7"); - for(d=0;d=0;--z) { - asm volatile("pcmpgtb %mm4,%mm5"); - asm volatile("pcmpgtb %mm6,%mm7"); - asm volatile("paddb %mm4,%mm4"); - asm volatile("paddb %mm6,%mm6"); - asm volatile("pand %mm0,%mm5"); - asm volatile("pand %mm0,%mm7"); - asm volatile("pxor %mm5,%mm4"); - asm volatile("pxor %mm7,%mm6"); - asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d])); - asm volatile("movq %0,%%mm7" : : "m" (buffer[z][d+8])); - asm volatile("pxor %mm5,%mm2"); - asm volatile("pxor %mm7,%mm3"); - asm volatile("pxor %mm5,%mm4"); - asm volatile("pxor %mm7,%mm6"); + for(o=0;o=0;--d) { + asm volatile("pcmpgtb %mm2,%mm4"); + asm volatile("pcmpgtb %mm3,%mm5"); + asm volatile("paddb %mm2,%mm2"); + asm volatile("paddb %mm3,%mm3"); + asm volatile("pand %mm7,%mm4"); + asm volatile("pand %mm7,%mm5"); + asm volatile("pxor %mm4,%mm2"); + asm volatile("pxor %mm5,%mm3"); + + asm volatile("movq %0,%%mm4" : : "m" (vbuf[d][o])); + asm volatile("movq %0,%%mm5" : : "m" (vbuf[d][o+8])); + asm volatile("pxor %mm4,%mm0"); + asm volatile("pxor %mm5,%mm1"); + asm volatile("pxor %mm4,%mm2"); + asm volatile("pxor %mm5,%mm3"); + + asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); - asm volatile("pxor %mm7,%mm7"); } - asm volatile("movq %%mm2,%0" : "=m" (p[d])); - asm volatile("movq %%mm3,%0" : "=m" (p[d+8])); - asm volatile("movq %%mm4,%0" : "=m" (q[d])); - asm volatile("movq %%mm6,%0" : "=m" (q[d+8])); + asm volatile("movq %%mm0,%0" : "=m" (p[o])); + asm volatile("movq %%mm1,%0" : "=m" (p[o+8])); + asm volatile("movq %%mm2,%0" : "=m" (q[o])); + asm volatile("movq %%mm3,%0" : "=m" (q[o+8])); } asm volatile("emms" : : : "memory"); } -static const struct raid6_sse_constants { - uint64_t x1d[2]; -} raid6_sse_constants __attribute__((aligned(16))) = { +static const struct raid_sse_poly { + uint64_t poly[2]; +} raid_sse_poly __attribute__((aligned(16))) = { { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, }; /* * RAID6 unrolled-by-2 SSE2 implementation */ -void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %xmm2,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("paddb %xmm2,%xmm2"); @@ -421,19 +431,21 @@ void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pand %xmm7,%xmm5"); asm volatile("pxor %xmm4,%xmm2"); asm volatile("pxor %xmm5,%xmm3"); - asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16])); + + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm5" : : "m" (vbuf[d][o+16])); asm volatile("pxor %xmm4,%xmm0"); asm volatile("pxor %xmm5,%xmm1"); asm volatile("pxor %xmm4,%xmm2"); asm volatile("pxor %xmm5,%xmm3"); + asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (q[d])); - asm volatile("movntdq %%xmm3,%0" : "=m" (q[d+16])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm3,%0" : "=m" (q[o+16])); } asm volatile("sfence" : : : "memory"); @@ -443,35 +455,35 @@ void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) #if defined(__x86_64__) /* * RAID6 unrolled-by-4 SSE2 implementation - * Note that it uses all the 16 registers, meaning that x64 is required. + * Note that it uses 16 registers, meaning that x64 is required. */ -void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) +void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; - asm volatile("movdqa %0,%%xmm15" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("movdqa %0,%%xmm15" : : "m" (raid_sse_poly.poly[0])); asm volatile("pxor %xmm8,%xmm8"); asm volatile("pxor %xmm9,%xmm9"); asm volatile("pxor %xmm10,%xmm10"); asm volatile("pxor %xmm11,%xmm11"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %xmm4,%xmm8"); asm volatile("pcmpgtb %xmm5,%xmm9"); asm volatile("pcmpgtb %xmm6,%xmm10"); @@ -488,10 +500,11 @@ void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm9,%xmm5"); asm volatile("pxor %xmm10,%xmm6"); asm volatile("pxor %xmm11,%xmm7"); - asm volatile("movdqa %0,%%xmm8" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm9" : : "m" (buffer[z][d+16])); - asm volatile("movdqa %0,%%xmm10" : : "m" (buffer[z][d+32])); - asm volatile("movdqa %0,%%xmm11" : : "m" (buffer[z][d+48])); + + asm volatile("movdqa %0,%%xmm8" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm9" : : "m" (vbuf[d][o+16])); + asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[d][o+32])); + asm volatile("movdqa %0,%%xmm11" : : "m" (vbuf[d][o+48])); asm volatile("pxor %xmm8,%xmm0"); asm volatile("pxor %xmm9,%xmm1"); asm volatile("pxor %xmm10,%xmm2"); @@ -500,19 +513,20 @@ void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm9,%xmm5"); asm volatile("pxor %xmm10,%xmm6"); asm volatile("pxor %xmm11,%xmm7"); + asm volatile("pxor %xmm8,%xmm8"); asm volatile("pxor %xmm9,%xmm9"); asm volatile("pxor %xmm10,%xmm10"); asm volatile("pxor %xmm11,%xmm11"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32])); - asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48])); - asm volatile("movntdq %%xmm4,%0" : "=m" (q[d])); - asm volatile("movntdq %%xmm5,%0" : "=m" (q[d+16])); - asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+32])); - asm volatile("movntdq %%xmm7,%0" : "=m" (q[d+48])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (p[o+32])); + asm volatile("movntdq %%xmm3,%0" : "=m" (p[o+48])); + asm volatile("movntdq %%xmm4,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm5,%0" : "=m" (q[o+16])); + asm volatile("movntdq %%xmm6,%0" : "=m" (q[o+32])); + asm volatile("movntdq %%xmm7,%0" : "=m" (q[o+48])); } asm volatile("sfence" : : : "memory"); @@ -522,77 +536,104 @@ void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size) /* * RAIDTP unrolled-by-2 C implementation */ -void raidTP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidTP_int32r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; - int z, z0; - unsigned d; + int d, l; + unsigned o; - uint32_t wd0, wr0, wq0, wp0, w10, w20; - uint32_t wd1, wr1, wq1, wp1, w11, w21; + uint32_t d0, r0, q0, p0; + uint32_t d1, r1, q1, p1; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; - for(d=0;d= 0;--z) { - wd0 = *(uint32_t *)&buffer[z][d+0*4]; - wd1 = *(uint32_t *)&buffer[z][d+1*4]; + for(o=0;o=0;--d) { + d0 = v_32(vbuf[d][o]); + d1 = v_32(vbuf[d][o+4]); - /* wp += wd */ - wp0 ^= wd0; - wp1 ^= wd1; + p0 ^= d0; + p1 ^= d1; - /* w1 = wq * {02} */ - w20 = MASK(wq0); - w21 = MASK(wq1); - w10 = SHLBYTE(wq0); - w11 = SHLBYTE(wq1); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; + q0 = x2_32(q0); + q1 = x2_32(q1); - /* wq = w1 + wd */ - wq0 = w10 ^ wd0; - wq1 = w11 ^ wd1; + q0 ^= d0; + q1 ^= d1; - /* w1 = wr * {02} */ - w20 = MASK(wr0); - w21 = MASK(wr1); - w10 = SHLBYTE(wr0); - w11 = SHLBYTE(wr1); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; + r0 = x2_32(r0); + r1 = x2_32(r1); + r0 = x2_32(r0); + r1 = x2_32(r1); - /* w1 = w1 * {02} */ - w20 = MASK(w10); - w21 = MASK(w11); - w10 = SHLBYTE(w10); - w11 = SHLBYTE(w11); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; - - /* wr = w1 + wd */ - wr0 = w10 ^ wd0; - wr1 = w11 ^ wd1; + r0 ^= d0; + r1 ^= d1; } - *(uint32_t *)&p[d+4*0] = wp0; - *(uint32_t *)&p[d+4*1] = wp1; - *(uint32_t *)&q[d+4*0] = wq0; - *(uint32_t *)&q[d+4*1] = wq1; - *(uint32_t *)&r[d+4*0] = wr0; - *(uint32_t *)&r[d+4*1] = wr1; + v_32(p[o]) = p0; + v_32(p[o+4]) = p1; + v_32(q[o]) = q0; + v_32(q[o+4]) = q1; + v_32(r[o]) = r0; + v_32(r[o+4]) = r1; + } +} + +/* + * RAIDTP unrolled-by-2 C implementation + */ +void raidTP_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + int d, l; + unsigned o; + + uint64_t d0, r0, q0, p0; + uint64_t d1, r1, q1, p1; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + + for(o=0;o=0;--d) { + d0 = v_64(vbuf[d][o]); + d1 = v_64(vbuf[d][o+8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0); + q1 = x2_64(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = x2_64(r0); + r1 = x2_64(r1); + r0 = x2_64(r0); + r1 = x2_64(r1); + + r0 ^= d0; + r1 ^= d1; + } + v_64(p[o]) = p0; + v_64(p[o+8]) = p1; + v_64(q[o]) = q0; + v_64(q[o+8]) = q1; + v_64(r[o]) = r0; + v_64(r[o+8]) = r1; } } @@ -600,29 +641,29 @@ void raidTP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size) /* * RAIDTP unrolled-by-1 MMX implementation */ -void raidTP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; - asm volatile("movq %0,%%mm7" : : "m" (raid6_mmx_constants.x1d)); + asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly)); asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); asm volatile("pxor %mm6,%mm6"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %mm1,%mm4"); asm volatile("pcmpgtb %mm2,%mm5"); asm volatile("paddb %mm1,%mm1"); @@ -635,17 +676,19 @@ void raidTP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("paddb %mm2,%mm2"); asm volatile("pand %mm7,%mm6"); asm volatile("pxor %mm6,%mm2"); - asm volatile("movq %0,%%mm3" : : "m" (buffer[z][d])); + + asm volatile("movq %0,%%mm3" : : "m" (vbuf[d][o])); asm volatile("pxor %mm3,%mm0"); asm volatile("pxor %mm3,%mm1"); asm volatile("pxor %mm3,%mm2"); + asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); asm volatile("pxor %mm6,%mm6"); } - asm volatile("movq %%mm0,%0" : "=m" (p[d])); - asm volatile("movq %%mm1,%0" : "=m" (q[d])); - asm volatile("movq %%mm2,%0" : "=m" (r[d])); + asm volatile("movq %%mm0,%0" : "=m" (p[o])); + asm volatile("movq %%mm1,%0" : "=m" (q[o])); + asm volatile("movq %%mm2,%0" : "=m" (r[o])); } asm volatile("emms" : : : "memory"); @@ -656,29 +699,29 @@ void raidTP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) /* * RAIDTP unrolled-by-1 SSE2 implementation */ -void raidTP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm6,%xmm6"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm2,%xmm5"); asm volatile("paddb %xmm1,%xmm1"); @@ -691,17 +734,19 @@ void raidTP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("paddb %xmm2,%xmm2"); asm volatile("pand %xmm7,%xmm6"); asm volatile("pxor %xmm6,%xmm2"); - asm volatile("movdqa %0,%%xmm3" : : "m" (buffer[z][d])); + + asm volatile("movdqa %0,%%xmm3" : : "m" (vbuf[d][o])); asm volatile("pxor %xmm3,%xmm0"); asm volatile("pxor %xmm3,%xmm1"); asm volatile("pxor %xmm3,%xmm2"); + asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm6,%xmm6"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (q[d])); - asm volatile("movntdq %%xmm2,%0" : "=m" (r[d])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); } asm volatile("sfence" : : : "memory"); @@ -711,22 +756,22 @@ void raidTP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) #if defined(__x86_64__) /* * RAIDTP unrolled-by-2 SSE2 implementation - * Note that it uses all the 16 registers, meaning that x64 is required. + * Note that it uses 16 registers, meaning that x64 is required. */ -void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm6,%xmm6"); @@ -734,14 +779,14 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm13,%xmm13"); asm volatile("pxor %xmm14,%xmm14"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm2,%xmm5"); asm volatile("pcmpgtb %xmm9,%xmm12"); @@ -766,14 +811,16 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pand %xmm7,%xmm14"); asm volatile("pxor %xmm6,%xmm2"); asm volatile("pxor %xmm14,%xmm10"); - asm volatile("movdqa %0,%%xmm3" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm11" : : "m" (buffer[z][d+16])); + + asm volatile("movdqa %0,%%xmm3" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm11" : : "m" (vbuf[d][o+16])); asm volatile("pxor %xmm3,%xmm0"); asm volatile("pxor %xmm3,%xmm1"); asm volatile("pxor %xmm3,%xmm2"); asm volatile("pxor %xmm11,%xmm8"); asm volatile("pxor %xmm11,%xmm9"); asm volatile("pxor %xmm11,%xmm10"); + asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm6,%xmm6"); @@ -781,12 +828,12 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm13,%xmm13"); asm volatile("pxor %xmm14,%xmm14"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm8,%0" : "=m" (p[d+16])); - asm volatile("movntdq %%xmm1,%0" : "=m" (q[d])); - asm volatile("movntdq %%xmm9,%0" : "=m" (q[d+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (r[d])); - asm volatile("movntdq %%xmm10,%0" : "=m" (r[d+16])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16])); } asm volatile("sfence" : : : "memory"); @@ -796,116 +843,132 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) /* * RAIDQP unrolled-by-2 C implementation */ -void raidQP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidQP_int32r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; unsigned char* s; - int z, z0; - unsigned d; + int d, l; + unsigned o; - uint32_t wd0, ws0, wr0, wq0, wp0, w10, w20; - uint32_t wd1, ws1, wr1, wq1, wp1, w11, w21; + uint32_t d0, s0, r0, q0, p0; + uint32_t d1, s1, r1, q1, p1; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; - s = buffer[diskmax+3]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; - for(d=0;d= 0;--z) { - wd0 = *(uint32_t *)&buffer[z][d+0*4]; - wd1 = *(uint32_t *)&buffer[z][d+1*4]; + for(o=0;o=0;--d) { + d0 = v_32(vbuf[d][o]); + d1 = v_32(vbuf[d][o+4]); - /* wp += wd */ - wp0 ^= wd0; - wp1 ^= wd1; + p0 ^= d0; + p1 ^= d1; - /* w1 = wq * {02} */ - w20 = MASK(wq0); - w21 = MASK(wq1); - w10 = SHLBYTE(wq0); - w11 = SHLBYTE(wq1); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; + q0 = x2_32(q0); + q1 = x2_32(q1); - /* wq = w1 + wd */ - wq0 = w10 ^ wd0; - wq1 = w11 ^ wd1; + q0 ^= d0; + q1 ^= d1; - /* w1 = wr * {02} */ - w20 = MASK(wr0); - w21 = MASK(wr1); - w10 = SHLBYTE(wr0); - w11 = SHLBYTE(wr1); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; + r0 = x2_32(r0); + r1 = x2_32(r1); + r0 = x2_32(r0); + r1 = x2_32(r1); - /* w1 = w1 * {02} */ - w20 = MASK(w10); - w21 = MASK(w11); - w10 = SHLBYTE(w10); - w11 = SHLBYTE(w11); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; + r0 ^= d0; + r1 ^= d1; - /* wr = w1 + wd */ - wr0 = w10 ^ wd0; - wr1 = w11 ^ wd1; - - /* w1 = ws * {02} */ - w20 = MASK(ws0); - w21 = MASK(ws1); - w10 = SHLBYTE(ws0); - w11 = SHLBYTE(ws1); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; - - /* w1 = w1 * {02} */ - w20 = MASK(w10); - w21 = MASK(w11); - w10 = SHLBYTE(w10); - w11 = SHLBYTE(w11); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; - - /* w1 = w1 * {02} */ - w20 = MASK(w10); - w21 = MASK(w11); - w10 = SHLBYTE(w10); - w11 = SHLBYTE(w11); - w20 &= NBYTES(0x1d); - w21 &= NBYTES(0x1d); - w10 ^= w20; - w11 ^= w21; - - /* ws = w1 + wd */ - ws0 = w10 ^ wd0; - ws1 = w11 ^ wd1; + s0 = x2_32(s0); + s1 = x2_32(s1); + s0 = x2_32(s0); + s1 = x2_32(s1); + s0 = x2_32(s0); + s1 = x2_32(s1); + s0 ^= d0; + s1 ^= d1; } - *(uint32_t *)&p[d+4*0] = wp0; - *(uint32_t *)&p[d+4*1] = wp1; - *(uint32_t *)&q[d+4*0] = wq0; - *(uint32_t *)&q[d+4*1] = wq1; - *(uint32_t *)&r[d+4*0] = wr0; - *(uint32_t *)&r[d+4*1] = wr1; - *(uint32_t *)&s[d+4*0] = ws0; - *(uint32_t *)&s[d+4*1] = ws1; + v_32(p[o]) = p0; + v_32(p[o+4]) = p1; + v_32(q[o]) = q0; + v_32(q[o+4]) = q1; + v_32(r[o]) = r0; + v_32(r[o+4]) = r1; + v_32(s[o]) = s0; + v_32(s[o+4]) = s1; + } +} + +/* + * RAIDQP unrolled-by-2 C implementation + */ +void raidQP_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + unsigned char* s; + int d, l; + unsigned o; + + uint64_t d0, s0, r0, q0, p0; + uint64_t d1, s1, r1, q1, p1; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; + + for(o=0;o=0;--d) { + d0 = v_64(vbuf[d][o]); + d1 = v_64(vbuf[d][o+8]); + + p0 ^= d0; + p1 ^= d1; + + q0 = x2_64(q0); + q1 = x2_64(q1); + + q0 ^= d0; + q1 ^= d1; + + r0 = x2_64(r0); + r1 = x2_64(r1); + r0 = x2_64(r0); + r1 = x2_64(r1); + + r0 ^= d0; + r1 ^= d1; + + s0 = x2_64(s0); + s1 = x2_64(s1); + s0 = x2_64(s0); + s1 = x2_64(s1); + s0 = x2_64(s0); + s1 = x2_64(s1); + + s0 ^= d0; + s1 ^= d1; + } + v_64(p[o]) = p0; + v_64(p[o+8]) = p1; + v_64(q[o]) = q0; + v_64(q[o+8]) = q1; + v_64(r[o]) = r0; + v_64(r[o+8]) = r1; + v_64(s[o]) = s0; + v_64(s[o+8]) = s1; } } @@ -913,31 +976,31 @@ void raidQP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size) /* * RAIDQP unrolled-by-1 MMX implementation */ -void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; unsigned char* s; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; - s = buffer[diskmax+3]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; - asm volatile("movq %0,%%mm7" : : "m" (raid6_mmx_constants.x1d)); + asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly)); asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %mm1,%mm4"); asm volatile("pcmpgtb %mm3,%mm5"); asm volatile("paddb %mm1,%mm1"); @@ -971,7 +1034,7 @@ void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %mm4,%mm2"); asm volatile("pxor %mm5,%mm3"); - asm volatile("movq %0,%%mm6" : : "m" (buffer[z][d])); + asm volatile("movq %0,%%mm6" : : "m" (vbuf[d][o])); asm volatile("pxor %mm6,%mm0"); asm volatile("pxor %mm6,%mm1"); asm volatile("pxor %mm6,%mm2"); @@ -980,10 +1043,10 @@ void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); } - asm volatile("movq %%mm0,%0" : "=m" (p[d])); - asm volatile("movq %%mm1,%0" : "=m" (q[d])); - asm volatile("movq %%mm2,%0" : "=m" (r[d])); - asm volatile("movq %%mm3,%0" : "=m" (s[d])); + asm volatile("movq %%mm0,%0" : "=m" (p[o])); + asm volatile("movq %%mm1,%0" : "=m" (q[o])); + asm volatile("movq %%mm2,%0" : "=m" (r[o])); + asm volatile("movq %%mm3,%0" : "=m" (s[o])); } asm volatile("emms" : : : "memory"); @@ -994,31 +1057,31 @@ void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size) /* * RAIDQP unrolled-by-1 SSE2 implementation */ -void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; unsigned char* s; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; - s = buffer[diskmax+3]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("paddb %xmm1,%xmm1"); @@ -1052,7 +1115,7 @@ void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm4,%xmm2"); asm volatile("pxor %xmm5,%xmm3"); - asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d])); + asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o])); asm volatile("pxor %xmm6,%xmm0"); asm volatile("pxor %xmm6,%xmm1"); asm volatile("pxor %xmm6,%xmm2"); @@ -1061,10 +1124,10 @@ void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm1,%0" : "=m" (q[d])); - asm volatile("movntdq %%xmm2,%0" : "=m" (r[d])); - asm volatile("movntdq %%xmm3,%0" : "=m" (s[d])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm3,%0" : "=m" (s[o])); } asm volatile("sfence" : : : "memory"); @@ -1074,39 +1137,39 @@ void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size) #if defined(__x86_64__) /* * RAIDQP unrolled-by-2 SSE2 implementation - * Note that it uses all the 16 registers, meaning that x64 is required. + * Note that it uses 16 registers, meaning that x64 is required. */ -void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) +void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; unsigned char* r; unsigned char* s; - int z, z0; - unsigned d; + int d, l; + unsigned o; - z0 = diskmax - 1; - p = buffer[diskmax]; - q = buffer[diskmax+1]; - r = buffer[diskmax+2]; - s = buffer[diskmax+3]; + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm12,%xmm12"); asm volatile("pxor %xmm13,%xmm13"); - for(d=0;d=0;--z) { + for(o=0;o=0;--d) { asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("pcmpgtb %xmm9,%xmm12"); @@ -1168,8 +1231,8 @@ void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm12,%xmm10"); asm volatile("pxor %xmm13,%xmm11"); - asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d])); - asm volatile("movdqa %0,%%xmm14" : : "m" (buffer[z][d+16])); + asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm14" : : "m" (vbuf[d][o+16])); asm volatile("pxor %xmm6,%xmm0"); asm volatile("pxor %xmm6,%xmm1"); asm volatile("pxor %xmm6,%xmm2"); @@ -1184,14 +1247,14 @@ void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) asm volatile("pxor %xmm12,%xmm12"); asm volatile("pxor %xmm13,%xmm13"); } - asm volatile("movntdq %%xmm0,%0" : "=m" (p[d])); - asm volatile("movntdq %%xmm8,%0" : "=m" (p[d+16])); - asm volatile("movntdq %%xmm1,%0" : "=m" (q[d])); - asm volatile("movntdq %%xmm9,%0" : "=m" (q[d+16])); - asm volatile("movntdq %%xmm2,%0" : "=m" (r[d])); - asm volatile("movntdq %%xmm10,%0" : "=m" (r[d+16])); - asm volatile("movntdq %%xmm3,%0" : "=m" (s[d])); - asm volatile("movntdq %%xmm11,%0" : "=m" (s[d+16])); + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16])); + asm volatile("movntdq %%xmm3,%0" : "=m" (s[o])); + asm volatile("movntdq %%xmm11,%0" : "=m" (s[o+16])); } asm volatile("sfence" : : : "memory"); @@ -1199,21 +1262,21 @@ void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size) #endif /****************************************************************************/ -/* generic computation */ +/* parity generation */ /* internal forwarder */ -static void (*raid5_gen)(unsigned char** buffer, unsigned diskmax, unsigned size); -static void (*raid6_gen)(unsigned char** buffer, unsigned diskmax, unsigned size); -static void (*raidTP_gen)(unsigned char** buffer, unsigned diskmax, unsigned size); -static void (*raidQP_gen)(unsigned char** buffer, unsigned diskmax, unsigned size); +static void (*raid5_gen)(unsigned char** vbuf, unsigned data, unsigned size); +static void (*raid6_gen)(unsigned char** vbuf, unsigned data, unsigned size); +static void (*raidTP_gen)(unsigned char** vbuf, unsigned data, unsigned size); +static void (*raidQP_gen)(unsigned char** vbuf, unsigned data, unsigned size); -void raid_gen(unsigned level, unsigned char** buffer, unsigned diskmax, unsigned size) +void raid_gen(unsigned level, unsigned char** vbuf, unsigned data, unsigned size) { switch (level) { - case 1 : raid5_gen(buffer, diskmax, size); break; - case 2 : raid6_gen(buffer, diskmax, size); break; - case 3 : raidTP_gen(buffer, diskmax, size); break; - case 4 : raidQP_gen(buffer, diskmax, size); break; + case 1 : raid5_gen(vbuf, data, size); break; + case 2 : raid6_gen(vbuf, data, size); break; + case 3 : raidTP_gen(vbuf, data, size); break; + case 4 : raidQP_gen(vbuf, data, size); break; } } @@ -1222,10 +1285,12 @@ void raid_gen(unsigned level, unsigned char** buffer, unsigned diskmax, unsigned /** * The data recovering is based on the paper "The mathematics of RAID-6" [1], - * that covers the RAID5 and RAID6 computations in the Galois Field GF(2^8) - * with the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 (285 decimal), - * using the following equations to compute syndrome P and Q from a set - * of n disk Di with 0<=i