diff --git a/check.c b/check.c
index a2797b0a..fa0a5683 100644
--- a/check.c
+++ b/check.c
@@ -228,7 +228,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO1[i].a], buffer_recov[COMBO1[i].a], state->block_size);
/* recover data */
- raid_recov_1data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, COMBO1[i].a, buffer_zero);
+ raid_recov_1data(failed[failed_map[0]].index, COMBO1[i].a, buffer, diskmax, buffer_zero, state->block_size);
if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero))
return 0;
@@ -249,7 +249,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO2[i].a], buffer_recov[COMBO2[i].a], state->block_size);
/* recover data */
- raid_recov_1data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, COMBO2[i].a, buffer_zero);
+ raid_recov_1data(failed[failed_map[0]].index, COMBO2[i].a, buffer, diskmax, buffer_zero, state->block_size);
if (is_parity_matching(state, diskmax, COMBO2[i].b, buffer, buffer_recov))
return 0;
@@ -271,7 +271,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO2[i].b], buffer_recov[COMBO2[i].b], state->block_size);
/* recover data */
- raid_recov_2data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO2[i].a, COMBO2[i].b, buffer_zero);
+ raid_recov_2data(failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO2[i].a, COMBO2[i].b, buffer, diskmax, buffer_zero, state->block_size);
if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero))
return 0;
@@ -293,7 +293,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO3[i].b], buffer_recov[COMBO3[i].b], state->block_size);
/* recover data */
- raid_recov_2data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO3[i].a, COMBO3[i].b, buffer_zero);
+ raid_recov_2data(failed[failed_map[0]].index, failed[failed_map[1]].index, COMBO3[i].a, COMBO3[i].b, buffer, diskmax, buffer_zero, state->block_size);
if (is_parity_matching(state, diskmax, COMBO3[i].c, buffer, buffer_recov))
return 0;
@@ -316,7 +316,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO3[i].c], buffer_recov[COMBO3[i].c], state->block_size);
/* recover data */
- raid_recov_3data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO3[i].a, COMBO3[i].b, COMBO3[i].c, buffer_zero);
+ raid_recov_3data(failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO3[i].a, COMBO3[i].b, COMBO3[i].c, buffer, diskmax, buffer_zero, state->block_size);
if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero))
return 0;
@@ -339,7 +339,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO4[i].c], buffer_recov[COMBO4[i].c], state->block_size);
/* recover data */
- raid_recov_3data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, buffer_zero);
+ raid_recov_3data(failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, buffer, diskmax, buffer_zero, state->block_size);
if (is_parity_matching(state, diskmax, COMBO4[i].d, buffer, buffer_recov))
return 0;
@@ -363,7 +363,7 @@ static int repair_step(struct snapraid_state* state, int rehash, unsigned pos, u
memcpy(buffer[diskmax+COMBO4[i].d], buffer_recov[COMBO4[i].d], state->block_size);
/* recover data */
- raid_recov_4data(buffer, diskmax, state->block_size, failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, failed[failed_map[3]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, COMBO4[i].d, buffer_zero);
+ raid_recov_4data(failed[failed_map[0]].index, failed[failed_map[1]].index, failed[failed_map[2]].index, failed[failed_map[3]].index, COMBO4[i].a, COMBO4[i].b, COMBO4[i].c, COMBO4[i].d, buffer, diskmax, buffer_zero, state->block_size);
if (is_hash_matching(state, rehash, diskmax, failed, failed_map, failed_count, buffer, buffer_zero))
return 0;
diff --git a/raid.c b/raid.c
index 02f159d9..25c6fc99 100644
--- a/raid.c
+++ b/raid.c
@@ -15,17 +15,18 @@
* along with this program. If not, see .
*/
-/* -*- linux-c -*- ------------------------------------------------------- *
+/*
+ * The RAID5/RAID6 support was originally derived by the libraid6 library
+ * by H. Peter Anvin released with license "GPL2 or any later version"
+ * used by the Linux Kernel 2.6.38.
*
- * Copyright 2002-2004 H. Peter Anvin - All Rights Reserved
+ * This support was later completely rewritten (many times), but the
+ * H. Peter Anvin's Copyright may still apply.
*
- * This program is free software; you can redistribute it and/or modify
- * it under the terms of the GNU General Public License as published by
- * the Free Software Foundation, Inc., 53 Temple Place Ste 330,
- * Boston MA 02111-1307, USA; either version 2 of the License, or
- * (at your option) any later version; incorporated herein by reference.
- *
- * ----------------------------------------------------------------------- */
+ * The RAIDTP/RAIDQP support is original work implemented from scratch.
+ * For some notes about the method used, see the "recovering" section
+ * in this file.
+ */
#include "portable.h"
@@ -34,159 +35,140 @@
#include "tables.h"
/****************************************************************************/
-/* internal computation */
+/* specialized parity generation */
+
+/**
+ * Dereference as uint32_t
+ */
+#define v_32(p) (*(uint32_t*)&(p))
+
+/**
+ * Dereference as uint64_t
+ */
+#define v_64(p) (*(uint64_t*)&(p))
/*
* RAID5 unrolled-by-2 C implementation
*/
-void raid5_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid5_int32r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- uint32_t wp0;
- uint32_t wp1;
+ uint32_t p0;
+ uint32_t p1;
- z0 = diskmax - 1;
- p = buffer[diskmax];
+ l = data - 1;
+ p = vbuf[data];
- for(d=0;d=0;--z) {
- wp0 ^= *(uint32_t *)&buffer[z][d+0*4];
- wp1 ^= *(uint32_t *)&buffer[z][d+1*4];
+ for(o=0;o=0;--d) {
+ p0 ^= v_32(vbuf[d][o]);
+ p1 ^= v_32(vbuf[d][o+4]);
}
- *(uint32_t *)&p[d+4*0] = wp0;
- *(uint32_t *)&p[d+4*1] = wp1;
+ v_32(p[o]) = p0;
+ v_32(p[o+4]) = p1;
+ }
+}
+
+/*
+ * RAID5 unrolled-by-2 C implementation
+ */
+void raid5_int64r2(unsigned char** vbuf, unsigned data, unsigned size)
+{
+ unsigned char* p;
+ int d, l;
+ unsigned o;
+
+ uint64_t p0;
+ uint64_t p1;
+
+ l = data - 1;
+ p = vbuf[data];
+
+ for(o=0;o=0;--d) {
+ p0 ^= v_64(vbuf[d][o]);
+ p1 ^= v_64(vbuf[d][o+8]);
+ }
+ v_64(p[o]) = p0;
+ v_64(p[o+8]) = p1;
}
}
#if defined(__i386__) || defined(__x86_64__)
-/*
- * RAID5 unrolled-by-2 MMX implementation
- */
-void raid5_mmxr2(unsigned char** buffer, unsigned diskmax, unsigned size)
-{
- unsigned char* p;
- int z, z0;
- unsigned d;
-
- z0 = diskmax - 1;
- p = buffer[diskmax];
-
- for(d=0;d=0;--z) {
- asm volatile("movq %0,%%mm4" : : "m" (buffer[z][d]));
- asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d+8]));
- asm volatile("pxor %mm4,%mm0");
- asm volatile("pxor %mm5,%mm1");
- }
- asm volatile("movq %%mm0,%0" : "=m" (p[d]));
- asm volatile("movq %%mm1,%0" : "=m" (p[d+8]));
- }
-
- asm volatile("emms" : : : "memory");
-}
-
/*
* RAID5 unrolled-by-4 MMX implementation
*/
-void raid5_mmxr4(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid5_mmxr4(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
+ l = data - 1;
+ p = vbuf[data];
- for(d=0;d=0;--z) {
- asm volatile("movq %0,%%mm4" : : "m" (buffer[z][d]));
- asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d+8]));
- asm volatile("movq %0,%%mm6" : : "m" (buffer[z][d+16]));
- asm volatile("movq %0,%%mm7" : : "m" (buffer[z][d+24]));
+ for(o=0;o=0;--d) {
+ asm volatile("movq %0,%%mm4" : : "m" (vbuf[d][o]));
+ asm volatile("movq %0,%%mm5" : : "m" (vbuf[d][o+8]));
+ asm volatile("movq %0,%%mm6" : : "m" (vbuf[d][o+16]));
+ asm volatile("movq %0,%%mm7" : : "m" (vbuf[d][o+24]));
asm volatile("pxor %mm4,%mm0");
asm volatile("pxor %mm5,%mm1");
asm volatile("pxor %mm6,%mm2");
asm volatile("pxor %mm7,%mm3");
}
- asm volatile("movq %%mm0,%0" : "=m" (p[d]));
- asm volatile("movq %%mm1,%0" : "=m" (p[d+8]));
- asm volatile("movq %%mm2,%0" : "=m" (p[d+16]));
- asm volatile("movq %%mm3,%0" : "=m" (p[d+24]));
+ asm volatile("movq %%mm0,%0" : "=m" (p[o]));
+ asm volatile("movq %%mm1,%0" : "=m" (p[o+8]));
+ asm volatile("movq %%mm2,%0" : "=m" (p[o+16]));
+ asm volatile("movq %%mm3,%0" : "=m" (p[o+24]));
}
asm volatile("emms" : : : "memory");
}
-/*
- * RAID5 unrolled-by-2 SSE2 implementation
- */
-void raid5_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
-{
- unsigned char* p;
- int z, z0;
- unsigned d;
-
- z0 = diskmax - 1;
- p = buffer[diskmax];
-
- for(d=0;d=0;--z) {
- asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16]));
- asm volatile("pxor %xmm4,%xmm0");
- asm volatile("pxor %xmm5,%xmm1");
- }
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16]));
- }
-
- asm volatile("sfence" : : : "memory");
-}
-
/*
* RAID5 unrolled-by-4 SSE2 implementation
*/
-void raid5_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid5_sse2r4(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
+ l = data - 1;
+ p = vbuf[data];
- for(d=0;d=0;--z) {
- asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16]));
- asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d+32]));
- asm volatile("movdqa %0,%%xmm7" : : "m" (buffer[z][d+48]));
+ for(o=0;o=0;--d) {
+ asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o]));
+ asm volatile("movdqa %0,%%xmm5" : : "m" (vbuf[d][o+16]));
+ asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o+32]));
+ asm volatile("movdqa %0,%%xmm7" : : "m" (vbuf[d][o+48]));
asm volatile("pxor %xmm4,%xmm0");
asm volatile("pxor %xmm5,%xmm1");
asm volatile("pxor %xmm6,%xmm2");
asm volatile("pxor %xmm7,%xmm3");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (p[o+32]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (p[o+48]));
}
asm volatile("sfence" : : : "memory");
@@ -196,35 +178,35 @@ void raid5_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size)
#if defined(__x86_64__)
/*
* RAID5 unrolled-by-8 SSE2 implementation
- * Note that it uses all the 16 registers, meaning that x64 is required.
+ * Note that it uses 16 registers, meaning that x64 is required.
*/
-void raid5_sse2r8(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid5_sse2r8(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
+ l = data - 1;
+ p = vbuf[data];
- for(d=0;d=0;--z) {
- asm volatile("movdqa %0,%%xmm8" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm9" : : "m" (buffer[z][d+16]));
- asm volatile("movdqa %0,%%xmm10" : : "m" (buffer[z][d+32]));
- asm volatile("movdqa %0,%%xmm11" : : "m" (buffer[z][d+48]));
- asm volatile("movdqa %0,%%xmm12" : : "m" (buffer[z][d+64]));
- asm volatile("movdqa %0,%%xmm13" : : "m" (buffer[z][d+80]));
- asm volatile("movdqa %0,%%xmm14" : : "m" (buffer[z][d+96]));
- asm volatile("movdqa %0,%%xmm15" : : "m" (buffer[z][d+112]));
+ for(o=0;o=0;--d) {
+ asm volatile("movdqa %0,%%xmm8" : : "m" (vbuf[d][o]));
+ asm volatile("movdqa %0,%%xmm9" : : "m" (vbuf[d][o+16]));
+ asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[d][o+32]));
+ asm volatile("movdqa %0,%%xmm11" : : "m" (vbuf[d][o+48]));
+ asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][o+64]));
+ asm volatile("movdqa %0,%%xmm13" : : "m" (vbuf[d][o+80]));
+ asm volatile("movdqa %0,%%xmm14" : : "m" (vbuf[d][o+96]));
+ asm volatile("movdqa %0,%%xmm15" : : "m" (vbuf[d][o+112]));
asm volatile("pxor %xmm8,%xmm0");
asm volatile("pxor %xmm9,%xmm1");
asm volatile("pxor %xmm10,%xmm2");
@@ -234,185 +216,213 @@ void raid5_sse2r8(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm14,%xmm6");
asm volatile("pxor %xmm15,%xmm7");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48]));
- asm volatile("movntdq %%xmm4,%0" : "=m" (p[d+64]));
- asm volatile("movntdq %%xmm5,%0" : "=m" (p[d+80]));
- asm volatile("movntdq %%xmm6,%0" : "=m" (p[d+96]));
- asm volatile("movntdq %%xmm7,%0" : "=m" (p[d+112]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (p[o+32]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (p[o+48]));
+ asm volatile("movntdq %%xmm4,%0" : "=m" (p[o+64]));
+ asm volatile("movntdq %%xmm5,%0" : "=m" (p[o+80]));
+ asm volatile("movntdq %%xmm6,%0" : "=m" (p[o+96]));
+ asm volatile("movntdq %%xmm7,%0" : "=m" (p[o+112]));
}
asm volatile("sfence" : : : "memory");
}
#endif
-/*
- * Repeats an 8 bit nibble to a full 32 bits value.
+/**
+ * Multiply each byte of a uint32 by 2 in the GF(2^8).
*/
-#define NBYTES(x) ((x) * 0x01010101U)
-
-/*
- * These sub-operations are separate inlines since they can sometimes be
- * specially optimized using architecture-specific hacks.
- */
-
-/*
- * The SHLBYTE() operation shifts each byte left by 1, *not*
- * rolling over into the next byte.
- */
-static inline uint32_t SHLBYTE(uint32_t v)
+__attribute__((always_inline)) static inline uint32_t x2_32(uint32_t v)
{
- uint32_t vv;
-
- vv = (v << 1) & NBYTES(0xfe);
- return vv;
+ uint32_t mask = v & 0x80808080U;
+ mask = (mask << 1) - (mask >> 7);
+ v = (v << 1) & 0xfefefefeU;
+ v ^= mask & 0x1d1d1d1dU;
+ return v;
}
-/*
- * The MASK() operation returns 0xFF in any byte for which the high
- * bit is 1, 0x00 for any byte for which the high bit is 0.
+/**
+ * Multiply each byte of a uint64 by 2 in the GF(2^8).
*/
-static inline uint32_t MASK(uint32_t v)
+__attribute__((always_inline)) static inline uint64_t x2_64(uint64_t v)
{
- uint32_t vv;
-
- vv = v & NBYTES(0x80);
- vv = (vv << 1) - (vv >> 7); /* Overflow on the top bit is OK */
- return vv;
+ uint64_t mask = v & 0x8080808080808080ULL;
+ mask = (mask << 1) - (mask >> 7);
+ v = (v << 1) & 0xfefefefefefefefeULL;
+ v ^= mask & 0x1d1d1d1d1d1d1d1dULL;
+ return v;
}
/*
* RAID6 unrolled-by-2 C implementation
*/
-void raid6_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid6_int32r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- uint32_t wd0, wq0, wp0, w10, w20;
- uint32_t wd1, wq1, wp1, w11, w21;
+ uint32_t d0, q0, p0;
+ uint32_t d1, q1, p1;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
- for(d=0;d= 0;--z) {
- wd0 = *(uint32_t *)&buffer[z][d+0*4];
- wd1 = *(uint32_t *)&buffer[z][d+1*4];
- wp0 ^= wd0;
- wp1 ^= wd1;
- w20 = MASK(wq0);
- w21 = MASK(wq1);
- w10 = SHLBYTE(wq0);
- w11 = SHLBYTE(wq1);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
- wq0 = w10 ^ wd0;
- wq1 = w11 ^ wd1;
+ for(o=0;o=0;--d) {
+ d0 = v_32(vbuf[d][o]);
+ d1 = v_32(vbuf[d][o+4]);
+
+ p0 ^= d0;
+ p1 ^= d1;
+
+ q0 = x2_32(q0);
+ q1 = x2_32(q1);
+
+ q0 ^= d0;
+ q1 ^= d1;
}
- *(uint32_t *)&p[d+4*0] = wp0;
- *(uint32_t *)&p[d+4*1] = wp1;
- *(uint32_t *)&q[d+4*0] = wq0;
- *(uint32_t *)&q[d+4*1] = wq1;
+ v_32(p[o]) = p0;
+ v_32(p[o+4]) = p1;
+ v_32(q[o]) = q0;
+ v_32(q[o+4]) = q1;
+ }
+}
+
+/*
+ * RAID6 unrolled-by-2 C implementation
+ */
+void raid6_int64r2(unsigned char** vbuf, unsigned data, unsigned size)
+{
+ unsigned char* p;
+ unsigned char* q;
+ int d, l;
+ unsigned o;
+
+ uint64_t d0, q0, p0;
+ uint64_t d1, q1, p1;
+
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+
+ for(o=0;o=0;--d) {
+ d0 = v_64(vbuf[d][o]);
+ d1 = v_64(vbuf[d][o+8]);
+
+ p0 ^= d0;
+ p1 ^= d1;
+
+ q0 = x2_64(q0);
+ q1 = x2_64(q1);
+
+ q0 ^= d0;
+ q1 ^= d1;
+ }
+ v_64(p[o]) = p0;
+ v_64(p[o+8]) = p1;
+ v_64(q[o]) = q0;
+ v_64(q[o+8]) = q1;
}
}
#if defined(__i386__) || defined(__x86_64__)
-static const struct raid6_mmx_constants {
- uint64_t x1d;
-} raid6_mmx_constants = {
+static const struct raid_mmx_poly {
+ uint64_t poly;
+} raid_mmx_poly = {
0x1d1d1d1d1d1d1d1dULL,
};
/*
* RAID6 unrolled-by-2 MMX implementation
*/
-void raid6_mmxr2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
- asm volatile("movq %0,%%mm0" : : "m" (raid6_mmx_constants.x1d));
+ asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly));
+ asm volatile("pxor %mm4,%mm4");
asm volatile("pxor %mm5,%mm5");
- asm volatile("pxor %mm7,%mm7");
- for(d=0;d=0;--z) {
- asm volatile("pcmpgtb %mm4,%mm5");
- asm volatile("pcmpgtb %mm6,%mm7");
- asm volatile("paddb %mm4,%mm4");
- asm volatile("paddb %mm6,%mm6");
- asm volatile("pand %mm0,%mm5");
- asm volatile("pand %mm0,%mm7");
- asm volatile("pxor %mm5,%mm4");
- asm volatile("pxor %mm7,%mm6");
- asm volatile("movq %0,%%mm5" : : "m" (buffer[z][d]));
- asm volatile("movq %0,%%mm7" : : "m" (buffer[z][d+8]));
- asm volatile("pxor %mm5,%mm2");
- asm volatile("pxor %mm7,%mm3");
- asm volatile("pxor %mm5,%mm4");
- asm volatile("pxor %mm7,%mm6");
+ for(o=0;o=0;--d) {
+ asm volatile("pcmpgtb %mm2,%mm4");
+ asm volatile("pcmpgtb %mm3,%mm5");
+ asm volatile("paddb %mm2,%mm2");
+ asm volatile("paddb %mm3,%mm3");
+ asm volatile("pand %mm7,%mm4");
+ asm volatile("pand %mm7,%mm5");
+ asm volatile("pxor %mm4,%mm2");
+ asm volatile("pxor %mm5,%mm3");
+
+ asm volatile("movq %0,%%mm4" : : "m" (vbuf[d][o]));
+ asm volatile("movq %0,%%mm5" : : "m" (vbuf[d][o+8]));
+ asm volatile("pxor %mm4,%mm0");
+ asm volatile("pxor %mm5,%mm1");
+ asm volatile("pxor %mm4,%mm2");
+ asm volatile("pxor %mm5,%mm3");
+
+ asm volatile("pxor %mm4,%mm4");
asm volatile("pxor %mm5,%mm5");
- asm volatile("pxor %mm7,%mm7");
}
- asm volatile("movq %%mm2,%0" : "=m" (p[d]));
- asm volatile("movq %%mm3,%0" : "=m" (p[d+8]));
- asm volatile("movq %%mm4,%0" : "=m" (q[d]));
- asm volatile("movq %%mm6,%0" : "=m" (q[d+8]));
+ asm volatile("movq %%mm0,%0" : "=m" (p[o]));
+ asm volatile("movq %%mm1,%0" : "=m" (p[o+8]));
+ asm volatile("movq %%mm2,%0" : "=m" (q[o]));
+ asm volatile("movq %%mm3,%0" : "=m" (q[o+8]));
}
asm volatile("emms" : : : "memory");
}
-static const struct raid6_sse_constants {
- uint64_t x1d[2];
-} raid6_sse_constants __attribute__((aligned(16))) = {
+static const struct raid_sse_poly {
+ uint64_t poly[2];
+} raid_sse_poly __attribute__((aligned(16))) = {
{ 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL },
};
/*
* RAID6 unrolled-by-2 SSE2 implementation
*/
-void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
- asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0]));
+ asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0]));
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %xmm2,%xmm4");
asm volatile("pcmpgtb %xmm3,%xmm5");
asm volatile("paddb %xmm2,%xmm2");
@@ -421,19 +431,21 @@ void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pand %xmm7,%xmm5");
asm volatile("pxor %xmm4,%xmm2");
asm volatile("pxor %xmm5,%xmm3");
- asm volatile("movdqa %0,%%xmm4" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm5" : : "m" (buffer[z][d+16]));
+
+ asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o]));
+ asm volatile("movdqa %0,%%xmm5" : : "m" (vbuf[d][o+16]));
asm volatile("pxor %xmm4,%xmm0");
asm volatile("pxor %xmm5,%xmm1");
asm volatile("pxor %xmm4,%xmm2");
asm volatile("pxor %xmm5,%xmm3");
+
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (q[d+16]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (q[o]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (q[o+16]));
}
asm volatile("sfence" : : : "memory");
@@ -443,35 +455,35 @@ void raid6_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
#if defined(__x86_64__)
/*
* RAID6 unrolled-by-4 SSE2 implementation
- * Note that it uses all the 16 registers, meaning that x64 is required.
+ * Note that it uses 16 registers, meaning that x64 is required.
*/
-void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
- asm volatile("movdqa %0,%%xmm15" : : "m" (raid6_sse_constants.x1d[0]));
+ asm volatile("movdqa %0,%%xmm15" : : "m" (raid_sse_poly.poly[0]));
asm volatile("pxor %xmm8,%xmm8");
asm volatile("pxor %xmm9,%xmm9");
asm volatile("pxor %xmm10,%xmm10");
asm volatile("pxor %xmm11,%xmm11");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %xmm4,%xmm8");
asm volatile("pcmpgtb %xmm5,%xmm9");
asm volatile("pcmpgtb %xmm6,%xmm10");
@@ -488,10 +500,11 @@ void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm9,%xmm5");
asm volatile("pxor %xmm10,%xmm6");
asm volatile("pxor %xmm11,%xmm7");
- asm volatile("movdqa %0,%%xmm8" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm9" : : "m" (buffer[z][d+16]));
- asm volatile("movdqa %0,%%xmm10" : : "m" (buffer[z][d+32]));
- asm volatile("movdqa %0,%%xmm11" : : "m" (buffer[z][d+48]));
+
+ asm volatile("movdqa %0,%%xmm8" : : "m" (vbuf[d][o]));
+ asm volatile("movdqa %0,%%xmm9" : : "m" (vbuf[d][o+16]));
+ asm volatile("movdqa %0,%%xmm10" : : "m" (vbuf[d][o+32]));
+ asm volatile("movdqa %0,%%xmm11" : : "m" (vbuf[d][o+48]));
asm volatile("pxor %xmm8,%xmm0");
asm volatile("pxor %xmm9,%xmm1");
asm volatile("pxor %xmm10,%xmm2");
@@ -500,19 +513,20 @@ void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm9,%xmm5");
asm volatile("pxor %xmm10,%xmm6");
asm volatile("pxor %xmm11,%xmm7");
+
asm volatile("pxor %xmm8,%xmm8");
asm volatile("pxor %xmm9,%xmm9");
asm volatile("pxor %xmm10,%xmm10");
asm volatile("pxor %xmm11,%xmm11");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (p[d+32]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (p[d+48]));
- asm volatile("movntdq %%xmm4,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm5,%0" : "=m" (q[d+16]));
- asm volatile("movntdq %%xmm6,%0" : "=m" (q[d+32]));
- asm volatile("movntdq %%xmm7,%0" : "=m" (q[d+48]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (p[o+32]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (p[o+48]));
+ asm volatile("movntdq %%xmm4,%0" : "=m" (q[o]));
+ asm volatile("movntdq %%xmm5,%0" : "=m" (q[o+16]));
+ asm volatile("movntdq %%xmm6,%0" : "=m" (q[o+32]));
+ asm volatile("movntdq %%xmm7,%0" : "=m" (q[o+48]));
}
asm volatile("sfence" : : : "memory");
@@ -522,77 +536,104 @@ void raid6_sse2r4(unsigned char** buffer, unsigned diskmax, unsigned size)
/*
* RAIDTP unrolled-by-2 C implementation
*/
-void raidTP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidTP_int32r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- uint32_t wd0, wr0, wq0, wp0, w10, w20;
- uint32_t wd1, wr1, wq1, wp1, w11, w21;
+ uint32_t d0, r0, q0, p0;
+ uint32_t d1, r1, q1, p1;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
- for(d=0;d= 0;--z) {
- wd0 = *(uint32_t *)&buffer[z][d+0*4];
- wd1 = *(uint32_t *)&buffer[z][d+1*4];
+ for(o=0;o=0;--d) {
+ d0 = v_32(vbuf[d][o]);
+ d1 = v_32(vbuf[d][o+4]);
- /* wp += wd */
- wp0 ^= wd0;
- wp1 ^= wd1;
+ p0 ^= d0;
+ p1 ^= d1;
- /* w1 = wq * {02} */
- w20 = MASK(wq0);
- w21 = MASK(wq1);
- w10 = SHLBYTE(wq0);
- w11 = SHLBYTE(wq1);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
+ q0 = x2_32(q0);
+ q1 = x2_32(q1);
- /* wq = w1 + wd */
- wq0 = w10 ^ wd0;
- wq1 = w11 ^ wd1;
+ q0 ^= d0;
+ q1 ^= d1;
- /* w1 = wr * {02} */
- w20 = MASK(wr0);
- w21 = MASK(wr1);
- w10 = SHLBYTE(wr0);
- w11 = SHLBYTE(wr1);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
+ r0 = x2_32(r0);
+ r1 = x2_32(r1);
+ r0 = x2_32(r0);
+ r1 = x2_32(r1);
- /* w1 = w1 * {02} */
- w20 = MASK(w10);
- w21 = MASK(w11);
- w10 = SHLBYTE(w10);
- w11 = SHLBYTE(w11);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
-
- /* wr = w1 + wd */
- wr0 = w10 ^ wd0;
- wr1 = w11 ^ wd1;
+ r0 ^= d0;
+ r1 ^= d1;
}
- *(uint32_t *)&p[d+4*0] = wp0;
- *(uint32_t *)&p[d+4*1] = wp1;
- *(uint32_t *)&q[d+4*0] = wq0;
- *(uint32_t *)&q[d+4*1] = wq1;
- *(uint32_t *)&r[d+4*0] = wr0;
- *(uint32_t *)&r[d+4*1] = wr1;
+ v_32(p[o]) = p0;
+ v_32(p[o+4]) = p1;
+ v_32(q[o]) = q0;
+ v_32(q[o+4]) = q1;
+ v_32(r[o]) = r0;
+ v_32(r[o+4]) = r1;
+ }
+}
+
+/*
+ * RAIDTP unrolled-by-2 C implementation
+ */
+void raidTP_int64r2(unsigned char** vbuf, unsigned data, unsigned size)
+{
+ unsigned char* p;
+ unsigned char* q;
+ unsigned char* r;
+ int d, l;
+ unsigned o;
+
+ uint64_t d0, r0, q0, p0;
+ uint64_t d1, r1, q1, p1;
+
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
+
+ for(o=0;o=0;--d) {
+ d0 = v_64(vbuf[d][o]);
+ d1 = v_64(vbuf[d][o+8]);
+
+ p0 ^= d0;
+ p1 ^= d1;
+
+ q0 = x2_64(q0);
+ q1 = x2_64(q1);
+
+ q0 ^= d0;
+ q1 ^= d1;
+
+ r0 = x2_64(r0);
+ r1 = x2_64(r1);
+ r0 = x2_64(r0);
+ r1 = x2_64(r1);
+
+ r0 ^= d0;
+ r1 ^= d1;
+ }
+ v_64(p[o]) = p0;
+ v_64(p[o+8]) = p1;
+ v_64(q[o]) = q0;
+ v_64(q[o+8]) = q1;
+ v_64(r[o]) = r0;
+ v_64(r[o+8]) = r1;
}
}
@@ -600,29 +641,29 @@ void raidTP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size)
/*
* RAIDTP unrolled-by-1 MMX implementation
*/
-void raidTP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
- asm volatile("movq %0,%%mm7" : : "m" (raid6_mmx_constants.x1d));
+ asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly));
asm volatile("pxor %mm4,%mm4");
asm volatile("pxor %mm5,%mm5");
asm volatile("pxor %mm6,%mm6");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %mm1,%mm4");
asm volatile("pcmpgtb %mm2,%mm5");
asm volatile("paddb %mm1,%mm1");
@@ -635,17 +676,19 @@ void raidTP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("paddb %mm2,%mm2");
asm volatile("pand %mm7,%mm6");
asm volatile("pxor %mm6,%mm2");
- asm volatile("movq %0,%%mm3" : : "m" (buffer[z][d]));
+
+ asm volatile("movq %0,%%mm3" : : "m" (vbuf[d][o]));
asm volatile("pxor %mm3,%mm0");
asm volatile("pxor %mm3,%mm1");
asm volatile("pxor %mm3,%mm2");
+
asm volatile("pxor %mm4,%mm4");
asm volatile("pxor %mm5,%mm5");
asm volatile("pxor %mm6,%mm6");
}
- asm volatile("movq %%mm0,%0" : "=m" (p[d]));
- asm volatile("movq %%mm1,%0" : "=m" (q[d]));
- asm volatile("movq %%mm2,%0" : "=m" (r[d]));
+ asm volatile("movq %%mm0,%0" : "=m" (p[o]));
+ asm volatile("movq %%mm1,%0" : "=m" (q[o]));
+ asm volatile("movq %%mm2,%0" : "=m" (r[o]));
}
asm volatile("emms" : : : "memory");
@@ -656,29 +699,29 @@ void raidTP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
/*
* RAIDTP unrolled-by-1 SSE2 implementation
*/
-void raidTP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
- asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0]));
+ asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0]));
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
asm volatile("pxor %xmm6,%xmm6");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %xmm1,%xmm4");
asm volatile("pcmpgtb %xmm2,%xmm5");
asm volatile("paddb %xmm1,%xmm1");
@@ -691,17 +734,19 @@ void raidTP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("paddb %xmm2,%xmm2");
asm volatile("pand %xmm7,%xmm6");
asm volatile("pxor %xmm6,%xmm2");
- asm volatile("movdqa %0,%%xmm3" : : "m" (buffer[z][d]));
+
+ asm volatile("movdqa %0,%%xmm3" : : "m" (vbuf[d][o]));
asm volatile("pxor %xmm3,%xmm0");
asm volatile("pxor %xmm3,%xmm1");
asm volatile("pxor %xmm3,%xmm2");
+
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
asm volatile("pxor %xmm6,%xmm6");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (r[d]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (q[o]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (r[o]));
}
asm volatile("sfence" : : : "memory");
@@ -711,22 +756,22 @@ void raidTP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
#if defined(__x86_64__)
/*
* RAIDTP unrolled-by-2 SSE2 implementation
- * Note that it uses all the 16 registers, meaning that x64 is required.
+ * Note that it uses 16 registers, meaning that x64 is required.
*/
-void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
- asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0]));
+ asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0]));
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
asm volatile("pxor %xmm6,%xmm6");
@@ -734,14 +779,14 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm13,%xmm13");
asm volatile("pxor %xmm14,%xmm14");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %xmm1,%xmm4");
asm volatile("pcmpgtb %xmm2,%xmm5");
asm volatile("pcmpgtb %xmm9,%xmm12");
@@ -766,14 +811,16 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pand %xmm7,%xmm14");
asm volatile("pxor %xmm6,%xmm2");
asm volatile("pxor %xmm14,%xmm10");
- asm volatile("movdqa %0,%%xmm3" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm11" : : "m" (buffer[z][d+16]));
+
+ asm volatile("movdqa %0,%%xmm3" : : "m" (vbuf[d][o]));
+ asm volatile("movdqa %0,%%xmm11" : : "m" (vbuf[d][o+16]));
asm volatile("pxor %xmm3,%xmm0");
asm volatile("pxor %xmm3,%xmm1");
asm volatile("pxor %xmm3,%xmm2");
asm volatile("pxor %xmm11,%xmm8");
asm volatile("pxor %xmm11,%xmm9");
asm volatile("pxor %xmm11,%xmm10");
+
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
asm volatile("pxor %xmm6,%xmm6");
@@ -781,12 +828,12 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm13,%xmm13");
asm volatile("pxor %xmm14,%xmm14");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm8,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm9,%0" : "=m" (q[d+16]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (r[d]));
- asm volatile("movntdq %%xmm10,%0" : "=m" (r[d+16]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (q[o]));
+ asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (r[o]));
+ asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16]));
}
asm volatile("sfence" : : : "memory");
@@ -796,116 +843,132 @@ void raidTP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
/*
* RAIDQP unrolled-by-2 C implementation
*/
-void raidQP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidQP_int32r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
unsigned char* s;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- uint32_t wd0, ws0, wr0, wq0, wp0, w10, w20;
- uint32_t wd1, ws1, wr1, wq1, wp1, w11, w21;
+ uint32_t d0, s0, r0, q0, p0;
+ uint32_t d1, s1, r1, q1, p1;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
- s = buffer[diskmax+3];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
+ s = vbuf[data+3];
- for(d=0;d= 0;--z) {
- wd0 = *(uint32_t *)&buffer[z][d+0*4];
- wd1 = *(uint32_t *)&buffer[z][d+1*4];
+ for(o=0;o=0;--d) {
+ d0 = v_32(vbuf[d][o]);
+ d1 = v_32(vbuf[d][o+4]);
- /* wp += wd */
- wp0 ^= wd0;
- wp1 ^= wd1;
+ p0 ^= d0;
+ p1 ^= d1;
- /* w1 = wq * {02} */
- w20 = MASK(wq0);
- w21 = MASK(wq1);
- w10 = SHLBYTE(wq0);
- w11 = SHLBYTE(wq1);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
+ q0 = x2_32(q0);
+ q1 = x2_32(q1);
- /* wq = w1 + wd */
- wq0 = w10 ^ wd0;
- wq1 = w11 ^ wd1;
+ q0 ^= d0;
+ q1 ^= d1;
- /* w1 = wr * {02} */
- w20 = MASK(wr0);
- w21 = MASK(wr1);
- w10 = SHLBYTE(wr0);
- w11 = SHLBYTE(wr1);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
+ r0 = x2_32(r0);
+ r1 = x2_32(r1);
+ r0 = x2_32(r0);
+ r1 = x2_32(r1);
- /* w1 = w1 * {02} */
- w20 = MASK(w10);
- w21 = MASK(w11);
- w10 = SHLBYTE(w10);
- w11 = SHLBYTE(w11);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
+ r0 ^= d0;
+ r1 ^= d1;
- /* wr = w1 + wd */
- wr0 = w10 ^ wd0;
- wr1 = w11 ^ wd1;
-
- /* w1 = ws * {02} */
- w20 = MASK(ws0);
- w21 = MASK(ws1);
- w10 = SHLBYTE(ws0);
- w11 = SHLBYTE(ws1);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
-
- /* w1 = w1 * {02} */
- w20 = MASK(w10);
- w21 = MASK(w11);
- w10 = SHLBYTE(w10);
- w11 = SHLBYTE(w11);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
-
- /* w1 = w1 * {02} */
- w20 = MASK(w10);
- w21 = MASK(w11);
- w10 = SHLBYTE(w10);
- w11 = SHLBYTE(w11);
- w20 &= NBYTES(0x1d);
- w21 &= NBYTES(0x1d);
- w10 ^= w20;
- w11 ^= w21;
-
- /* ws = w1 + wd */
- ws0 = w10 ^ wd0;
- ws1 = w11 ^ wd1;
+ s0 = x2_32(s0);
+ s1 = x2_32(s1);
+ s0 = x2_32(s0);
+ s1 = x2_32(s1);
+ s0 = x2_32(s0);
+ s1 = x2_32(s1);
+ s0 ^= d0;
+ s1 ^= d1;
}
- *(uint32_t *)&p[d+4*0] = wp0;
- *(uint32_t *)&p[d+4*1] = wp1;
- *(uint32_t *)&q[d+4*0] = wq0;
- *(uint32_t *)&q[d+4*1] = wq1;
- *(uint32_t *)&r[d+4*0] = wr0;
- *(uint32_t *)&r[d+4*1] = wr1;
- *(uint32_t *)&s[d+4*0] = ws0;
- *(uint32_t *)&s[d+4*1] = ws1;
+ v_32(p[o]) = p0;
+ v_32(p[o+4]) = p1;
+ v_32(q[o]) = q0;
+ v_32(q[o+4]) = q1;
+ v_32(r[o]) = r0;
+ v_32(r[o+4]) = r1;
+ v_32(s[o]) = s0;
+ v_32(s[o+4]) = s1;
+ }
+}
+
+/*
+ * RAIDQP unrolled-by-2 C implementation
+ */
+void raidQP_int64r2(unsigned char** vbuf, unsigned data, unsigned size)
+{
+ unsigned char* p;
+ unsigned char* q;
+ unsigned char* r;
+ unsigned char* s;
+ int d, l;
+ unsigned o;
+
+ uint64_t d0, s0, r0, q0, p0;
+ uint64_t d1, s1, r1, q1, p1;
+
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
+ s = vbuf[data+3];
+
+ for(o=0;o=0;--d) {
+ d0 = v_64(vbuf[d][o]);
+ d1 = v_64(vbuf[d][o+8]);
+
+ p0 ^= d0;
+ p1 ^= d1;
+
+ q0 = x2_64(q0);
+ q1 = x2_64(q1);
+
+ q0 ^= d0;
+ q1 ^= d1;
+
+ r0 = x2_64(r0);
+ r1 = x2_64(r1);
+ r0 = x2_64(r0);
+ r1 = x2_64(r1);
+
+ r0 ^= d0;
+ r1 ^= d1;
+
+ s0 = x2_64(s0);
+ s1 = x2_64(s1);
+ s0 = x2_64(s0);
+ s1 = x2_64(s1);
+ s0 = x2_64(s0);
+ s1 = x2_64(s1);
+
+ s0 ^= d0;
+ s1 ^= d1;
+ }
+ v_64(p[o]) = p0;
+ v_64(p[o+8]) = p1;
+ v_64(q[o]) = q0;
+ v_64(q[o+8]) = q1;
+ v_64(r[o]) = r0;
+ v_64(r[o+8]) = r1;
+ v_64(s[o]) = s0;
+ v_64(s[o+8]) = s1;
}
}
@@ -913,31 +976,31 @@ void raidQP_int32r2(unsigned char** buffer, unsigned diskmax, unsigned size)
/*
* RAIDQP unrolled-by-1 MMX implementation
*/
-void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
unsigned char* s;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
- s = buffer[diskmax+3];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
+ s = vbuf[data+3];
- asm volatile("movq %0,%%mm7" : : "m" (raid6_mmx_constants.x1d));
+ asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly));
asm volatile("pxor %mm4,%mm4");
asm volatile("pxor %mm5,%mm5");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %mm1,%mm4");
asm volatile("pcmpgtb %mm3,%mm5");
asm volatile("paddb %mm1,%mm1");
@@ -971,7 +1034,7 @@ void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %mm4,%mm2");
asm volatile("pxor %mm5,%mm3");
- asm volatile("movq %0,%%mm6" : : "m" (buffer[z][d]));
+ asm volatile("movq %0,%%mm6" : : "m" (vbuf[d][o]));
asm volatile("pxor %mm6,%mm0");
asm volatile("pxor %mm6,%mm1");
asm volatile("pxor %mm6,%mm2");
@@ -980,10 +1043,10 @@ void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %mm4,%mm4");
asm volatile("pxor %mm5,%mm5");
}
- asm volatile("movq %%mm0,%0" : "=m" (p[d]));
- asm volatile("movq %%mm1,%0" : "=m" (q[d]));
- asm volatile("movq %%mm2,%0" : "=m" (r[d]));
- asm volatile("movq %%mm3,%0" : "=m" (s[d]));
+ asm volatile("movq %%mm0,%0" : "=m" (p[o]));
+ asm volatile("movq %%mm1,%0" : "=m" (q[o]));
+ asm volatile("movq %%mm2,%0" : "=m" (r[o]));
+ asm volatile("movq %%mm3,%0" : "=m" (s[o]));
}
asm volatile("emms" : : : "memory");
@@ -994,31 +1057,31 @@ void raidQP_mmxr1(unsigned char** buffer, unsigned diskmax, unsigned size)
/*
* RAIDQP unrolled-by-1 SSE2 implementation
*/
-void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
unsigned char* s;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
- s = buffer[diskmax+3];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
+ s = vbuf[data+3];
- asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0]));
+ asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0]));
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %xmm1,%xmm4");
asm volatile("pcmpgtb %xmm3,%xmm5");
asm volatile("paddb %xmm1,%xmm1");
@@ -1052,7 +1115,7 @@ void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm4,%xmm2");
asm volatile("pxor %xmm5,%xmm3");
- asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d]));
+ asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o]));
asm volatile("pxor %xmm6,%xmm0");
asm volatile("pxor %xmm6,%xmm1");
asm volatile("pxor %xmm6,%xmm2");
@@ -1061,10 +1124,10 @@ void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (r[d]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (s[d]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (q[o]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (r[o]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (s[o]));
}
asm volatile("sfence" : : : "memory");
@@ -1074,39 +1137,39 @@ void raidQP_sse2r1(unsigned char** buffer, unsigned diskmax, unsigned size)
#if defined(__x86_64__)
/*
* RAIDQP unrolled-by-2 SSE2 implementation
- * Note that it uses all the 16 registers, meaning that x64 is required.
+ * Note that it uses 16 registers, meaning that x64 is required.
*/
-void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
+void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size)
{
unsigned char* p;
unsigned char* q;
unsigned char* r;
unsigned char* s;
- int z, z0;
- unsigned d;
+ int d, l;
+ unsigned o;
- z0 = diskmax - 1;
- p = buffer[diskmax];
- q = buffer[diskmax+1];
- r = buffer[diskmax+2];
- s = buffer[diskmax+3];
+ l = data - 1;
+ p = vbuf[data];
+ q = vbuf[data+1];
+ r = vbuf[data+2];
+ s = vbuf[data+3];
- asm volatile("movdqa %0,%%xmm7" : : "m" (raid6_sse_constants.x1d[0]));
+ asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0]));
asm volatile("pxor %xmm4,%xmm4");
asm volatile("pxor %xmm5,%xmm5");
asm volatile("pxor %xmm12,%xmm12");
asm volatile("pxor %xmm13,%xmm13");
- for(d=0;d=0;--z) {
+ for(o=0;o=0;--d) {
asm volatile("pcmpgtb %xmm1,%xmm4");
asm volatile("pcmpgtb %xmm3,%xmm5");
asm volatile("pcmpgtb %xmm9,%xmm12");
@@ -1168,8 +1231,8 @@ void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm12,%xmm10");
asm volatile("pxor %xmm13,%xmm11");
- asm volatile("movdqa %0,%%xmm6" : : "m" (buffer[z][d]));
- asm volatile("movdqa %0,%%xmm14" : : "m" (buffer[z][d+16]));
+ asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o]));
+ asm volatile("movdqa %0,%%xmm14" : : "m" (vbuf[d][o+16]));
asm volatile("pxor %xmm6,%xmm0");
asm volatile("pxor %xmm6,%xmm1");
asm volatile("pxor %xmm6,%xmm2");
@@ -1184,14 +1247,14 @@ void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
asm volatile("pxor %xmm12,%xmm12");
asm volatile("pxor %xmm13,%xmm13");
}
- asm volatile("movntdq %%xmm0,%0" : "=m" (p[d]));
- asm volatile("movntdq %%xmm8,%0" : "=m" (p[d+16]));
- asm volatile("movntdq %%xmm1,%0" : "=m" (q[d]));
- asm volatile("movntdq %%xmm9,%0" : "=m" (q[d+16]));
- asm volatile("movntdq %%xmm2,%0" : "=m" (r[d]));
- asm volatile("movntdq %%xmm10,%0" : "=m" (r[d+16]));
- asm volatile("movntdq %%xmm3,%0" : "=m" (s[d]));
- asm volatile("movntdq %%xmm11,%0" : "=m" (s[d+16]));
+ asm volatile("movntdq %%xmm0,%0" : "=m" (p[o]));
+ asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16]));
+ asm volatile("movntdq %%xmm1,%0" : "=m" (q[o]));
+ asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16]));
+ asm volatile("movntdq %%xmm2,%0" : "=m" (r[o]));
+ asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16]));
+ asm volatile("movntdq %%xmm3,%0" : "=m" (s[o]));
+ asm volatile("movntdq %%xmm11,%0" : "=m" (s[o+16]));
}
asm volatile("sfence" : : : "memory");
@@ -1199,21 +1262,21 @@ void raidQP_sse2r2(unsigned char** buffer, unsigned diskmax, unsigned size)
#endif
/****************************************************************************/
-/* generic computation */
+/* parity generation */
/* internal forwarder */
-static void (*raid5_gen)(unsigned char** buffer, unsigned diskmax, unsigned size);
-static void (*raid6_gen)(unsigned char** buffer, unsigned diskmax, unsigned size);
-static void (*raidTP_gen)(unsigned char** buffer, unsigned diskmax, unsigned size);
-static void (*raidQP_gen)(unsigned char** buffer, unsigned diskmax, unsigned size);
+static void (*raid5_gen)(unsigned char** vbuf, unsigned data, unsigned size);
+static void (*raid6_gen)(unsigned char** vbuf, unsigned data, unsigned size);
+static void (*raidTP_gen)(unsigned char** vbuf, unsigned data, unsigned size);
+static void (*raidQP_gen)(unsigned char** vbuf, unsigned data, unsigned size);
-void raid_gen(unsigned level, unsigned char** buffer, unsigned diskmax, unsigned size)
+void raid_gen(unsigned level, unsigned char** vbuf, unsigned data, unsigned size)
{
switch (level) {
- case 1 : raid5_gen(buffer, diskmax, size); break;
- case 2 : raid6_gen(buffer, diskmax, size); break;
- case 3 : raidTP_gen(buffer, diskmax, size); break;
- case 4 : raidQP_gen(buffer, diskmax, size); break;
+ case 1 : raid5_gen(vbuf, data, size); break;
+ case 2 : raid6_gen(vbuf, data, size); break;
+ case 3 : raidTP_gen(vbuf, data, size); break;
+ case 4 : raidQP_gen(vbuf, data, size); break;
}
}
@@ -1222,10 +1285,12 @@ void raid_gen(unsigned level, unsigned char** buffer, unsigned diskmax, unsigned
/**
* The data recovering is based on the paper "The mathematics of RAID-6" [1],
- * that covers the RAID5 and RAID6 computations in the Galois Field GF(2^8)
- * with the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 (285 decimal),
- * using the following equations to compute syndrome P and Q from a set
- * of n disk Di with 0<=i