From 823bc8a8c0669ae596d8a90be816600d5306a3ae Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 6 Oct 2013 16:06:17 +0200 Subject: [PATCH] Adds SSSE3 and AVX parity computation. --- HISTORY | 4 + cpu.h | 18 ++ mktables.c | 47 ++++ raid.c | 791 ++++++++++++++++++++++++++++++++++++++++++----------- raid.h | 53 ++-- selftest.c | 55 ++-- speed.c | 220 ++++++--------- tables.h | 8 + 8 files changed, 871 insertions(+), 325 deletions(-) diff --git a/HISTORY b/HISTORY index c35faf78..162751d6 100644 --- a/HISTORY +++ b/HISTORY @@ -10,6 +10,8 @@ SnapRAID HISTORY up to 89 disks with Hexa (6) parity is possible, but not yet implemented. Note that more general approaches without these limitations are also possible, but they would affect badly the performance. + Parity computation uses SSSE3 and AVX instructions if available for the + best performance. * In sync, if a silent error is found, directly mark the block as bad just like in scrub, and continue with the sync process. * Sort files by inode when listing the directory. This improves @@ -20,6 +22,8 @@ SnapRAID HISTORY of more info. * For files with changes only in some blocks, updates the parity only for blocks that really are changed. + This improves the performance in sync for edited files. + * Faster parity computation for ARM 64 bit CPU. * Reduced memory usage avoiding to allocate unnecessary memory. 4.2 2013/10 (BETA) diff --git a/cpu.h b/cpu.h index b7ec1cc9..a1ed93f0 100644 --- a/cpu.h +++ b/cpu.h @@ -91,6 +91,15 @@ static inline int cpu_has_sse2(void) return (reg[3] >> 26) & 1; } +static inline int cpu_has_ssse3(void) +{ + uint32_t reg[4]; + + cpuid(1, reg); + + return (reg[2] >> 9) & 1; +} + static inline int cpu_has_sse42(void) { uint32_t reg[4]; @@ -100,6 +109,15 @@ static inline int cpu_has_sse42(void) return (reg[2] >> 20) & 1; } +static inline int cpu_has_avx(void) +{ + uint32_t reg[4]; + + cpuid(1, reg); + + return (reg[2] >> 28) & 1; +} + /** * Check if the processor has a slow MULT implementation. * If yes, it's better to use a hash not based on multiplication. diff --git a/mktables.c b/mktables.c index ee5d7c0b..28d7e2e0 100644 --- a/mktables.c +++ b/mktables.c @@ -133,6 +133,53 @@ int main(void) } printf("};\n\n"); + /* x4l */ + printf("#define GFX4L { "); + for(i=0;i<16;++i) { + v = gfmul(0x4, i); + printf("0x%02x", (unsigned)v); + if (i != 15) + printf(", "); + } + printf(" }\n"); + + /* x4h */ + printf("#define GFX4H { "); + for(i=0;i<16;++i) { + v = gfmul(0x40, i); + printf("0x%02x", (unsigned)v); + if (i != 15) + printf(", "); + } + printf(" }\n"); + + /* x8l */ + printf("#define GFX8L { "); + for(i=0;i<16;++i) { + v = gfmul(0x8, i); + printf("0x%02x", (unsigned)v); + if (i != 15) + printf(", "); + } + printf(" }\n"); + + /* x8h */ + printf("#define GFX8H { "); + for(i=0;i<16;++i) { + v = gfmul(0x80, i); + printf("0x%02x", (unsigned)v); + if (i != 15) + printf(", "); + } + printf(" }\n"); + + /* poly */ + printf("#define GFPOLY8 { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d } \n"); + printf("#define GFPOLY16 { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d } \n"); + printf("#define GFMASK16 { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f } \n"); + + printf("\n"); + return 0; } diff --git a/raid.c b/raid.c index 9c7fa10e..5b5f71f9 100644 --- a/raid.c +++ b/raid.c @@ -48,9 +48,9 @@ #define v_64(p) (*(uint64_t*)&(p)) /* - * RAID5 unrolled-by-2 C implementation + * RAID5 32bit C implementation */ -void raid5_int32r2(unsigned char** vbuf, unsigned data, unsigned size) +void raid5_int32(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; int d, l; @@ -75,9 +75,9 @@ void raid5_int32r2(unsigned char** vbuf, unsigned data, unsigned size) } /* - * RAID5 unrolled-by-2 C implementation + * RAID5 64bit C implementation */ -void raid5_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +void raid5_int64(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; int d, l; @@ -103,9 +103,9 @@ void raid5_int64r2(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__i386__) || defined(__x86_64__) /* - * RAID5 unrolled-by-4 MMX implementation + * RAID5 MMX implementation */ -void raid5_mmxr4(unsigned char** vbuf, unsigned data, unsigned size) +void raid5_mmx(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; int d, l; @@ -139,9 +139,9 @@ void raid5_mmxr4(unsigned char** vbuf, unsigned data, unsigned size) } /* - * RAID5 unrolled-by-4 SSE2 implementation + * RAID5 SSE2 implementation */ -void raid5_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) +void raid5_sse2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; int d, l; @@ -177,10 +177,10 @@ void raid5_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__x86_64__) /* - * RAID5 unrolled-by-8 SSE2 implementation + * RAID5 SSE2 implementation * Note that it uses 16 registers, meaning that x64 is required. */ -void raid5_sse2r8(unsigned char** vbuf, unsigned data, unsigned size) +void raid5_sse2ext(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; int d, l; @@ -255,9 +255,9 @@ __attribute__((always_inline)) static inline uint64_t x2_64(uint64_t v) } /* - * RAID6 unrolled-by-2 C implementation + * RAID6 32bit C implementation */ -void raid6_int32r2(unsigned char** vbuf, unsigned data, unsigned size) +void raid6_int32(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -295,9 +295,9 @@ void raid6_int32r2(unsigned char** vbuf, unsigned data, unsigned size) } /* - * RAID6 unrolled-by-2 C implementation + * RAID6 64bit C implementation */ -void raid6_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +void raid6_int64(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -335,16 +335,16 @@ void raid6_int64r2(unsigned char** vbuf, unsigned data, unsigned size) } #if defined(__i386__) || defined(__x86_64__) -static const struct raid_mmx_poly { - uint64_t poly; -} raid_mmx_poly = { - 0x1d1d1d1d1d1d1d1dULL, +static const struct raid_8_const { + unsigned char poly[8]; +} raid_8_const __attribute__((aligned(8))) = { + GFPOLY8 }; /* - * RAID6 unrolled-by-2 MMX implementation + * RAID6 MMX implementation */ -void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size) +void raid6_mmx(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -355,9 +355,7 @@ void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size) p = vbuf[data]; q = vbuf[data+1]; - asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly)); - asm volatile("pxor %mm4,%mm4"); - asm volatile("pxor %mm5,%mm5"); + asm volatile("movq %0,%%mm7" : : "m" (raid_8_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %mm4,%mm4"); + asm volatile("pxor %mm5,%mm5"); asm volatile("pcmpgtb %mm2,%mm4"); asm volatile("pcmpgtb %mm3,%mm5"); asm volatile("paddb %mm2,%mm2"); @@ -380,9 +380,6 @@ void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %mm5,%mm1"); asm volatile("pxor %mm4,%mm2"); asm volatile("pxor %mm5,%mm3"); - - asm volatile("pxor %mm4,%mm4"); - asm volatile("pxor %mm5,%mm5"); } asm volatile("movq %%mm0,%0" : "=m" (p[o])); asm volatile("movq %%mm1,%0" : "=m" (p[o+8])); @@ -393,16 +390,26 @@ void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("emms" : : : "memory"); } -static const struct raid_sse_poly { - uint64_t poly[2]; -} raid_sse_poly __attribute__((aligned(16))) = { - { 0x1d1d1d1d1d1d1d1dULL, 0x1d1d1d1d1d1d1d1dULL }, +static const struct raid_16_const { + unsigned char poly[16]; + unsigned char low_nibble[16]; + unsigned char x8l[16]; + unsigned char x8h[16]; + unsigned char x4l[16]; + unsigned char x4h[16]; +} raid_16_const __attribute__((aligned(16))) = { + GFPOLY16, + GFMASK16, + GFX8L, + GFX8H, + GFX4L, + GFX4H, }; /* - * RAID6 unrolled-by-2 SSE2 implementation + * RAID6 SSE2 implementation */ -void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) +void raid6_sse2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -413,9 +420,7 @@ void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) p = vbuf[data]; q = vbuf[data+1]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm2,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("paddb %xmm2,%xmm2"); @@ -438,9 +445,6 @@ void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm5,%xmm1"); asm volatile("pxor %xmm4,%xmm2"); asm volatile("pxor %xmm5,%xmm3"); - - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); } asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16])); @@ -454,10 +458,10 @@ void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__x86_64__) /* - * RAID6 unrolled-by-4 SSE2 implementation + * RAID6 SSE2 implementation * Note that it uses 16 registers, meaning that x64 is required. */ -void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) +void raid6_sse2ext(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -468,11 +472,7 @@ void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) p = vbuf[data]; q = vbuf[data+1]; - asm volatile("movdqa %0,%%xmm15" : : "m" (raid_sse_poly.poly[0])); - asm volatile("pxor %xmm8,%xmm8"); - asm volatile("pxor %xmm9,%xmm9"); - asm volatile("pxor %xmm10,%xmm10"); - asm volatile("pxor %xmm11,%xmm11"); + asm volatile("movdqa %0,%%xmm15" : : "m" (raid_16_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %xmm8,%xmm8"); + asm volatile("pxor %xmm9,%xmm9"); + asm volatile("pxor %xmm10,%xmm10"); + asm volatile("pxor %xmm11,%xmm11"); asm volatile("pcmpgtb %xmm4,%xmm8"); asm volatile("pcmpgtb %xmm5,%xmm9"); asm volatile("pcmpgtb %xmm6,%xmm10"); @@ -513,11 +517,6 @@ void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm9,%xmm5"); asm volatile("pxor %xmm10,%xmm6"); asm volatile("pxor %xmm11,%xmm7"); - - asm volatile("pxor %xmm8,%xmm8"); - asm volatile("pxor %xmm9,%xmm9"); - asm volatile("pxor %xmm10,%xmm10"); - asm volatile("pxor %xmm11,%xmm11"); } asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); asm volatile("movntdq %%xmm1,%0" : "=m" (p[o+16])); @@ -534,9 +533,9 @@ void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size) #endif /* - * RAIDTP unrolled-by-2 C implementation + * RAIDTP 32bit C implementation */ -void raidTP_int32r2(unsigned char** vbuf, unsigned data, unsigned size) +void raidTP_int32(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -586,9 +585,9 @@ void raidTP_int32r2(unsigned char** vbuf, unsigned data, unsigned size) } /* - * RAIDTP unrolled-by-2 C implementation + * RAIDTP 64bit C implementation */ -void raidTP_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +void raidTP_int64(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -639,9 +638,9 @@ void raidTP_int64r2(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__i386__) || defined(__x86_64__) /* - * RAIDTP unrolled-by-1 MMX implementation + * RAIDTP MMX implementation */ -void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) +void raidTP_mmx(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -654,16 +653,16 @@ void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) q = vbuf[data+1]; r = vbuf[data+2]; - asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly)); - asm volatile("pxor %mm4,%mm4"); - asm volatile("pxor %mm5,%mm5"); - asm volatile("pxor %mm6,%mm6"); + asm volatile("movq %0,%%mm7" : : "m" (raid_8_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %mm4,%mm4"); + asm volatile("pxor %mm5,%mm5"); + asm volatile("pxor %mm6,%mm6"); asm volatile("pcmpgtb %mm1,%mm4"); asm volatile("pcmpgtb %mm2,%mm5"); asm volatile("paddb %mm1,%mm1"); @@ -681,10 +680,6 @@ void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %mm3,%mm0"); asm volatile("pxor %mm3,%mm1"); asm volatile("pxor %mm3,%mm2"); - - asm volatile("pxor %mm4,%mm4"); - asm volatile("pxor %mm5,%mm5"); - asm volatile("pxor %mm6,%mm6"); } asm volatile("movq %%mm0,%0" : "=m" (p[o])); asm volatile("movq %%mm1,%0" : "=m" (q[o])); @@ -697,9 +692,9 @@ void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__i386__) || defined(__x86_64__) /* - * RAIDTP unrolled-by-1 SSE2 implementation + * RAIDTP SSE2 implementation */ -void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) +void raidTP_sse2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -712,16 +707,16 @@ void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) q = vbuf[data+1]; r = vbuf[data+2]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pxor %xmm6,%xmm6"); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm5,%xmm5"); + asm volatile("pxor %xmm6,%xmm6"); asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm2,%xmm5"); asm volatile("paddb %xmm1,%xmm1"); @@ -739,10 +734,6 @@ void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm3,%xmm0"); asm volatile("pxor %xmm3,%xmm1"); asm volatile("pxor %xmm3,%xmm2"); - - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pxor %xmm6,%xmm6"); } asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); @@ -753,12 +744,11 @@ void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) } #endif -#if defined(__x86_64__) +#if defined(__i386__) || defined(__x86_64__) /* - * RAIDTP unrolled-by-2 SSE2 implementation - * Note that it uses 16 registers, meaning that x64 is required. + * RAIDTP SSSE3 implementation */ -void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) +void raidTP_ssse3(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -771,13 +761,64 @@ void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) q = vbuf[data+1]; r = vbuf[data+2]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pxor %xmm6,%xmm6"); - asm volatile("pxor %xmm12,%xmm12"); - asm volatile("pxor %xmm13,%xmm13"); - asm volatile("pxor %xmm14,%xmm14"); + asm volatile("movdqa %0,%%xmm3" : : "m" (raid_16_const.low_nibble[0])); + asm volatile("movdqa %0,%%xmm5" : : "m" (raid_16_const.x4l[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); + + for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pxor %xmm4,%xmm1"); + + asm volatile("movdqa %0,%%xmm6" : : "m" (raid_16_const.x4h[0])); + asm volatile("movdqa %xmm2,%xmm4"); + asm volatile("psraw $4,%xmm2"); + asm volatile("pand %xmm3,%xmm4"); + asm volatile("pand %xmm3,%xmm2"); + asm volatile("pshufb %xmm2,%xmm6"); + asm volatile("movdqa %xmm5,%xmm2"); + asm volatile("pshufb %xmm4,%xmm2"); + asm volatile("pxor %xmm6,%xmm2"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + } + + asm volatile("sfence" : : : "memory"); +} +#endif + +#if defined(__x86_64__) +/* + * RAIDTP SSE2 implementation + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raidTP_sse2ext(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + int d, l; + unsigned o; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm5,%xmm5"); + asm volatile("pxor %xmm6,%xmm6"); + asm volatile("pxor %xmm12,%xmm12"); + asm volatile("pxor %xmm13,%xmm13"); + asm volatile("pxor %xmm14,%xmm14"); asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm2,%xmm5"); asm volatile("pcmpgtb %xmm9,%xmm12"); @@ -820,13 +867,88 @@ void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm11,%xmm8"); asm volatile("pxor %xmm11,%xmm9"); asm volatile("pxor %xmm11,%xmm10"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16])); + } + asm volatile("sfence" : : : "memory"); +} +#endif + +#if defined(__x86_64__) +/* + * RAIDTP SSSE3 implementation + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raidTP_ssse3ext(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + int d, l; + unsigned o; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + + asm volatile("movdqa %0,%%xmm3" : : "m" (raid_16_const.low_nibble[0])); + asm volatile("movdqa %0,%%xmm5" : : "m" (raid_16_const.x4l[0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (raid_16_const.x4h[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); + + for(o=0;o=0;--d) { asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pxor %xmm6,%xmm6"); asm volatile("pxor %xmm12,%xmm12"); - asm volatile("pxor %xmm13,%xmm13"); - asm volatile("pxor %xmm14,%xmm14"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("pcmpgtb %xmm9,%xmm12"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("paddb %xmm9,%xmm9"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pand %xmm7,%xmm12"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm12,%xmm9"); + + asm volatile("movdqa %xmm6,%xmm13"); + asm volatile("movdqa %xmm6,%xmm14"); + asm volatile("movdqa %xmm2,%xmm4"); + asm volatile("movdqa %xmm10,%xmm12"); + asm volatile("psraw $4,%xmm2"); + asm volatile("psraw $4,%xmm10"); + asm volatile("pand %xmm3,%xmm4"); + asm volatile("pand %xmm3,%xmm12"); + asm volatile("pand %xmm3,%xmm2"); + asm volatile("pand %xmm3,%xmm10"); + asm volatile("pshufb %xmm2,%xmm13"); + asm volatile("pshufb %xmm10,%xmm14"); + asm volatile("movdqa %xmm5,%xmm2"); + asm volatile("movdqa %xmm5,%xmm10"); + asm volatile("pshufb %xmm4,%xmm2"); + asm volatile("pshufb %xmm12,%xmm10"); + asm volatile("pxor %xmm13,%xmm2"); + asm volatile("pxor %xmm14,%xmm10"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][o+16])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm12,%xmm8"); + asm volatile("pxor %xmm12,%xmm9"); + asm volatile("pxor %xmm12,%xmm10"); } asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); @@ -841,9 +963,9 @@ void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) #endif /* - * RAIDQP unrolled-by-2 C implementation + * RAIDQP 32bit C implementation */ -void raidQP_int32r2(unsigned char** vbuf, unsigned data, unsigned size) +void raidQP_int32(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -907,9 +1029,9 @@ void raidQP_int32r2(unsigned char** vbuf, unsigned data, unsigned size) } /* - * RAIDQP unrolled-by-2 C implementation + * RAIDQP 64bit C implementation */ -void raidQP_int64r2(unsigned char** vbuf, unsigned data, unsigned size) +void raidQP_int64(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -974,9 +1096,9 @@ void raidQP_int64r2(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__i386__) || defined(__x86_64__) /* - * RAIDQP unrolled-by-1 MMX implementation + * RAIDQP MMX implementation */ -void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) +void raidQP_mmx(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -991,9 +1113,7 @@ void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) r = vbuf[data+2]; s = vbuf[data+3]; - asm volatile("movq %0,%%mm7" : : "m" (raid_mmx_poly.poly)); - asm volatile("pxor %mm4,%mm4"); - asm volatile("pxor %mm5,%mm5"); + asm volatile("movq %0,%%mm7" : : "m" (raid_8_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %mm4,%mm4"); + asm volatile("pxor %mm5,%mm5"); asm volatile("pcmpgtb %mm1,%mm4"); asm volatile("pcmpgtb %mm3,%mm5"); asm volatile("paddb %mm1,%mm1"); @@ -1012,7 +1134,6 @@ void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); - asm volatile("pcmpgtb %mm2,%mm4"); asm volatile("pcmpgtb %mm3,%mm5"); asm volatile("paddb %mm2,%mm2"); @@ -1024,7 +1145,6 @@ void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %mm4,%mm4"); asm volatile("pxor %mm5,%mm5"); - asm volatile("pcmpgtb %mm2,%mm4"); asm volatile("pcmpgtb %mm3,%mm5"); asm volatile("paddb %mm2,%mm2"); @@ -1039,9 +1159,6 @@ void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %mm6,%mm1"); asm volatile("pxor %mm6,%mm2"); asm volatile("pxor %mm6,%mm3"); - - asm volatile("pxor %mm4,%mm4"); - asm volatile("pxor %mm5,%mm5"); } asm volatile("movq %%mm0,%0" : "=m" (p[o])); asm volatile("movq %%mm1,%0" : "=m" (q[o])); @@ -1055,9 +1172,9 @@ void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__i386__) || defined(__x86_64__) /* - * RAIDQP unrolled-by-1 SSE2 implementation + * RAIDQP SSE2 implementation */ -void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) +void raidQP_sse2(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -1072,9 +1189,7 @@ void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) r = vbuf[data+2]; s = vbuf[data+3]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("paddb %xmm1,%xmm1"); @@ -1093,7 +1210,6 @@ void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pcmpgtb %xmm2,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("paddb %xmm2,%xmm2"); @@ -1105,7 +1221,6 @@ void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pcmpgtb %xmm2,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("paddb %xmm2,%xmm2"); @@ -1115,14 +1230,82 @@ void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm4,%xmm2"); asm volatile("pxor %xmm5,%xmm3"); - asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o])); - asm volatile("pxor %xmm6,%xmm0"); - asm volatile("pxor %xmm6,%xmm1"); - asm volatile("pxor %xmm6,%xmm2"); - asm volatile("pxor %xmm6,%xmm3"); + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm4,%xmm3"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm3,%0" : "=m" (s[o])); + } + asm volatile("sfence" : : : "memory"); +} +#endif + +#if defined(__i386__) || defined(__x86_64__) +/* + * RAIDQP SSSE3 implementation + */ +void raidQP_ssse3(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + unsigned char* s; + int d, l; + unsigned o; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; + + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); + + for(o=0;o=0;--d) { asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("pcmpgtb %xmm2,%xmm5"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("paddb %xmm2,%xmm2"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pand %xmm7,%xmm5"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm5,%xmm2"); + + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pcmpgtb %xmm2,%xmm4"); + asm volatile("paddb %xmm2,%xmm2"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pxor %xmm4,%xmm2"); + + asm volatile("movdqa %0,%%xmm5" : : "m" (raid_16_const.x8l[0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (raid_16_const.x8h[0])); + asm volatile("movdqa %xmm3,%xmm4"); + asm volatile("psraw $4,%xmm3"); + asm volatile("pand %0,%%xmm4" : : "m" (raid_16_const.low_nibble[0])); + asm volatile("pand %0,%%xmm3" : : "m" (raid_16_const.low_nibble[0])); + asm volatile("pshufb %xmm3,%xmm6"); + asm volatile("movdqa %xmm5,%xmm3"); + asm volatile("pshufb %xmm4,%xmm3"); + asm volatile("pxor %xmm6,%xmm3"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm4,%xmm3"); } asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); @@ -1136,10 +1319,10 @@ void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size) #if defined(__x86_64__) /* - * RAIDQP unrolled-by-2 SSE2 implementation + * RAIDQP SSE2 implementation * Note that it uses 16 registers, meaning that x64 is required. */ -void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) +void raidQP_sse2ext(unsigned char** vbuf, unsigned data, unsigned size) { unsigned char* p; unsigned char* q; @@ -1154,11 +1337,7 @@ void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) r = vbuf[data+2]; s = vbuf[data+3]; - asm volatile("movdqa %0,%%xmm7" : : "m" (raid_sse_poly.poly[0])); - asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); - asm volatile("pxor %xmm12,%xmm12"); - asm volatile("pxor %xmm13,%xmm13"); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm5,%xmm5"); + asm volatile("pxor %xmm12,%xmm12"); + asm volatile("pxor %xmm13,%xmm13"); asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("pcmpgtb %xmm9,%xmm12"); @@ -1191,7 +1374,6 @@ void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm12,%xmm12"); asm volatile("pxor %xmm13,%xmm13"); - asm volatile("pcmpgtb %xmm2,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("pcmpgtb %xmm10,%xmm12"); @@ -1213,7 +1395,6 @@ void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm12,%xmm12"); asm volatile("pxor %xmm13,%xmm13"); - asm volatile("pcmpgtb %xmm2,%xmm4"); asm volatile("pcmpgtb %xmm3,%xmm5"); asm volatile("pcmpgtb %xmm10,%xmm12"); @@ -1231,21 +1412,228 @@ void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size) asm volatile("pxor %xmm12,%xmm10"); asm volatile("pxor %xmm13,%xmm11"); - asm volatile("movdqa %0,%%xmm6" : : "m" (vbuf[d][o])); - asm volatile("movdqa %0,%%xmm14" : : "m" (vbuf[d][o+16])); - asm volatile("pxor %xmm6,%xmm0"); - asm volatile("pxor %xmm6,%xmm1"); - asm volatile("pxor %xmm6,%xmm2"); - asm volatile("pxor %xmm6,%xmm3"); - asm volatile("pxor %xmm14,%xmm8"); - asm volatile("pxor %xmm14,%xmm9"); - asm volatile("pxor %xmm14,%xmm10"); - asm volatile("pxor %xmm14,%xmm11"); + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][o+16])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm4,%xmm3"); + asm volatile("pxor %xmm12,%xmm8"); + asm volatile("pxor %xmm12,%xmm9"); + asm volatile("pxor %xmm12,%xmm10"); + asm volatile("pxor %xmm12,%xmm11"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16])); + asm volatile("movntdq %%xmm3,%0" : "=m" (s[o])); + asm volatile("movntdq %%xmm11,%0" : "=m" (s[o+16])); + } + asm volatile("sfence" : : : "memory"); +} +#endif + +#if defined(__x86_64__) +/* + * RAIDQP SSSE3 implementation + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raidQP_ssse3ext(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + unsigned char* s; + int d, l; + unsigned o; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; + + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); + asm volatile("movdqa %0,%%xmm15" : : "m" (raid_16_const.low_nibble[0])); + + for(o=0;o=0;--d) { asm volatile("pxor %xmm4,%xmm4"); - asm volatile("pxor %xmm5,%xmm5"); asm volatile("pxor %xmm12,%xmm12"); - asm volatile("pxor %xmm13,%xmm13"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("pcmpgtb %xmm9,%xmm12"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("paddb %xmm9,%xmm9"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pand %xmm7,%xmm12"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm12,%xmm9"); + + asm volatile("movdqa %0,%%xmm5" : : "m" (raid_16_const.x4l[0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (raid_16_const.x4h[0])); + asm volatile("movdqa %xmm5,%xmm13"); + asm volatile("movdqa %xmm6,%xmm14"); + asm volatile("movdqa %xmm2,%xmm4"); + asm volatile("movdqa %xmm10,%xmm12"); + asm volatile("psraw $4,%xmm2"); + asm volatile("psraw $4,%xmm10"); + asm volatile("pand %xmm15,%xmm4"); + asm volatile("pand %xmm15,%xmm12"); + asm volatile("pand %xmm15,%xmm2"); + asm volatile("pand %xmm15,%xmm10"); + asm volatile("pshufb %xmm2,%xmm6"); + asm volatile("pshufb %xmm10,%xmm14"); + asm volatile("movdqa %xmm5,%xmm2"); + asm volatile("movdqa %xmm13,%xmm10"); + asm volatile("pshufb %xmm4,%xmm2"); + asm volatile("pshufb %xmm12,%xmm10"); + asm volatile("pxor %xmm6,%xmm2"); + asm volatile("pxor %xmm14,%xmm10"); + + asm volatile("movdqa %0,%%xmm5" : : "m" (raid_16_const.x8l[0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (raid_16_const.x8h[0])); + asm volatile("movdqa %xmm5,%xmm13"); + asm volatile("movdqa %xmm6,%xmm14"); + asm volatile("movdqa %xmm3,%xmm4"); + asm volatile("movdqa %xmm11,%xmm12"); + asm volatile("psraw $4,%xmm3"); + asm volatile("psraw $4,%xmm11"); + asm volatile("pand %xmm15,%xmm4"); + asm volatile("pand %xmm15,%xmm12"); + asm volatile("pand %xmm15,%xmm3"); + asm volatile("pand %xmm15,%xmm11"); + asm volatile("pshufb %xmm3,%xmm6"); + asm volatile("pshufb %xmm11,%xmm14"); + asm volatile("movdqa %xmm5,%xmm3"); + asm volatile("movdqa %xmm13,%xmm11"); + asm volatile("pshufb %xmm4,%xmm3"); + asm volatile("pshufb %xmm12,%xmm11"); + asm volatile("pxor %xmm6,%xmm3"); + asm volatile("pxor %xmm14,%xmm11"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][o+16])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm4,%xmm3"); + asm volatile("pxor %xmm12,%xmm8"); + asm volatile("pxor %xmm12,%xmm9"); + asm volatile("pxor %xmm12,%xmm10"); + asm volatile("pxor %xmm12,%xmm11"); + } + asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); + asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); + asm volatile("movntdq %%xmm1,%0" : "=m" (q[o])); + asm volatile("movntdq %%xmm9,%0" : "=m" (q[o+16])); + asm volatile("movntdq %%xmm2,%0" : "=m" (r[o])); + asm volatile("movntdq %%xmm10,%0" : "=m" (r[o+16])); + asm volatile("movntdq %%xmm3,%0" : "=m" (s[o])); + asm volatile("movntdq %%xmm11,%0" : "=m" (s[o+16])); + } + + asm volatile("sfence" : : : "memory"); +} + +/* + * RAIDQP AVX implementation + * Note that it uses 16 registers, meaning that x64 is required. + */ +void raidQP_avxext(unsigned char** vbuf, unsigned data, unsigned size) +{ + unsigned char* p; + unsigned char* q; + unsigned char* r; + unsigned char* s; + int d, l; + unsigned o; + + l = data - 1; + p = vbuf[data]; + q = vbuf[data+1]; + r = vbuf[data+2]; + s = vbuf[data+3]; + + asm volatile("movdqa %0,%%xmm5" : : "m" (raid_16_const.x8l[0])); + asm volatile("movdqa %0,%%xmm6" : : "m" (raid_16_const.x8h[0])); + asm volatile("movdqa %0,%%xmm7" : : "m" (raid_16_const.poly[0])); + asm volatile("movdqa %0,%%xmm13" : : "m" (raid_16_const.x4l[0])); + asm volatile("movdqa %0,%%xmm14" : : "m" (raid_16_const.x4h[0])); + asm volatile("movdqa %0,%%xmm15" : : "m" (raid_16_const.low_nibble[0])); + + for(o=0;o=0;--d) { + asm volatile("pxor %xmm4,%xmm4"); + asm volatile("pxor %xmm12,%xmm12"); + asm volatile("pcmpgtb %xmm1,%xmm4"); + asm volatile("pcmpgtb %xmm9,%xmm12"); + asm volatile("paddb %xmm1,%xmm1"); + asm volatile("paddb %xmm9,%xmm9"); + asm volatile("pand %xmm7,%xmm4"); + asm volatile("pand %xmm7,%xmm12"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm12,%xmm9"); + + asm volatile("movdqa %xmm2,%xmm4"); + asm volatile("movdqa %xmm10,%xmm12"); + asm volatile("psraw $4,%xmm2"); + asm volatile("psraw $4,%xmm10"); + asm volatile("pand %xmm15,%xmm4"); + asm volatile("pand %xmm15,%xmm12"); + asm volatile("pand %xmm15,%xmm2"); + asm volatile("pand %xmm15,%xmm10"); + asm volatile("vpshufb %xmm2,%xmm14,%xmm2"); + asm volatile("vpshufb %xmm10,%xmm14,%xmm10"); + asm volatile("vpshufb %xmm4,%xmm13,%xmm4"); + asm volatile("vpshufb %xmm12,%xmm13,%xmm12"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm12,%xmm10"); + + asm volatile("movdqa %xmm3,%xmm4"); + asm volatile("movdqa %xmm11,%xmm12"); + asm volatile("psraw $4,%xmm3"); + asm volatile("psraw $4,%xmm11"); + asm volatile("pand %xmm15,%xmm4"); + asm volatile("pand %xmm15,%xmm12"); + asm volatile("pand %xmm15,%xmm3"); + asm volatile("pand %xmm15,%xmm11"); + asm volatile("vpshufb %xmm3,%xmm6,%xmm3"); + asm volatile("vpshufb %xmm11,%xmm6,%xmm11"); + asm volatile("vpshufb %xmm4,%xmm5,%xmm4"); + asm volatile("vpshufb %xmm12,%xmm5,%xmm12"); + asm volatile("pxor %xmm4,%xmm3"); + asm volatile("pxor %xmm12,%xmm11"); + + asm volatile("movdqa %0,%%xmm4" : : "m" (vbuf[d][o])); + asm volatile("movdqa %0,%%xmm12" : : "m" (vbuf[d][o+16])); + asm volatile("pxor %xmm4,%xmm0"); + asm volatile("pxor %xmm4,%xmm1"); + asm volatile("pxor %xmm4,%xmm2"); + asm volatile("pxor %xmm4,%xmm3"); + asm volatile("pxor %xmm12,%xmm8"); + asm volatile("pxor %xmm12,%xmm9"); + asm volatile("pxor %xmm12,%xmm10"); + asm volatile("pxor %xmm12,%xmm11"); } asm volatile("movntdq %%xmm0,%0" : "=m" (p[o])); asm volatile("movntdq %%xmm8,%0" : "=m" (p[o+16])); @@ -2488,39 +2876,130 @@ void raid_recov_4data(int x, int y, int z, int v, int i, int j, int k, int l, un void raid_init(void) { if (sizeof(void*) == 4) { - raid5_gen = raid5_int32r2; - raid6_gen = raid6_int32r2; + raid5_gen = raid5_int32; + raid6_gen = raid6_int32; + raidTP_gen = raidTP_int32; + raidQP_gen = raidQP_int32; } else { - raid5_gen = raid5_int64r2; - raid6_gen = raid6_int64r2; + raid5_gen = raid5_int64; + raid6_gen = raid6_int64; + raidTP_gen = raidTP_int64; + raidQP_gen = raidQP_int64; } - raidTP_gen = raidTP_int32r2; - raidQP_gen = raidQP_int32r2; + #if defined(__i386__) || defined(__x86_64__) if (cpu_has_mmx()) { - raid5_gen = raid5_mmxr4; - raid6_gen = raid6_mmxr2; - raidTP_gen = raidTP_mmxr1; - raidQP_gen = raidQP_mmxr1; + raid5_gen = raid5_mmx; + raid6_gen = raid6_mmx; + raidTP_gen = raidTP_mmx; + raidQP_gen = raidQP_mmx; } + if (cpu_has_sse2()) { #if defined(__x86_64__) if (cpu_has_slowextendedreg()) { - raid5_gen = raid5_sse2r4; - raid6_gen = raid6_sse2r2; + raid5_gen = raid5_sse2; + raid6_gen = raid6_sse2; } else { - raid5_gen = raid5_sse2r8; - raid6_gen = raid6_sse2r4; + raid5_gen = raid5_sse2ext; + raid6_gen = raid6_sse2ext; } - raidTP_gen = raidTP_sse2r2; - raidQP_gen = raidQP_sse2r2; + raidTP_gen = raidTP_sse2ext; + raidQP_gen = raidQP_sse2ext; #else - raid5_gen = raid5_sse2r4; - raid6_gen = raid6_sse2r2; - raidTP_gen = raidTP_sse2r1; - raidQP_gen = raidQP_sse2r1; + raid5_gen = raid5_sse2; + raid6_gen = raid6_sse2; + raidTP_gen = raidTP_sse2; + raidQP_gen = raidQP_sse2; #endif } + + if (cpu_has_ssse3()) { +#if defined(__x86_64__) + raidTP_gen = raidTP_ssse3ext; + raidQP_gen = raidQP_ssse3ext; +#else + raidTP_gen = raidTP_ssse3; + raidQP_gen = raidQP_ssse3; +#endif + } + + if (cpu_has_avx()) { +#if defined(__x86_64__) + raidQP_gen = raidQP_avxext; +#endif + } + #endif } +static struct raid_func { + const char* name; + void* p; +} RAID_FUNC[] = { + { "int32", raid5_int32 }, + { "int64", raid5_int64 }, + { "int32", raid6_int32 }, + { "int64", raid6_int64 }, + { "int32", raidTP_int32 }, + { "int64", raidTP_int64 }, + { "int32", raidQP_int32 }, + { "int64", raidQP_int64 }, + +#if defined(__i386__) || defined(__x86_64__) + { "mmx", raid5_mmx }, + { "mmx", raid6_mmx }, + { "mmx", raidTP_mmx }, + { "mmx", raidQP_mmx }, + { "sse2", raid5_sse2 }, + { "sse2", raid6_sse2 }, + { "sse2", raidTP_sse2 }, + { "sse2", raidQP_sse2 }, + { "ssse3", raidTP_ssse3 }, + { "ssse3", raidQP_ssse3 }, +#endif + +#if defined(__x86_64__) + { "sse2ext", raid5_sse2ext }, + { "sse2ext", raid6_sse2ext }, + { "sse2ext", raidTP_sse2ext }, + { "sse2ext", raidQP_sse2ext }, + { "ssse3ext", raidTP_ssse3ext }, + { "ssse3ext", raidQP_ssse3ext }, + { "avxext", raidQP_avxext }, +#endif + { 0, 0 } +}; + +static const char* raid_tag(void* func) +{ + struct raid_func* i = RAID_FUNC; + while (i->name != 0) { + if (i->p == func) + return i->name; + ++i; + } + return "unknown"; +} + +const char* raid5_tag(void) +{ + return raid_tag(raid5_gen); +} + +const char* raid6_tag(void) +{ + return raid_tag(raid6_gen); +} + +const char* raidTP_tag(void) +{ + return raid_tag(raidTP_gen); +} + +const char* raidQP_tag(void) +{ + return raid_tag(raidQP_gen); +} + + diff --git a/raid.h b/raid.h index 6a939765..09ca1adb 100644 --- a/raid.h +++ b/raid.h @@ -78,30 +78,43 @@ void raid_recov_3data(int x, int y, int z, int i, int j, int k, unsigned char** */ void raid_recov_4data(int x, int y, int z, int v, int i, int j, int k, int l, unsigned char** vbuf, unsigned data, unsigned char* zero, unsigned size); +/** + * Gets the name of the selected function to compute parity. + */ +const char* raid5_tag(void); +const char* raid6_tag(void); +const char* raidTP_tag(void); +const char* raidQP_tag(void); + /** * Internal specialized parity computation. * Only for testing. */ -void raid5_int32r2(unsigned char** vbuf, unsigned data, unsigned size); -void raid5_int64r2(unsigned char** vbuf, unsigned data, unsigned size); -void raid5_mmxr4(unsigned char** vbuf, unsigned data, unsigned size); -void raid5_sse2r4(unsigned char** vbuf, unsigned data, unsigned size); -void raid5_sse2r8(unsigned char** vbuf, unsigned data, unsigned size); -void raid6_int32r2(unsigned char** vbuf, unsigned data, unsigned size); -void raid6_int64r2(unsigned char** vbuf, unsigned data, unsigned size); -void raid6_mmxr2(unsigned char** vbuf, unsigned data, unsigned size); -void raid6_sse2r2(unsigned char** vbuf, unsigned data, unsigned size); -void raid6_sse2r4(unsigned char** vbuf, unsigned data, unsigned size); -void raidTP_int32r2(unsigned char** vbuf, unsigned data, unsigned size); -void raidTP_int64r2(unsigned char** vbuf, unsigned data, unsigned size); -void raidTP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size); -void raidTP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size); -void raidTP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size); -void raidQP_int32r2(unsigned char** vbuf, unsigned data, unsigned size); -void raidQP_int64r2(unsigned char** vbuf, unsigned data, unsigned size); -void raidQP_mmxr1(unsigned char** vbuf, unsigned data, unsigned size); -void raidQP_sse2r1(unsigned char** vbuf, unsigned data, unsigned size); -void raidQP_sse2r2(unsigned char** vbuf, unsigned data, unsigned size); +void raid5_int32(unsigned char** vbuf, unsigned data, unsigned size); +void raid5_int64(unsigned char** vbuf, unsigned data, unsigned size); +void raid5_mmx(unsigned char** vbuf, unsigned data, unsigned size); +void raid5_sse2(unsigned char** vbuf, unsigned data, unsigned size); +void raid5_sse2ext(unsigned char** vbuf, unsigned data, unsigned size); +void raid6_int32(unsigned char** vbuf, unsigned data, unsigned size); +void raid6_int64(unsigned char** vbuf, unsigned data, unsigned size); +void raid6_mmx(unsigned char** vbuf, unsigned data, unsigned size); +void raid6_sse2(unsigned char** vbuf, unsigned data, unsigned size); +void raid6_sse2ext(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_int32(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_int64(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_mmx(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_sse2(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_sse2ext(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_ssse3(unsigned char** vbuf, unsigned data, unsigned size); +void raidTP_ssse3ext(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_int32(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_int64(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_mmx(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_sse2(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_sse2ext(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_ssse3(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_ssse3ext(unsigned char** vbuf, unsigned data, unsigned size); +void raidQP_avxext(unsigned char** vbuf, unsigned data, unsigned size); #endif diff --git a/selftest.c b/selftest.c index 51186326..ff5c0988 100644 --- a/selftest.c +++ b/selftest.c @@ -335,33 +335,48 @@ static void gentest(unsigned diskmax, unsigned block_size) /* load all the available functions */ mac = 0; - map[mac++] = raid5_int32r2; - map[mac++] = raid5_int64r2; - map[mac++] = raid6_int32r2; - map[mac++] = raid6_int64r2; - map[mac++] = raidTP_int32r2; - map[mac++] = raidTP_int64r2; - map[mac++] = raidQP_int32r2; - map[mac++] = raidQP_int64r2; + map[mac++] = raid5_int32; + map[mac++] = raid5_int64; + map[mac++] = raid6_int32; + map[mac++] = raid6_int64; + map[mac++] = raidTP_int32; + map[mac++] = raidTP_int64; + map[mac++] = raidQP_int32; + map[mac++] = raidQP_int64; #if defined(__i386__) || defined(__x86_64__) if (cpu_has_mmx()) { - map[mac++] = raid5_mmxr4; - map[mac++] = raid6_mmxr2; - map[mac++] = raidQP_mmxr1; - map[mac++] = raidTP_mmxr1; + map[mac++] = raid5_mmx; + map[mac++] = raid6_mmx; + map[mac++] = raidQP_mmx; + map[mac++] = raidTP_mmx; } if (cpu_has_sse2()) { - map[mac++] = raid5_sse2r4; - map[mac++] = raid6_sse2r2; - map[mac++] = raidTP_sse2r1; - map[mac++] = raidQP_sse2r1; + map[mac++] = raid5_sse2; + map[mac++] = raid6_sse2; + map[mac++] = raidTP_sse2; + map[mac++] = raidQP_sse2; #if defined(__x86_64__) - map[mac++] = raid5_sse2r8; - map[mac++] = raid6_sse2r4; - map[mac++] = raidTP_sse2r2; - map[mac++] = raidQP_sse2r2; + map[mac++] = raid5_sse2ext; + map[mac++] = raid6_sse2ext; + map[mac++] = raidTP_sse2ext; + map[mac++] = raidQP_sse2ext; +#endif + } + + if (cpu_has_ssse3()) { + map[mac++] = raidTP_ssse3; + map[mac++] = raidQP_ssse3; +#if defined(__x86_64__) + map[mac++] = raidTP_ssse3ext; + map[mac++] = raidQP_ssse3ext; +#endif + } + + if (cpu_has_avx()) { +#if defined(__x86_64__) + map[mac++] = raidQP_avxext; #endif } #endif diff --git a/speed.c b/speed.c index 3c3525e9..841ada4f 100644 --- a/speed.c +++ b/speed.c @@ -89,9 +89,12 @@ void speed(void) cpu_info(vendor, &family, &model); - printf("CPU %s, family %u, model %u, flags%s%s%s\n", vendor, family, model, + printf("CPU %s, family %u, model %u, flags%s%s%s%s%s%s\n", vendor, family, model, cpu_has_mmx() ? " mmx" : "", cpu_has_sse2() ? " sse2" : "", + cpu_has_ssse3() ? " ssse3" : "", + cpu_has_sse42() ? " sse42" : "", + cpu_has_avx() ? " avx" : "", cpu_has_slowmult() ? " slowmult" : "" ); } @@ -130,93 +133,10 @@ void speed(void) #endif printf("\n"); - printf("Expected fastest RAID5 is "); -#if defined(__i386__) || defined(__x86_64__) - if (cpu_has_sse2()) { -#if defined(__x86_64__) - if (cpu_has_slowextendedreg()) - printf("sse2x4"); - else - printf("sse2x8"); -#else - printf("sse2x4"); -#endif - } else if (cpu_has_mmx()) - printf("mmxx4"); - else - printf("int32x2"); -#else - if (sizeof(void*) == 4) - printf("int32x2"); - else - printf("int64x2"); -#endif - printf("\n"); - - printf("Expected fastest RAID6 is "); -#if defined(__i386__) || defined(__x86_64__) - if (cpu_has_sse2()) { -#if defined(__x86_64__) - if (cpu_has_slowextendedreg()) - printf("sse2x2"); - else - printf("sse2x4"); -#else - printf("sse2x2"); -#endif - } else if (cpu_has_mmx()) - printf("mmxx2"); - else - printf("int32x2"); -#else - if (sizeof(void*) == 4) - printf("int32x2"); - else - printf("int64x2"); -#endif - printf("\n"); - - printf("Expected fastest RAIDTP is "); -#if defined(__i386__) || defined(__x86_64__) - if (cpu_has_sse2()) -#if defined(__x86_64__) - printf("sse2x2"); -#else - printf("sse2x1"); -#endif - else if (cpu_has_mmx()) - printf("mmxx1"); - else - printf("int32x2"); -#else - if (sizeof(void*) == 4) - printf("int32x2"); - else - printf("int64x2"); -#endif - printf("\n"); - - - printf("Expected fastest RAIDQP is "); -#if defined(__i386__) || defined(__x86_64__) - if (cpu_has_sse2()) -#if defined(__x86_64__) - printf("sse2x2"); -#else - printf("sse2x1"); -#endif - else if (cpu_has_mmx()) - printf("mmxx1"); - else - printf("int32x2"); -#else - if (sizeof(void*) == 4) - printf("int32x2"); - else - printf("int64x2"); -#endif - printf("\n"); - + printf("Expected fastest RAID5 is %s\n", raid5_tag()); + printf("Expected fastest RAID6 is %s\n", raid6_tag()); + printf("Expected fastest RAIDTP is %s\n", raidTP_tag()); + printf("Expected fastest RAIDQP is %s\n", raidQP_tag()); printf("If these expectations are false, please report it in the SnapRAID forum\n"); @@ -268,135 +188,151 @@ void speed(void) printf("HASH Spooky2 %"PRIu64" [MB/s]\n", ds / dt); SPEED_START { - raid5_int32r2(buffer, diskmax, block_size); + raid5_int32(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID5 int32x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID5 int32 %"PRIu64" [MB/s]\n", ds / dt); SPEED_START { - raid5_int64r2(buffer, diskmax, block_size); + raid5_int64(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID5 int64x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID5 int64 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__i386__) || defined(__x86_64__) if (cpu_has_mmx()) { SPEED_START { - raid5_mmxr4(buffer, diskmax, block_size); + raid5_mmx(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID5 mmxx4 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID5 mmx %"PRIu64" [MB/s]\n", ds / dt); } if (cpu_has_sse2()) { SPEED_START { - raid5_sse2r4(buffer, diskmax, block_size); + raid5_sse2(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID5 sse2x4 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID5 sse2 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__x86_64__) SPEED_START { - raid5_sse2r8(buffer, diskmax, block_size); + raid5_sse2ext(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID5 sse2x8 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID5 sse2ext %"PRIu64" [MB/s]\n", ds / dt); #endif } #endif SPEED_START { - raid6_int32r2(buffer, diskmax, block_size); + raid6_int32(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID6 int32x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID6 int32 %"PRIu64" [MB/s]\n", ds / dt); SPEED_START { - raid6_int64r2(buffer, diskmax, block_size); + raid6_int64(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID6 int64x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID6 int64 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__i386__) || defined(__x86_64__) if (cpu_has_mmx()) { SPEED_START { - raid6_mmxr2(buffer, diskmax, block_size); + raid6_mmx(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID6 mmxx2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID6 mmx %"PRIu64" [MB/s]\n", ds / dt); } if (cpu_has_sse2()) { SPEED_START { - raid6_sse2r2(buffer, diskmax, block_size); + raid6_sse2(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID6 sse2x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID6 sse2 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__x86_64__) SPEED_START { - raid6_sse2r4(buffer, diskmax, block_size); + raid6_sse2ext(buffer, diskmax, block_size); } SPEED_STOP - printf("RAID6 sse2x4 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAID6 sse2ext %"PRIu64" [MB/s]\n", ds / dt); #endif } #endif SPEED_START { - raidTP_int32r2(buffer, diskmax, block_size); + raidTP_int32(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDTP int32x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDTP int32 %"PRIu64" [MB/s]\n", ds / dt); SPEED_START { - raidTP_int64r2(buffer, diskmax, block_size); + raidTP_int64(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDTP int64x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDTP int64 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__i386__) || defined(__x86_64__) if (cpu_has_mmx()) { SPEED_START { - raidTP_mmxr1(buffer, diskmax, block_size); + raidTP_mmx(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDTP mmxr1 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDTP mmx %"PRIu64" [MB/s]\n", ds / dt); } if (cpu_has_sse2()) { SPEED_START { - raidTP_sse2r1(buffer, diskmax, block_size); + raidTP_sse2(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDTP sse2x1 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDTP sse2 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__x86_64__) SPEED_START { - raidTP_sse2r2(buffer, diskmax, block_size); + raidTP_sse2ext(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDTP sse2x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDTP sse2ext %"PRIu64" [MB/s]\n", ds / dt); #endif } #endif - SPEED_START { - raidQP_int32r2(buffer, diskmax, block_size); - } SPEED_STOP + if (cpu_has_ssse3()) { + SPEED_START { + raidTP_ssse3(buffer, diskmax, block_size); + } SPEED_STOP - printf("RAIDQP int32x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDTP ssse3 %"PRIu64" [MB/s]\n", ds / dt); + +#if defined(__x86_64__) + SPEED_START { + raidTP_ssse3ext(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAIDTP ssse3ext %"PRIu64" [MB/s]\n", ds / dt); +#endif + } SPEED_START { - raidQP_int64r2(buffer, diskmax, block_size); + raidQP_int32(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDQP int64x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDQP int32 %"PRIu64" [MB/s]\n", ds / dt); + + SPEED_START { + raidQP_int64(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAIDQP int64 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__i386__) || defined(__x86_64__) if (cpu_has_mmx()) { SPEED_START { - raidQP_mmxr1(buffer, diskmax, block_size); + raidQP_mmx(buffer, diskmax, block_size); } SPEED_STOP printf("RAIDQP mmxr1 %"PRIu64" [MB/s]\n", ds / dt); @@ -404,21 +340,47 @@ void speed(void) if (cpu_has_sse2()) { SPEED_START { - raidQP_sse2r1(buffer, diskmax, block_size); + raidQP_sse2(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDQP sse2x1 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDQP sse2 %"PRIu64" [MB/s]\n", ds / dt); #if defined(__x86_64__) SPEED_START { - raidQP_sse2r2(buffer, diskmax, block_size); + raidQP_sse2ext(buffer, diskmax, block_size); } SPEED_STOP - printf("RAIDQP sse2x2 %"PRIu64" [MB/s]\n", ds / dt); + printf("RAIDQP sse2ext %"PRIu64" [MB/s]\n", ds / dt); #endif } #endif + if (cpu_has_ssse3()) { + SPEED_START { + raidQP_ssse3(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAIDQP ssse3 %"PRIu64" [MB/s]\n", ds / dt); + +#if defined(__x86_64__) + SPEED_START { + raidQP_ssse3ext(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAIDQP ssse3ext %"PRIu64" [MB/s]\n", ds / dt); +#endif + } + + if (cpu_has_avx()) { +#if defined(__x86_64__) + SPEED_START { + raidQP_avxext(buffer, diskmax, block_size); + } SPEED_STOP + + printf("RAIDQP avxext %"PRIu64" [MB/s]\n", ds / dt); +#endif + } + free(buffer_alloc); free(buffer); } diff --git a/tables.h b/tables.h index 927a8820..98dfdc68 100644 --- a/tables.h +++ b/tables.h @@ -8868,3 +8868,11 @@ const unsigned char __attribute__((aligned(256))) gfinv[256] = 0x42, 0xd4, 0xe8, 0x75, 0x7f, 0xff, 0x7e, 0xfd, }; +#define GFX4L { 0x00, 0x04, 0x08, 0x0c, 0x10, 0x14, 0x18, 0x1c, 0x20, 0x24, 0x28, 0x2c, 0x30, 0x34, 0x38, 0x3c } +#define GFX4H { 0x00, 0x40, 0x80, 0xc0, 0x1d, 0x5d, 0x9d, 0xdd, 0x3a, 0x7a, 0xba, 0xfa, 0x27, 0x67, 0xa7, 0xe7 } +#define GFX8L { 0x00, 0x08, 0x10, 0x18, 0x20, 0x28, 0x30, 0x38, 0x40, 0x48, 0x50, 0x58, 0x60, 0x68, 0x70, 0x78 } +#define GFX8H { 0x00, 0x80, 0x1d, 0x9d, 0x3a, 0xba, 0x27, 0xa7, 0x74, 0xf4, 0x69, 0xe9, 0x4e, 0xce, 0x53, 0xd3 } +#define GFPOLY8 { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d } +#define GFPOLY16 { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d } +#define GFMASK16 { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f } +