From 684edf21732316a11217966c1aa143133b4d15b9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sat, 28 Dec 2013 12:04:16 +0100 Subject: [PATCH] Updates the coding style of the RAID engine. --- portable.h | 7 ++ raid/cpu.h | 12 +- raid/gf.h | 8 +- raid/int.c | 212 ++++++++++++++++---------------- raid/internal.h | 117 +++++++++--------- raid/intz.c | 28 ++--- raid/mktables.c | 79 ++++++------ raid/module.c | 30 ++--- raid/raid.c | 114 +++++++++--------- raid/raid.h | 23 ++-- raid/tables.c | 14 +-- raid/x86.c | 314 ++++++++++++++++++++++++++---------------------- raid/x86z.c | 38 +++--- 13 files changed, 515 insertions(+), 481 deletions(-) diff --git a/portable.h b/portable.h index b89a05f7..eb5f311c 100644 --- a/portable.h +++ b/portable.h @@ -180,6 +180,13 @@ #define __always_inline inline __attribute__((always_inline)) #endif +/** + * Missing __aligned, like in user mode. + */ +#ifndef __aligned +#define __aligned(a) __attribute__((aligned(a))) +#endif + /** * Another name for link() to avoid confusion with local variables called "link". */ diff --git a/raid/cpu.h b/raid/cpu.h index 62504820..966055b2 100644 --- a/raid/cpu.h +++ b/raid/cpu.h @@ -20,7 +20,7 @@ #if defined(__i386__) || defined(__x86_64__) -static __always_inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t* reg) +static __always_inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint32_t *reg) { asm volatile( #if defined(__i386__) && defined(__PIC__) @@ -40,16 +40,16 @@ static __always_inline void raid_cpuid(uint32_t func_eax, uint32_t sub_ecx, uint #define CPU_VENDOR_MAX 13 -static __always_inline void raid_cpu_info(char* vendor, unsigned* family, unsigned* model) +static __always_inline void raid_cpu_info(char *vendor, unsigned *family, unsigned *model) { uint32_t reg[4]; unsigned f, ef, m, em; raid_cpuid(0, 0, reg); - ((uint32_t*)vendor)[0] = reg[1]; - ((uint32_t*)vendor)[1] = reg[3]; - ((uint32_t*)vendor)[2] = reg[2]; + ((uint32_t *)vendor)[0] = reg[1]; + ((uint32_t *)vendor)[1] = reg[3]; + ((uint32_t *)vendor)[2] = reg[2]; vendor[12] = 0; raid_cpuid(1, 0, reg); @@ -185,7 +185,7 @@ static __always_inline int raid_cpu_has_slowpshufb(void) * CPU GenuineIntel, family 6, model 28, flags mmx sse2 ssse3 slowmult * Memory is little-endian 64-bit * Support nanosecond timestamps with futimens() - * + * * Speed test with 8 disk and 262144 buffer size, for a total of 2816 KiB... * memset0 1850 [MB/s] * CRC table 316 [MB/s] diff --git a/raid/gf.h b/raid/gf.h index 2eaedfe5..7d165ad1 100644 --- a/raid/gf.h +++ b/raid/gf.h @@ -62,7 +62,7 @@ static __always_inline uint8_t pow2(int v) /* * Gets the multiplication table for a specified value. */ -static __always_inline const uint8_t* table(uint8_t v) +static __always_inline const uint8_t *table(uint8_t v) { return gfmul[v]; } @@ -80,17 +80,17 @@ static __always_inline uint8_t A(int p, int d) /* * Dereference as uint8_t */ -#define v_8(p) (*(uint8_t*)&(p)) +#define v_8(p) (*(uint8_t *)&(p)) /* * Dereference as uint32_t */ -#define v_32(p) (*(uint32_t*)&(p)) +#define v_32(p) (*(uint32_t *)&(p)) /* * Dereference as uint64_t */ -#define v_64(p) (*(uint64_t*)&(p)) +#define v_64(p) (*(uint64_t *)&(p)) /* * Multiply each byte of a uint32 by 2 in the GF(2^8). diff --git a/raid/int.c b/raid/int.c index 21417021..437e1acd 100644 --- a/raid/int.c +++ b/raid/int.c @@ -21,10 +21,10 @@ /* * PAR1 (RAID5 with xor) 32bit C implementation */ -void raid_par1_int32(int nd, size_t size, void** vv) +void raid_par1_int32(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; int d, l; size_t i; @@ -34,12 +34,12 @@ void raid_par1_int32(int nd, size_t size, void** vv) l = nd - 1; p = v[nd]; - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { p0 ^= v_32(v[d][i]); p1 ^= v_32(v[d][i+4]); } @@ -51,10 +51,10 @@ void raid_par1_int32(int nd, size_t size, void** vv) /* * PAR1 (RAID5 with xor) 64bit C implementation */ -void raid_par1_int64(int nd, size_t size, void** vv) +void raid_par1_int64(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; int d, l; size_t i; @@ -64,12 +64,12 @@ void raid_par1_int64(int nd, size_t size, void** vv) l = nd - 1; p = v[nd]; - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { p0 ^= v_64(v[d][i]); p1 ^= v_64(v[d][i+8]); } @@ -81,11 +81,11 @@ void raid_par1_int64(int nd, size_t size, void** vv) /* * PAR2 (RAID6 with powers of 2) 32bit C implementation */ -void raid_par2_int32(int nd, size_t size, void** vv) +void raid_par2_int32(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; int d, l; size_t i; @@ -96,10 +96,10 @@ void raid_par2_int32(int nd, size_t size, void** vv) p = v[nd]; q = v[nd+1]; - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { d0 = v_32(v[d][i]); d1 = v_32(v[d][i+4]); @@ -122,11 +122,11 @@ void raid_par2_int32(int nd, size_t size, void** vv) /* * PAR2 (RAID6 with powers of 2) 64bit C implementation */ -void raid_par2_int64(int nd, size_t size, void** vv) +void raid_par2_int64(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; int d, l; size_t i; @@ -137,10 +137,10 @@ void raid_par2_int64(int nd, size_t size, void** vv) p = v[nd]; q = v[nd+1]; - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { d0 = v_64(v[d][i]); d1 = v_64(v[d][i+8]); @@ -162,18 +162,18 @@ void raid_par2_int64(int nd, size_t size, void** vv) /* * PAR3 (triple parity with Cauchy matrix) 8bit C implementation - * + * * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. + * in multiple cache misses, a precomputed table could be used. * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par3_int8(int nd, size_t size, void** vv) +void raid_par3_int8(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; int d, l; size_t i; @@ -184,9 +184,9 @@ void raid_par3_int8(int nd, size_t size, void** vv) q = v[nd+1]; r = v[nd+2]; - for(i=0;i0;--d) { + for (d = l; d > 0; --d) { d0 = v_8(v[d][i]); p0 ^= d0; @@ -209,19 +209,19 @@ void raid_par3_int8(int nd, size_t size, void** vv) /* * PAR4 (quad parity with Cauchy matrix) 8bit C implementation - * + * * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. + * in multiple cache misses, a precomputed table could be used. * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par4_int8(int nd, size_t size, void** vv) +void raid_par4_int8(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; int d, l; size_t i; @@ -233,9 +233,9 @@ void raid_par4_int8(int nd, size_t size, void** vv) r = v[nd+2]; s = v[nd+3]; - for(i=0;i0;--d) { + for (d = l; d > 0; --d) { d0 = v_8(v[d][i]); p0 ^= d0; @@ -261,20 +261,20 @@ void raid_par4_int8(int nd, size_t size, void** vv) /* * PAR5 (penta parity with Cauchy matrix) 8bit C implementation - * + * * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. + * in multiple cache misses, a precomputed table could be used. * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par5_int8(int nd, size_t size, void** vv) +void raid_par5_int8(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; int d, l; size_t i; @@ -287,9 +287,9 @@ void raid_par5_int8(int nd, size_t size, void** vv) s = v[nd+3]; t = v[nd+4]; - for(i=0;i0;--d) { + for (d = l; d > 0; --d) { d0 = v_8(v[d][i]); p0 ^= d0; @@ -318,21 +318,21 @@ void raid_par5_int8(int nd, size_t size, void** vv) /* * PAR6 (hexa parity with Cauchy matrix) 8bit C implementation - * + * * Note that instead of a generic multiplicationt table, likely resulting - * in multiple cache misses, a precomputed table could be used. + * in multiple cache misses, a precomputed table could be used. * But this is only a kind of reference function, and we are not really * interested in speed. */ -void raid_par6_int8(int nd, size_t size, void** vv) +void raid_par6_int8(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; - uint8_t* u; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; int d, l; size_t i; @@ -346,9 +346,9 @@ void raid_par6_int8(int nd, size_t size, void** vv) t = v[nd+4]; u = v[nd+5]; - for(i=0;i0;--d) { + for (d = l; d > 0; --d) { d0 = v_8(v[d][i]); p0 ^= d0; @@ -390,12 +390,12 @@ void raid_par6_int8(int nd, size_t size, void** vv) * * Dx = A[ip[0],id[0]]^-1 * Pd */ -void raid_rec1_int8(int nr, const int* id, const int* ip, int nd, size_t size, void** vv) +void raid_rec1_int8(int nr, const int *id, const int *ip, int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* pa; - const uint8_t* T; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + const uint8_t *T; uint8_t G; uint8_t V; size_t i; @@ -409,7 +409,7 @@ void raid_rec1_int8(int nr, const int* id, const int* ip, int nd, size_t size, v } /* setup the coefficients matrix */ - G = A(ip[0],id[0]); + G = A(ip[0], id[0]); /* invert it to solve the system of linear equations */ V = inv(G); @@ -423,7 +423,7 @@ void raid_rec1_int8(int nr, const int* id, const int* ip, int nd, size_t size, v p = v[nd+ip[0]]; pa = v[id[0]]; - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { d0 = v_32(v[d][i]); d1 = v_32(v[d][i+4]); @@ -72,12 +72,12 @@ void raid_parz_int32(int nd, size_t size, void** vv) /* * PARz (triple parity with powers of 2^-1) 64bit C implementation */ -void raid_parz_int64(int nd, size_t size, void** vv) +void raid_parz_int64(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; int d, l; size_t i; @@ -89,10 +89,10 @@ void raid_parz_int64(int nd, size_t size, void** vv) q = v[nd+1]; r = v[nd+2]; - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { d0 = v_64(v[d][i]); d1 = v_64(v[d][i+8]); diff --git a/raid/mktables.c b/raid/mktables.c index 932ba1fb..ae2beb47 100644 --- a/raid/mktables.c +++ b/raid/mktables.c @@ -63,7 +63,7 @@ uint8_t gfinv[256]; /** * Setup the Cauchy matrix used to generate the parity. */ -static void set_cauchy(uint8_t* matrix) +static void set_cauchy(uint8_t *matrix) { int i, j; uint8_t inv_x, y; @@ -74,9 +74,8 @@ static void set_cauchy(uint8_t* matrix) * This is an Extended Cauchy matrix built from a Cauchy matrix * adding the first row of all 1. */ - for(i=0;i=0,j>=1,i+j<255 */ y = 2; - for(j=0;jname != 0) { if (i->p == func) return i->name; @@ -137,52 +137,52 @@ static const char* raid_tag(void* func) return "unknown"; } -const char* raid_par1_tag(void) +const char *raid_par1_tag(void) { return raid_tag(raid_par_ptr[0]); } -const char* raid_par2_tag(void) +const char *raid_par2_tag(void) { return raid_tag(raid_par_ptr[1]); } -const char* raid_parz_tag(void) +const char *raid_parz_tag(void) { return raid_tag(raid_parz_ptr); } -const char* raid_par3_tag(void) +const char *raid_par3_tag(void) { return raid_tag(raid_par_ptr[2]); } -const char* raid_par4_tag(void) +const char *raid_par4_tag(void) { return raid_tag(raid_par_ptr[3]); } -const char* raid_par5_tag(void) +const char *raid_par5_tag(void) { return raid_tag(raid_par_ptr[4]); } -const char* raid_par6_tag(void) +const char *raid_par6_tag(void) { return raid_tag(raid_par_ptr[5]); } -const char* raid_rec1_tag(void) +const char *raid_rec1_tag(void) { return raid_tag(raid_rec_ptr[0]); } -const char* raid_rec2_tag(void) +const char *raid_rec2_tag(void) { return raid_tag(raid_rec_ptr[1]); } -const char* raid_recX_tag(void) +const char *raid_recX_tag(void) { return raid_tag(raid_rec_ptr[2]); } diff --git a/raid/raid.c b/raid/raid.c index bdad2a26..714cbf8d 100644 --- a/raid/raid.c +++ b/raid/raid.c @@ -32,7 +32,7 @@ /* * The RAID5 and RAID6 support is implemented using the Galois Field - * GF(2^8) with the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 + * GF(2^8) with the primitive polynomial x^8 + x^4 + x^3 + x^2 + 1 * (285 decimal). * * The parity P and Q of a set of N disk Di with 0<=i RAID_PARITY_MAX); + BUG_ON(size % 64 != 0); raid_par_ptr[np - 1](nd, size, v); } @@ -235,19 +236,17 @@ void raid_par(int np, int nd, size_t size, void** v) * Inverts the square matrix M of size nxn into V. * We use Gauss elimination to invert. */ -void raid_invert(uint8_t* M, uint8_t* V, int n) +void raid_invert(uint8_t *M, uint8_t *V, int n) { - int i,j,k; + int i, j, k; /* set the identity matrix in V */ - for(i=0;i nd); + BUG_ON(size % 64 != 0); /* if failed data is present */ - if (nr != 0) { + if (nr != 0) raid_rec_ptr[nr - 1](nr, id, ip, nd, size, v); - } } -void raid_recpar(int nrd, const int* id, int nrp, int* ip, int np, int nd, size_t size, void** v) +void raid_recpar(int nrd, const int *id, int nrp, int *ip, int np, int nd, size_t size, void **v) { BUG_ON(nrd > nd); BUG_ON(nrd + nrp > np); + BUG_ON(size % 64 != 0); /* if failed data is present */ if (nrd != 0) { @@ -436,7 +436,7 @@ void raid_recpar(int nrd, const int* id, int nrp, int* ip, int np, int nd, size_ int i, j, k; /* setup the vector of parities to use */ - for(i=0,j=0,k=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("movdqa %0,%%xmm5" : : "m" (v[d][i+16])); asm volatile("movdqa %0,%%xmm6" : : "m" (v[d][i+32])); @@ -60,7 +62,7 @@ void raid_par1_sse2(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm3,%0" : "=m" (p[i+48])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -68,19 +70,21 @@ void raid_par1_sse2(int nd, size_t size, void** vv) static const struct gfconst16 { uint8_t poly[16]; uint8_t low4[16]; -} gfconst16 __attribute__((aligned(64))) = { +} gfconst16 __aligned(32) = { { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f }, }; +#endif +#if defined(__i386__) || defined(__x86_64__) /* * PAR2 (RAID6 with powers of 2) SSE2 implementation */ -void raid_par2_sse2(int nd, size_t size, void** vv) +void raid_par2_sse2(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; int d, l; size_t i; @@ -88,14 +92,16 @@ void raid_par2_sse2(int nd, size_t size, void** vv) p = v[nd]; q = v[nd+1]; + asm_begin(); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { asm volatile("pxor %xmm4,%xmm4"); asm volatile("pxor %xmm5,%xmm5"); asm volatile("pcmpgtb %xmm2,%xmm4"); @@ -120,7 +126,7 @@ void raid_par2_sse2(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm3,%0" : "=m" (q[i+16])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -130,11 +136,11 @@ void raid_par2_sse2(int nd, size_t size, void** vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par2_sse2ext(int nd, size_t size, void** vv) +void raid_par2_sse2ext(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; int d, l; size_t i; @@ -142,9 +148,11 @@ void raid_par2_sse2ext(int nd, size_t size, void** vv) p = v[nd]; q = v[nd+1]; + asm_begin(); + asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { asm volatile("pxor %xmm8,%xmm8"); asm volatile("pxor %xmm9,%xmm9"); asm volatile("pxor %xmm10,%xmm10"); @@ -198,7 +206,7 @@ void raid_par2_sse2ext(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm7,%0" : "=m" (q[i+48])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -206,12 +214,12 @@ void raid_par2_sse2ext(int nd, size_t size, void** vv) /* * PAR3 (triple parity with Cauchy matrix) SSSE3 implementation */ -void raid_par3_ssse3(int nd, size_t size, void** vv) +void raid_par3_ssse3(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; int d, l; size_t i; @@ -222,16 +230,18 @@ void raid_par3_ssse3(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<3;++i) + for (i = 0; i < 3; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ asm volatile("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("pxor %xmm5,%xmm5"); @@ -287,13 +297,13 @@ void raid_par3_ssse3(int nd, size_t size, void** vv) asm volatile("pxor %xmm4,%xmm0"); asm volatile("pxor %xmm4,%xmm1"); asm volatile("pxor %xmm4,%xmm2"); - + asm volatile("movntdq %%xmm0,%0" : "=m" (p[i])); asm volatile("movntdq %%xmm1,%0" : "=m" (q[i])); asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -303,12 +313,12 @@ void raid_par3_ssse3(int nd, size_t size, void** vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par3_ssse3ext(int nd, size_t size, void** vv) +void raid_par3_ssse3ext(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; int d, l; size_t i; @@ -319,16 +329,18 @@ void raid_par3_ssse3ext(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<3;++i) + for (i = 0; i < 3; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ asm volatile("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm11" : : "m" (gfconst16.low4[0])); - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("movdqa %0,%%xmm12" : : "m" (v[d][i+16])); @@ -432,7 +444,7 @@ void raid_par3_ssse3ext(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm10,%0" : "=m" (r[i+16])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -440,13 +452,13 @@ void raid_par3_ssse3ext(int nd, size_t size, void** vv) /* * PAR4 (quad parity with Cauchy matrix) SSSE3 implementation */ -void raid_par4_ssse3(int nd, size_t size, void** vv) +void raid_par4_ssse3(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; int d, l; size_t i; @@ -458,13 +470,15 @@ void raid_par4_ssse3(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<4;++i) + for (i = 0; i < 4; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); @@ -546,7 +560,7 @@ void raid_par4_ssse3(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm3,%0" : "=m" (s[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -556,13 +570,13 @@ void raid_par4_ssse3(int nd, size_t size, void** vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par4_ssse3ext(int nd, size_t size, void** vv) +void raid_par4_ssse3ext(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; int d, l; size_t i; @@ -574,13 +588,15 @@ void raid_par4_ssse3ext(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<4;++i) + for (i = 0; i < 4; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); @@ -717,7 +733,7 @@ void raid_par4_ssse3ext(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm11,%0" : "=m" (s[i+16])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -729,18 +745,18 @@ void raid_par4_ssse3ext(int nd, size_t size, void** vv) /* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ __attribute__((force_align_arg_pointer)) #endif -void raid_par5_ssse3(int nd, size_t size, void** vv) +void raid_par5_ssse3(int nd, size_t size, void **vv) /* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; int d, l; size_t i; - uint8_t p0[16] __attribute__((aligned(16))); + uint8_t p0[16] __aligned(16); l = nd - 1; p = v[nd]; @@ -751,13 +767,15 @@ void raid_par5_ssse3(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<5;++i) + for (i = 0; i < 5; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm4" : : "m" (v[d][i])); asm volatile("movdqa %0,%%xmm6" : : "m" (p0[0])); asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); @@ -856,7 +874,7 @@ void raid_par5_ssse3(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm3,%0" : "=m" (t[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -866,14 +884,14 @@ void raid_par5_ssse3(int nd, size_t size, void** vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par5_ssse3ext(int nd, size_t size, void** vv) +void raid_par5_ssse3ext(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; int d, l; size_t i; @@ -886,16 +904,18 @@ void raid_par5_ssse3ext(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<5;++i) + for (i = 0; i < 5; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ asm volatile("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm10" : : "m" (v[d][i])); asm volatile("pxor %xmm11,%xmm11"); @@ -987,7 +1007,7 @@ void raid_par5_ssse3ext(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm4,%0" : "=m" (t[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -999,19 +1019,19 @@ void raid_par5_ssse3ext(int nd, size_t size, void** vv) /* ensures that stack is aligned at 16 bytes because we allocate SSE registers in it */ __attribute__((force_align_arg_pointer)) #endif -void raid_par6_ssse3(int nd, size_t size, void** vv) +void raid_par6_ssse3(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; - uint8_t* u; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; int d, l; size_t i; - uint8_t p0[16] __attribute__((aligned(16))); - uint8_t q0[16] __attribute__((aligned(16))); + uint8_t p0[16] __aligned(16); + uint8_t q0[16] __aligned(16); l = nd - 1; p = v[nd]; @@ -1023,13 +1043,15 @@ void raid_par6_ssse3(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<6;++i) + for (i = 0; i < 6; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm5" : : "m" (p0[0])); asm volatile("movdqa %0,%%xmm6" : : "m" (q0[0])); asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); @@ -1147,7 +1169,7 @@ void raid_par6_ssse3(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm3,%0" : "=m" (u[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -1157,15 +1179,15 @@ void raid_par6_ssse3(int nd, size_t size, void** vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_par6_ssse3ext(int nd, size_t size, void** vv) +void raid_par6_ssse3ext(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; - uint8_t* s; - uint8_t* t; - uint8_t* u; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; + uint8_t *s; + uint8_t *t; + uint8_t *u; int d, l; size_t i; @@ -1179,16 +1201,18 @@ void raid_par6_ssse3ext(int nd, size_t size, void** vv) /* special case with only one data disk */ if (l == 0) { - for(i=0;i<6;++i) + for (i = 0; i < 6; ++i) memcpy(v[1+i], v[0], size); return; } + asm_begin(); + /* generic case with at least two data disks */ asm volatile("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); asm volatile("movdqa %0,%%xmm15" : : "m" (gfconst16.low4[0])); - for(i=0;i0;--d) { + for (d = l-1; d > 0; --d) { asm volatile("movdqa %0,%%xmm10" : : "m" (v[d][i])); asm volatile("pxor %xmm11,%xmm11"); @@ -1295,7 +1319,7 @@ void raid_par6_ssse3ext(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm5,%0" : "=m" (u[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -1303,11 +1327,11 @@ void raid_par6_ssse3ext(int nd, size_t size, void** vv) /* * RAID recovering for one disk SSSE3 implementation */ -void raid_rec1_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, void** vv) +void raid_rec1_ssse3(int nr, const int *id, const int *ip, int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* pa; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; uint8_t G; uint8_t V; size_t i; @@ -1321,7 +1345,7 @@ void raid_rec1_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, } /* setup the coefficients matrix */ - G = A(ip[0],id[0]); + G = A(ip[0], id[0]); /* invert it to solve the system of linear equations */ V = inv(G); @@ -1332,11 +1356,13 @@ void raid_rec1_ssse3(int nr, const int* id, const int* ip, int nd, size_t size, p = v[nd+ip[0]]; pa = v[id[0]]; + asm_begin(); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); asm volatile("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0])); asm volatile("movdqa %0,%%xmm5" : : "m" (gfmulpshufb[V][1][0])); - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { asm volatile("pxor %xmm4,%xmm4"); asm volatile("pcmpgtb %xmm1,%xmm4"); asm volatile("paddb %xmm1,%xmm1"); @@ -79,7 +81,7 @@ void raid_parz_sse2(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif @@ -89,12 +91,12 @@ void raid_parz_sse2(int nd, size_t size, void** vv) * * Note that it uses 16 registers, meaning that x64 is required. */ -void raid_parz_sse2ext(int nd, size_t size, void** vv) +void raid_parz_sse2ext(int nd, size_t size, void **vv) { - uint8_t** v = (uint8_t**)vv; - uint8_t* p; - uint8_t* q; - uint8_t* r; + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + uint8_t *r; int d, l; size_t i; @@ -103,18 +105,20 @@ void raid_parz_sse2ext(int nd, size_t size, void** vv) q = v[nd+1]; r = v[nd+2]; + asm_begin(); + asm volatile("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); asm volatile("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); asm volatile("movdqa %0,%%xmm11" : : "m" (gfzconst16.low7[0])); - for(i=0;i=0;--d) { + for (d = l-1; d >= 0; --d) { asm volatile("movdqa %xmm2,%xmm6"); asm volatile("movdqa %xmm10,%xmm14"); asm volatile("pxor %xmm4,%xmm4"); @@ -159,7 +163,7 @@ void raid_parz_sse2ext(int nd, size_t size, void** vv) asm volatile("movntdq %%xmm10,%0" : "=m" (r[i+16])); } - asm volatile("sfence" : : : "memory"); + asm_end(); } #endif