From a99cd9daff80c1eac162d1e738cd83b9075332e0 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Tue, 10 Mar 2015 19:02:23 +0100 Subject: [PATCH] Use specialized assembler header and footer for sse and avx --- raid/internal.h | 14 +++++--- raid/x86.c | 92 ++++++++++++++++++++++++------------------------- raid/x86z.c | 12 +++---- 3 files changed, 61 insertions(+), 57 deletions(-) diff --git a/raid/internal.h b/raid/internal.h index 027d9e33..59848925 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -191,12 +191,12 @@ extern const uint8_t (*raid_gfgen)[256]; * Assembler blocks. */ #ifdef CONFIG_X86 -static __always_inline void raid_asm_begin(void) +#ifdef CONFIG_SSE2 +static __always_inline void raid_sse_begin(void) { } -#ifdef CONFIG_SSE2 -static __always_inline void raid_asm_sse_end(void) +static __always_inline void raid_sse_end(void) { /* SSE and AVX code uses non-temporal writes, like MOVNTDQ, */ /* that use a weak memory model. To ensure that other processors */ @@ -223,9 +223,13 @@ static __always_inline void raid_asm_sse_end(void) #endif #ifdef CONFIG_AVX2 -static __always_inline void raid_asm_avx_end(void) +static __always_inline void raid_avx_begin(void) { - raid_asm_sse_end(); +} + +static __always_inline void raid_avx_end(void) +{ + raid_sse_end(); /* reset the upper part of the ymm registers */ /* to avoid the 70 clocks penality on the next */ diff --git a/raid/x86.c b/raid/x86.c index 1382ad2c..5086e0dc 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -33,7 +33,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) l = nd - 1; p = v[nd]; - raid_asm_begin(); + raid_sse_begin(); for (i = 0; i < size; i += 64) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); @@ -52,7 +52,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -74,7 +74,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) l = nd - 1; p = v[nd]; - raid_asm_begin(); + raid_avx_begin(); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); @@ -87,7 +87,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -124,7 +124,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) p = v[nd]; q = v[nd + 1]; - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0])); @@ -158,7 +158,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -178,7 +178,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) p = v[nd]; q = v[nd + 1]; - raid_asm_begin(); + raid_avx_begin(); asm volatile ("vbroadcasti128 %0, %%ymm7" : : "m" (gfconst16.poly[0])); asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); @@ -211,7 +211,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -233,7 +233,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv) p = v[nd]; q = v[nd + 1]; - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0])); @@ -291,7 +291,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -320,7 +320,7 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); @@ -388,7 +388,7 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -419,7 +419,7 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0])); @@ -529,7 +529,7 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -560,7 +560,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_avx_begin(); /* generic case with at least two data disks */ asm volatile ("vbroadcasti128 %0, %%ymm3" : : "m" (gfconst16.poly[0])); @@ -662,7 +662,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -693,7 +693,7 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 16) { @@ -778,7 +778,7 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -811,7 +811,7 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 32) { @@ -951,7 +951,7 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -984,7 +984,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_avx_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 64) { @@ -1112,7 +1112,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -1150,7 +1150,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 16) { @@ -1252,7 +1252,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -1287,7 +1287,7 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); @@ -1385,7 +1385,7 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -1420,7 +1420,7 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_avx_begin(); /* generic case with at least two data disks */ asm volatile ("vpxor %ymm8,%ymm8,%ymm8"); @@ -1515,7 +1515,7 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -1556,7 +1556,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ for (i = 0; i < size; i += 16) { @@ -1677,7 +1677,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -1714,7 +1714,7 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_sse_begin(); /* generic case with at least two data disks */ asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0])); @@ -1827,7 +1827,7 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -1864,7 +1864,7 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv) return; } - raid_asm_begin(); + raid_avx_begin(); /* generic case with at least two data disks */ asm volatile ("vpxor %ymm8,%ymm8,%ymm8"); @@ -1974,7 +1974,7 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm5,%0" : "=m" (u[i])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -2011,7 +2011,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) p = v[nd + ip[0]]; pa = v[id[0]]; - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0])); @@ -2033,7 +2033,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -2070,7 +2070,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) pa[j] = v[id[j]]; } - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); @@ -2139,7 +2139,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -2174,7 +2174,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) pa[j] = v[id[j]]; } - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); @@ -2215,7 +2215,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -2252,7 +2252,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) p = v[nd + ip[0]]; pa = v[id[0]]; - raid_asm_begin(); + raid_avx_begin(); asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0])); @@ -2271,7 +2271,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -2308,7 +2308,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) pa[j] = v[id[j]]; } - raid_asm_begin(); + raid_avx_begin(); asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); @@ -2369,7 +2369,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif @@ -2404,7 +2404,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) pa[j] = v[id[j]]; } - raid_asm_begin(); + raid_avx_begin(); asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); @@ -2444,7 +2444,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } - raid_asm_avx_end(); + raid_avx_end(); } #endif diff --git a/raid/x86z.c b/raid/x86z.c index 77d5ffd9..1e3fe89a 100644 --- a/raid/x86z.c +++ b/raid/x86z.c @@ -54,7 +54,7 @@ void raid_genz_sse2(int nd, size_t size, void **vv) q = v[nd + 1]; r = v[nd + 2]; - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); asm volatile ("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); @@ -90,7 +90,7 @@ void raid_genz_sse2(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -114,7 +114,7 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv) q = v[nd + 1]; r = v[nd + 2]; - raid_asm_begin(); + raid_sse_begin(); asm volatile ("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0])); asm volatile ("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0])); @@ -172,7 +172,7 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); } - raid_asm_sse_end(); + raid_sse_end(); } #endif @@ -196,7 +196,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv) q = v[nd + 1]; r = v[nd + 2]; - raid_asm_begin(); + raid_avx_begin(); asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfzconst16.poly[0])); asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfzconst16.half[0])); @@ -249,7 +249,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); } - raid_asm_avx_end(); + raid_avx_end(); } #endif