From 3274b7cd2e4dae3077c91bffa775b2175bd3b43c Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 21 Sep 2014 08:45:33 +0200 Subject: [PATCH] Uses a different way to handle registers conflict with the compiler. --- raid/internal.h | 56 ++++++++++++++++++++++++++++++++++++++++++ raid/x86.c | 65 ++++++++++++++++++++++++++++++------------------- raid/x86z.c | 18 ++++---------- 3 files changed, 101 insertions(+), 38 deletions(-) diff --git a/raid/internal.h b/raid/internal.h index 357bbd90..eacbe1ce 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -160,8 +160,64 @@ static __always_inline void raid_asm_begin(void) static __always_inline void raid_asm_end(void) { + /* SSE2 and AVX2 code uses non-temporal writes, like movntdq, */ + /* that use a weak memory model. To ensure that other processors */ + /* see correctly the data written, we use a store-store memory */ + /* barrier at the end of the asm code */ asm volatile("sfence" : : : "memory"); } + +static __always_inline void raid_asm_clobber_xmm4(void) +{ + /* clobbers registers used in the asm code */ + /* this is required because in the Windows ABI, */ + /* registers xmm6-xmm15 should be kept by the callee. */ + /* this clobber list force the compiler to same any */ + /* register that needs to be saved */ +#ifdef __SSE__ + asm volatile("" : : : "%xmm0", "%xmm1", "%xmm2", "%xmm3"); +#endif +} + +static __always_inline void raid_asm_clobber_xmm8(void) +{ + raid_asm_clobber_xmm4(); +#ifdef __SSE__ + asm volatile("" : : : "%xmm4", "%xmm5", "%xmm6", "%xmm7"); +#endif +} + +static __always_inline void raid_asm_clobber_ymm4(void) +{ + raid_asm_clobber_xmm4(); + /* reset the upper part of the ymm registers */ + /* to avoid the 70 clocks penality on the next */ + /* xmm register use */ + asm volatile("vzeroupper" : : : "memory"); +} + +static __always_inline void raid_asm_clobber_ymm8(void) +{ + raid_asm_clobber_xmm8(); + asm volatile("vzeroupper" : : : "memory"); +} +#endif + +#ifdef CONFIG_X86_64 +static __always_inline void raid_asm_clobber_xmm16(void) +{ + raid_asm_clobber_xmm8(); +#ifdef __SSE__ + asm volatile("" : : : "%xmm8", "%xmm9", "%xmm10", "%xmm11"); + asm volatile("" : : : "%xmm12", "%xmm13", "%xmm14", "%xmm15"); +#endif +} + +static __always_inline void raid_asm_clobber_ymm16(void) +{ + raid_asm_clobber_xmm16(); + asm volatile("vzeroupper" : : : "memory"); +} #endif #endif diff --git a/raid/x86.c b/raid/x86.c index 550093ed..c50ee525 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -15,15 +15,6 @@ #include "internal.h" #include "gf.h" -/** - * Disables in this file any use of SSE/AVX registers by the compiler - * to avoid to messup with the registers use made by the following functions. - */ -#ifdef CONFIG_X86 -#pragma GCC push_options -#pragma GCC target ("no-sse,no-avx") -#endif - #ifdef CONFIG_X86 /* * GEN1 (RAID5 with xor) SSE2 implementation @@ -61,6 +52,8 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm3,%0" : "=m" (p[i+48])); } + raid_asm_clobber_xmm4(); + raid_asm_end(); } #endif @@ -87,16 +80,16 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) for (i = 0; i < size; i += 64) { asm volatile("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); - asm volatile("vmovdqa %0,%%ymm2" : : "m" (v[l][i+32])); + asm volatile("vmovdqa %0,%%ymm1" : : "m" (v[l][i+32])); for (d = l-1; d >= 0; --d) { asm volatile("vpxor %0,%%ymm0,%%ymm0" : : "m" (v[d][i])); - asm volatile("vpxor %0,%%ymm2,%%ymm2" : : "m" (v[d][i+32])); + asm volatile("vpxor %0,%%ymm1,%%ymm1" : : "m" (v[d][i+32])); } asm volatile("vmovntdq %%ymm0,%0" : "=m" (p[i])); - asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[i+32])); + asm volatile("vmovntdq %%ymm1,%0" : "=m" (p[i+32])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm4(); raid_asm_end(); } @@ -168,6 +161,8 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm3,%0" : "=m" (q[i+16])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -221,7 +216,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) asm volatile("vmovntdq %%ymm3,%0" : "=m" (q[i+32])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm8(); raid_asm_end(); } @@ -303,6 +298,8 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm7,%0" : "=m" (q[i+48])); } + raid_asm_clobber_xmm16(); + raid_asm_end(); } #endif @@ -400,6 +397,8 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -541,6 +540,8 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm10,%0" : "=m" (r[i+16])); } + raid_asm_clobber_xmm16(); + raid_asm_end(); } #endif @@ -674,7 +675,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv) asm volatile("vmovntdq %%ymm10,%0" : "=m" (r[i+32])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm16(); raid_asm_end(); } @@ -792,6 +793,8 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm3,%0" : "=m" (s[i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -965,6 +968,8 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm11,%0" : "=m" (s[i+16])); } + raid_asm_clobber_xmm16(); + raid_asm_end(); } #endif @@ -1126,7 +1131,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) asm volatile("vmovntdq %%ymm11,%0" : "=m" (s[i+32])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm16(); raid_asm_end(); } @@ -1268,6 +1273,8 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm3,%0" : "=m" (t[i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -1401,6 +1408,8 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm4,%0" : "=m" (t[i])); } + raid_asm_clobber_xmm16(); + raid_asm_end(); } #endif @@ -1531,7 +1540,7 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv) asm volatile("vmovntdq %%ymm4,%0" : "=m" (t[i])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm16(); raid_asm_end(); } @@ -1695,6 +1704,8 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm3,%0" : "=m" (u[i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -1845,6 +1856,8 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm5,%0" : "=m" (u[i])); } + raid_asm_clobber_xmm16(); + raid_asm_end(); } #endif @@ -1992,7 +2005,7 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv) asm volatile("vmovntdq %%ymm5,%0" : "=m" (u[i])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm16(); raid_asm_end(); } @@ -2053,6 +2066,8 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile("movdqa %%xmm2,%0" : "=m" (pa[i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -2159,6 +2174,8 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile("movdqa %%xmm6,%0" : "=m" (pa[1][i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -2235,6 +2252,8 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -2292,7 +2311,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile("vmovdqa %%ymm2,%0" : "=m" (pa[i])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm8(); raid_asm_end(); } @@ -2392,7 +2411,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm8(); raid_asm_end(); } @@ -2469,13 +2488,9 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm8(); raid_asm_end(); } #endif -#ifdef CONFIG_X86 -#pragma GCC pop_options -#endif - diff --git a/raid/x86z.c b/raid/x86z.c index 67a96b35..33518b10 100644 --- a/raid/x86z.c +++ b/raid/x86z.c @@ -14,15 +14,6 @@ #include "internal.h" -/** - * Disables in this file any use of SSE/AVX registers by the compiler - * to avoid to messup with the registers use made by the following functions. - */ -#ifdef CONFIG_X86 -#pragma GCC push_options -#pragma GCC target ("no-sse,no-avx") -#endif - #ifdef CONFIG_X86 static const struct gfzconst16 { uint8_t poly[16]; @@ -98,6 +89,8 @@ void raid_genz_sse2(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm2,%0" : "=m" (r[i])); } + raid_asm_clobber_xmm8(); + raid_asm_end(); } #endif @@ -180,6 +173,8 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv) asm volatile("movntdq %%xmm10,%0" : "=m" (r[i+16])); } + raid_asm_clobber_xmm16(); + raid_asm_end(); } #endif @@ -257,13 +252,10 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv) asm volatile("vmovntdq %%ymm10,%0" : "=m" (r[i+32])); } - asm volatile("vzeroupper" : : : "memory"); + raid_asm_clobber_ymm16(); raid_asm_end(); } #endif -#ifdef CONFIG_X86 -#pragma GCC pop_options -#endif