diff --git a/raid/internal.h b/raid/internal.h index e3995546..027d9e33 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -195,20 +195,15 @@ static __always_inline void raid_asm_begin(void) { } -static __always_inline void raid_asm_end(void) +#ifdef CONFIG_SSE2 +static __always_inline void raid_asm_sse_end(void) { - /* SSE2 and AVX2 code uses non-temporal writes, like MOVNTDQ, */ + /* SSE and AVX code uses non-temporal writes, like MOVNTDQ, */ /* that use a weak memory model. To ensure that other processors */ /* see correctly the data written, we use a store-store memory */ /* barrier at the end of the asm code */ -#ifdef CONFIG_SSE2 asm volatile ("sfence" : : : "memory"); -#endif -} -#ifdef CONFIG_SSE2 -static __always_inline void raid_asm_clobber_xmm4(void) -{ /* clobbers registers used in the asm code */ /* this is required because in the Windows ABI, */ /* registers xmm6-xmm15 should be kept by the callee. */ @@ -218,60 +213,27 @@ static __always_inline void raid_asm_clobber_xmm4(void) /* compiler supports SSE2 registers in the clobber list */ #ifdef __SSE2__ asm volatile ("" : : : "%xmm0", "%xmm1", "%xmm2", "%xmm3"); -#endif -} -#endif - -#ifdef CONFIG_SSE2 -static __always_inline void raid_asm_clobber_xmm8(void) -{ - raid_asm_clobber_xmm4(); -#ifdef __SSE2__ asm volatile ("" : : : "%xmm4", "%xmm5", "%xmm6", "%xmm7"); +#ifdef CONFIG_X86_64 + asm volatile ("" : : : "%xmm8", "%xmm9", "%xmm10", "%xmm11"); + asm volatile ("" : : : "%xmm12", "%xmm13", "%xmm14", "%xmm15"); +#endif #endif } #endif #ifdef CONFIG_AVX2 -static __always_inline void raid_asm_clobber_ymm4(void) +static __always_inline void raid_asm_avx_end(void) { - raid_asm_clobber_xmm4(); + raid_asm_sse_end(); + /* reset the upper part of the ymm registers */ /* to avoid the 70 clocks penality on the next */ /* xmm register use */ asm volatile ("vzeroupper" : : : "memory"); } #endif - -#ifdef CONFIG_AVX2 -static __always_inline void raid_asm_clobber_ymm8(void) -{ - raid_asm_clobber_xmm8(); - asm volatile ("vzeroupper" : : : "memory"); -} -#endif #endif /* CONFIG_X86 */ -#ifdef CONFIG_X86_64 -#ifdef CONFIG_SSE2 -static __always_inline void raid_asm_clobber_xmm16(void) -{ - raid_asm_clobber_xmm8(); -#ifdef __SSE2__ - asm volatile ("" : : : "%xmm8", "%xmm9", "%xmm10", "%xmm11"); - asm volatile ("" : : : "%xmm12", "%xmm13", "%xmm14", "%xmm15"); -#endif -} -#endif - -#ifdef CONFIG_AVX2 -static __always_inline void raid_asm_clobber_ymm16(void) -{ - raid_asm_clobber_xmm16(); - asm volatile ("vzeroupper" : : : "memory"); -} -#endif -#endif /* CONFIG_X86_64 */ - #endif diff --git a/raid/x86.c b/raid/x86.c index 148b8035..2c470a60 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -52,9 +52,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48])); } - raid_asm_clobber_xmm4(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -89,9 +87,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); } - raid_asm_clobber_ymm4(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -162,9 +158,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -217,9 +211,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); } - raid_asm_clobber_ymm8(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -299,9 +291,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48])); } - raid_asm_clobber_xmm16(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -398,9 +388,7 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -541,9 +529,7 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); } - raid_asm_clobber_xmm16(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -676,9 +662,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); } - raid_asm_clobber_ymm16(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -794,9 +778,7 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -969,9 +951,7 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16])); } - raid_asm_clobber_xmm16(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -1132,9 +1112,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32])); } - raid_asm_clobber_ymm16(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -1274,9 +1252,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -1409,9 +1385,7 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i])); } - raid_asm_clobber_xmm16(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -1541,9 +1515,7 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i])); } - raid_asm_clobber_ymm16(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -1705,9 +1677,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -1857,9 +1827,7 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i])); } - raid_asm_clobber_xmm16(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -2006,9 +1974,7 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm5,%0" : "=m" (u[i])); } - raid_asm_clobber_ymm16(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -2067,9 +2033,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -2175,9 +2139,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -2253,9 +2215,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -2312,9 +2272,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i])); } - raid_asm_clobber_ymm8(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -2412,9 +2370,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); } - raid_asm_clobber_ymm8(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif @@ -2489,9 +2445,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) } } - raid_asm_clobber_ymm8(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif diff --git a/raid/x86z.c b/raid/x86z.c index 0451c026..77d5ffd9 100644 --- a/raid/x86z.c +++ b/raid/x86z.c @@ -90,9 +90,7 @@ void raid_genz_sse2(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i])); } - raid_asm_clobber_xmm8(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -174,9 +172,7 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv) asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16])); } - raid_asm_clobber_xmm16(); - - raid_asm_end(); + raid_asm_sse_end(); } #endif @@ -253,9 +249,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv) asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32])); } - raid_asm_clobber_ymm16(); - - raid_asm_end(); + raid_asm_avx_end(); } #endif