mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Use specialized assembler header and footer for sse and avx
This commit is contained in:
Andrea Mazzoleni
parent
555b161df1
commit
a99cd9daff
3 changed files with 61 additions and 57 deletions
|
|
@ -191,12 +191,12 @@ extern const uint8_t (*raid_gfgen)[256];
|
|||
* Assembler blocks.
|
||||
*/
|
||||
#ifdef CONFIG_X86
|
||||
static __always_inline void raid_asm_begin(void)
|
||||
#ifdef CONFIG_SSE2
|
||||
static __always_inline void raid_sse_begin(void)
|
||||
{
|
||||
}
|
||||
|
||||
#ifdef CONFIG_SSE2
|
||||
static __always_inline void raid_asm_sse_end(void)
|
||||
static __always_inline void raid_sse_end(void)
|
||||
{
|
||||
/* SSE and AVX code uses non-temporal writes, like MOVNTDQ, */
|
||||
/* that use a weak memory model. To ensure that other processors */
|
||||
|
|
@ -223,9 +223,13 @@ static __always_inline void raid_asm_sse_end(void)
|
|||
#endif
|
||||
|
||||
#ifdef CONFIG_AVX2
|
||||
static __always_inline void raid_asm_avx_end(void)
|
||||
static __always_inline void raid_avx_begin(void)
|
||||
{
|
||||
raid_asm_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_avx_end(void)
|
||||
{
|
||||
raid_sse_end();
|
||||
|
||||
/* reset the upper part of the ymm registers */
|
||||
/* to avoid the 70 clocks penality on the next */
|
||||
|
|
|
|||
92
raid/x86.c
92
raid/x86.c
|
|
@ -33,7 +33,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
|
|||
l = nd - 1;
|
||||
p = v[nd];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
|
||||
|
|
@ -52,7 +52,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm3,%0" : "=m" (p[i + 48]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -74,7 +74,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
|
|||
l = nd - 1;
|
||||
p = v[nd];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));
|
||||
|
|
@ -87,7 +87,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -124,7 +124,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv)
|
|||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
|
||||
|
||||
|
|
@ -158,7 +158,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm3,%0" : "=m" (q[i + 16]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -178,7 +178,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
|
|||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0, %%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
|
||||
|
|
@ -211,7 +211,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -233,7 +233,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv)
|
|||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm15" : : "m" (gfconst16.poly[0]));
|
||||
|
||||
|
|
@ -291,7 +291,7 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm7,%0" : "=m" (q[i + 48]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -320,7 +320,7 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));
|
||||
|
|
@ -388,7 +388,7 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -419,7 +419,7 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfconst16.poly[0]));
|
||||
|
|
@ -529,7 +529,7 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -560,7 +560,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vbroadcasti128 %0, %%ymm3" : : "m" (gfconst16.poly[0]));
|
||||
|
|
@ -662,7 +662,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -693,7 +693,7 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 16) {
|
||||
|
|
@ -778,7 +778,7 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm3,%0" : "=m" (s[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -811,7 +811,7 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 32) {
|
||||
|
|
@ -951,7 +951,7 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm11,%0" : "=m" (s[i + 16]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -984,7 +984,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 64) {
|
||||
|
|
@ -1112,7 +1112,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm11,%0" : "=m" (s[i + 32]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1150,7 +1150,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 16) {
|
||||
|
|
@ -1252,7 +1252,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm3,%0" : "=m" (t[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1287,7 +1287,7 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0]));
|
||||
|
|
@ -1385,7 +1385,7 @@ void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm4,%0" : "=m" (t[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1420,7 +1420,7 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxor %ymm8,%ymm8,%ymm8");
|
||||
|
|
@ -1515,7 +1515,7 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (t[i]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1556,7 +1556,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 16) {
|
||||
|
|
@ -1677,7 +1677,7 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm3,%0" : "=m" (u[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1714,7 +1714,7 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("movdqa %0,%%xmm14" : : "m" (gfconst16.poly[0]));
|
||||
|
|
@ -1827,7 +1827,7 @@ void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm5,%0" : "=m" (u[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -1864,7 +1864,7 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv)
|
|||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxor %ymm8,%ymm8,%ymm8");
|
||||
|
|
@ -1974,7 +1974,7 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (u[i]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -2011,7 +2011,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (gfmulpshufb[V][0][0]));
|
||||
|
|
@ -2033,7 +2033,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -2070,7 +2070,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
|
|
@ -2139,7 +2139,7 @@ void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
asm volatile ("movdqa %%xmm6,%0" : "=m" (pa[1][i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -2174,7 +2174,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
|
|
@ -2215,7 +2215,7 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
}
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -2252,7 +2252,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0]));
|
||||
|
|
@ -2271,7 +2271,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -2308,7 +2308,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
|
|
@ -2369,7 +2369,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -2404,7 +2404,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
|
|
@ -2444,7 +2444,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
}
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
12
raid/x86z.c
12
raid/x86z.c
|
|
@ -54,7 +54,7 @@ void raid_genz_sse2(int nd, size_t size, void **vv)
|
|||
q = v[nd + 1];
|
||||
r = v[nd + 2];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0]));
|
||||
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0]));
|
||||
|
|
@ -90,7 +90,7 @@ void raid_genz_sse2(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm2,%0" : "=m" (r[i]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -114,7 +114,7 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv)
|
|||
q = v[nd + 1];
|
||||
r = v[nd + 2];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfzconst16.poly[0]));
|
||||
asm volatile ("movdqa %0,%%xmm3" : : "m" (gfzconst16.half[0]));
|
||||
|
|
@ -172,7 +172,7 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("movntdq %%xmm10,%0" : "=m" (r[i + 16]));
|
||||
}
|
||||
|
||||
raid_asm_sse_end();
|
||||
raid_sse_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
@ -196,7 +196,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv)
|
|||
q = v[nd + 1];
|
||||
r = v[nd + 2];
|
||||
|
||||
raid_asm_begin();
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfzconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfzconst16.half[0]));
|
||||
|
|
@ -249,7 +249,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv)
|
|||
asm volatile ("vmovntdq %%ymm10,%0" : "=m" (r[i + 32]));
|
||||
}
|
||||
|
||||
raid_asm_avx_end();
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue