mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Unroll rec1 single-disk recovery to 64-byte chunks in AVX2, SSSE3, and NEON
This commit is contained in:
Andrea Mazzoleni
parent
7f6ef3b4cd
commit
5b0e1bcd6b
3 changed files with 150 additions and 21 deletions
32
raid/avx2.c
32
raid/avx2.c
|
|
@ -998,17 +998,33 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[V][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfmulpshufb[V][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[i]));
|
||||
asm volatile ("vpxor %ymm1,%ymm0,%ymm0");
|
||||
asm volatile ("vpsrlw $4,%ymm0,%ymm1");
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpsrlw $4,%ymm0,%ymm2");
|
||||
asm volatile ("vpsrlw $4,%ymm1,%ymm3");
|
||||
|
||||
asm volatile ("vpand %ymm7,%ymm0,%ymm0");
|
||||
asm volatile ("vpand %ymm7,%ymm1,%ymm1");
|
||||
asm volatile ("vpshufb %ymm0,%ymm4,%ymm2");
|
||||
asm volatile ("vpshufb %ymm1,%ymm5,%ymm3");
|
||||
asm volatile ("vpxor %ymm3,%ymm2,%ymm2");
|
||||
asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vpand %ymm7,%ymm2,%ymm2");
|
||||
asm volatile ("vpand %ymm7,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vpshufb %ymm0,%ymm4,%ymm0");
|
||||
asm volatile ("vpshufb %ymm1,%ymm4,%ymm1");
|
||||
asm volatile ("vpshufb %ymm2,%ymm5,%ymm2");
|
||||
asm volatile ("vpshufb %ymm3,%ymm5,%ymm3");
|
||||
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
|
|
|
|||
66
raid/neon.c
66
raid/neon.c
|
|
@ -673,20 +673,60 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
"m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile (
|
||||
"ldr q0, %1\n"
|
||||
"ldr q1, %2\n"
|
||||
"eor v0.16b, v0.16b, v1.16b\n"
|
||||
"ushr v17.16b, v0.16b, #4\n"
|
||||
"and v16.16b, v0.16b, v28.16b\n"
|
||||
"and v17.16b, v17.16b, v28.16b\n"
|
||||
"tbl v2.16b, {v24.16b}, v16.16b\n"
|
||||
"tbl v3.16b, {v25.16b}, v17.16b\n"
|
||||
"eor v2.16b, v2.16b, v3.16b\n"
|
||||
"str q2, %0\n"
|
||||
: "=m" (pa[i])
|
||||
: "m" (p[i]), "m" (pa[i])
|
||||
"ldr q0, %4\n"
|
||||
"ldr q1, %5\n"
|
||||
"ldr q2, %6\n"
|
||||
"ldr q3, %7\n"
|
||||
|
||||
"ldr q8, %8\n"
|
||||
"ldr q9, %9\n"
|
||||
"ldr q10, %10\n"
|
||||
"ldr q11, %11\n"
|
||||
|
||||
"eor v0.16b, v0.16b, v8.16b\n"
|
||||
"eor v1.16b, v1.16b, v9.16b\n"
|
||||
"eor v2.16b, v2.16b, v10.16b\n"
|
||||
"eor v3.16b, v3.16b, v11.16b\n"
|
||||
|
||||
"ushr v8.16b, v0.16b, #4\n"
|
||||
"ushr v9.16b, v1.16b, #4\n"
|
||||
"ushr v10.16b, v2.16b, #4\n"
|
||||
"ushr v11.16b, v3.16b, #4\n"
|
||||
|
||||
"and v0.16b, v0.16b, v28.16b\n"
|
||||
"and v1.16b, v1.16b, v28.16b\n"
|
||||
"and v2.16b, v2.16b, v28.16b\n"
|
||||
"and v3.16b, v3.16b, v28.16b\n"
|
||||
|
||||
"and v8.16b, v8.16b, v28.16b\n"
|
||||
"and v9.16b, v9.16b, v28.16b\n"
|
||||
"and v10.16b, v10.16b, v28.16b\n"
|
||||
"and v11.16b, v11.16b, v28.16b\n"
|
||||
|
||||
"tbl v12.16b, {v24.16b}, v0.16b\n"
|
||||
"tbl v13.16b, {v24.16b}, v1.16b\n"
|
||||
"tbl v14.16b, {v24.16b}, v2.16b\n"
|
||||
"tbl v15.16b, {v24.16b}, v3.16b\n"
|
||||
|
||||
"tbl v8.16b, {v25.16b}, v8.16b\n"
|
||||
"tbl v9.16b, {v25.16b}, v9.16b\n"
|
||||
"tbl v10.16b, {v25.16b}, v10.16b\n"
|
||||
"tbl v11.16b, {v25.16b}, v11.16b\n"
|
||||
|
||||
"eor v0.16b, v12.16b, v8.16b\n"
|
||||
"eor v1.16b, v13.16b, v9.16b\n"
|
||||
"eor v2.16b, v14.16b, v10.16b\n"
|
||||
"eor v3.16b, v15.16b, v11.16b\n"
|
||||
|
||||
"str q0, %0\n"
|
||||
"str q1, %1\n"
|
||||
"str q2, %2\n"
|
||||
"str q3, %3\n"
|
||||
: "=m" (pa[i]), "=m" (pa[i + 16]), "=m" (pa[i + 32]), "=m" (pa[i + 48])
|
||||
: "m" (p[i]), "m" (p[i + 16]), "m" (p[i + 32]), "m" (p[i + 48]),
|
||||
"m" (pa[i]), "m" (pa[i + 16]), "m" (pa[i + 32]), "m" (pa[i + 48])
|
||||
);
|
||||
}
|
||||
|
||||
|
|
|
|||
73
raid/ssse3.c
73
raid/ssse3.c
|
|
@ -1258,6 +1258,78 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
raid_sse_begin();
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16]));
|
||||
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[i + 32]));
|
||||
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[i + 48]));
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (pa[i]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (pa[i + 16]));
|
||||
asm volatile ("movdqa %0,%%xmm10" : : "m" (pa[i + 32]));
|
||||
asm volatile ("movdqa %0,%%xmm11" : : "m" (pa[i + 48]));
|
||||
|
||||
asm volatile ("pxor %xmm8,%xmm0");
|
||||
asm volatile ("pxor %xmm9,%xmm1");
|
||||
asm volatile ("pxor %xmm10,%xmm2");
|
||||
asm volatile ("pxor %xmm11,%xmm3");
|
||||
|
||||
asm volatile ("movdqa %xmm0,%xmm8");
|
||||
asm volatile ("movdqa %xmm1,%xmm9");
|
||||
asm volatile ("movdqa %xmm2,%xmm10");
|
||||
asm volatile ("movdqa %xmm3,%xmm11");
|
||||
|
||||
asm volatile ("psrlw $4,%xmm8");
|
||||
asm volatile ("psrlw $4,%xmm9");
|
||||
asm volatile ("psrlw $4,%xmm10");
|
||||
asm volatile ("psrlw $4,%xmm11");
|
||||
|
||||
asm volatile ("pand %xmm7,%xmm0");
|
||||
asm volatile ("pand %xmm7,%xmm1");
|
||||
asm volatile ("pand %xmm7,%xmm2");
|
||||
asm volatile ("pand %xmm7,%xmm3");
|
||||
|
||||
asm volatile ("pand %xmm7,%xmm8");
|
||||
asm volatile ("pand %xmm7,%xmm9");
|
||||
asm volatile ("pand %xmm7,%xmm10");
|
||||
asm volatile ("pand %xmm7,%xmm11");
|
||||
|
||||
asm volatile ("movdqa %xmm4,%xmm12");
|
||||
asm volatile ("movdqa %xmm4,%xmm13");
|
||||
asm volatile ("movdqa %xmm4,%xmm14");
|
||||
asm volatile ("movdqa %xmm4,%xmm15");
|
||||
|
||||
asm volatile ("pshufb %xmm0,%xmm12");
|
||||
asm volatile ("pshufb %xmm1,%xmm13");
|
||||
asm volatile ("pshufb %xmm2,%xmm14");
|
||||
asm volatile ("pshufb %xmm3,%xmm15");
|
||||
|
||||
asm volatile ("movdqa %xmm5,%xmm0");
|
||||
asm volatile ("movdqa %xmm5,%xmm1");
|
||||
asm volatile ("movdqa %xmm5,%xmm2");
|
||||
asm volatile ("movdqa %xmm5,%xmm3");
|
||||
|
||||
asm volatile ("pshufb %xmm8,%xmm0");
|
||||
asm volatile ("pshufb %xmm9,%xmm1");
|
||||
asm volatile ("pshufb %xmm10,%xmm2");
|
||||
asm volatile ("pshufb %xmm11,%xmm3");
|
||||
|
||||
asm volatile ("pxor %xmm12,%xmm0");
|
||||
asm volatile ("pxor %xmm13,%xmm1");
|
||||
asm volatile ("pxor %xmm14,%xmm2");
|
||||
asm volatile ("pxor %xmm15,%xmm3");
|
||||
|
||||
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16]));
|
||||
asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32]));
|
||||
asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48]));
|
||||
}
|
||||
#else /* CONFIG_X86_32 */
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
|
||||
|
|
@ -1277,6 +1349,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
asm volatile ("pxor %xmm3,%xmm2");
|
||||
asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i]));
|
||||
}
|
||||
#endif
|
||||
|
||||
raid_sse_end();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue