From 5b0e1bcd6bd2aa063e5487935b970b447640f8dc Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Thu, 20 Aug 2026 17:20:45 +0200 Subject: [PATCH] Unroll rec1 single-disk recovery to 64-byte chunks in AVX2, SSSE3, and NEON --- raid/avx2.c | 32 +++++++++++++++++------ raid/neon.c | 66 +++++++++++++++++++++++++++++++++++++---------- raid/ssse3.c | 73 ++++++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 21 deletions(-) diff --git a/raid/avx2.c b/raid/avx2.c index e3db58fd..10c53226 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -998,17 +998,33 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[V][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (raid_gfmulpshufb[V][1][0])); - for (i = 0; i < size; i += 32) { + for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (pa[i])); - asm volatile ("vpxor %ymm1,%ymm0,%ymm0"); - asm volatile ("vpsrlw $4,%ymm0,%ymm1"); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vpsrlw $4,%ymm0,%ymm2"); + asm volatile ("vpsrlw $4,%ymm1,%ymm3"); + asm volatile ("vpand %ymm7,%ymm0,%ymm0"); asm volatile ("vpand %ymm7,%ymm1,%ymm1"); - asm volatile ("vpshufb %ymm0,%ymm4,%ymm2"); - asm volatile ("vpshufb %ymm1,%ymm5,%ymm3"); - asm volatile ("vpxor %ymm3,%ymm2,%ymm2"); - asm volatile ("vmovdqa %%ymm2,%0" : "=m" (pa[i])); + asm volatile ("vpand %ymm7,%ymm2,%ymm2"); + asm volatile ("vpand %ymm7,%ymm3,%ymm3"); + + asm volatile ("vpshufb %ymm0,%ymm4,%ymm0"); + asm volatile ("vpshufb %ymm1,%ymm4,%ymm1"); + asm volatile ("vpshufb %ymm2,%ymm5,%ymm2"); + asm volatile ("vpshufb %ymm3,%ymm5,%ymm3"); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); } raid_avx_end(); diff --git a/raid/neon.c b/raid/neon.c index b4247bf1..ea79c65b 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -673,20 +673,60 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) "m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0]) ); - for (i = 0; i < size; i += 16) { + for (i = 0; i < size; i += 64) { asm volatile ( - "ldr q0, %1\n" - "ldr q1, %2\n" - "eor v0.16b, v0.16b, v1.16b\n" - "ushr v17.16b, v0.16b, #4\n" - "and v16.16b, v0.16b, v28.16b\n" - "and v17.16b, v17.16b, v28.16b\n" - "tbl v2.16b, {v24.16b}, v16.16b\n" - "tbl v3.16b, {v25.16b}, v17.16b\n" - "eor v2.16b, v2.16b, v3.16b\n" - "str q2, %0\n" - : "=m" (pa[i]) - : "m" (p[i]), "m" (pa[i]) + "ldr q0, %4\n" + "ldr q1, %5\n" + "ldr q2, %6\n" + "ldr q3, %7\n" + + "ldr q8, %8\n" + "ldr q9, %9\n" + "ldr q10, %10\n" + "ldr q11, %11\n" + + "eor v0.16b, v0.16b, v8.16b\n" + "eor v1.16b, v1.16b, v9.16b\n" + "eor v2.16b, v2.16b, v10.16b\n" + "eor v3.16b, v3.16b, v11.16b\n" + + "ushr v8.16b, v0.16b, #4\n" + "ushr v9.16b, v1.16b, #4\n" + "ushr v10.16b, v2.16b, #4\n" + "ushr v11.16b, v3.16b, #4\n" + + "and v0.16b, v0.16b, v28.16b\n" + "and v1.16b, v1.16b, v28.16b\n" + "and v2.16b, v2.16b, v28.16b\n" + "and v3.16b, v3.16b, v28.16b\n" + + "and v8.16b, v8.16b, v28.16b\n" + "and v9.16b, v9.16b, v28.16b\n" + "and v10.16b, v10.16b, v28.16b\n" + "and v11.16b, v11.16b, v28.16b\n" + + "tbl v12.16b, {v24.16b}, v0.16b\n" + "tbl v13.16b, {v24.16b}, v1.16b\n" + "tbl v14.16b, {v24.16b}, v2.16b\n" + "tbl v15.16b, {v24.16b}, v3.16b\n" + + "tbl v8.16b, {v25.16b}, v8.16b\n" + "tbl v9.16b, {v25.16b}, v9.16b\n" + "tbl v10.16b, {v25.16b}, v10.16b\n" + "tbl v11.16b, {v25.16b}, v11.16b\n" + + "eor v0.16b, v12.16b, v8.16b\n" + "eor v1.16b, v13.16b, v9.16b\n" + "eor v2.16b, v14.16b, v10.16b\n" + "eor v3.16b, v15.16b, v11.16b\n" + + "str q0, %0\n" + "str q1, %1\n" + "str q2, %2\n" + "str q3, %3\n" + : "=m" (pa[i]), "=m" (pa[i + 16]), "=m" (pa[i + 32]), "=m" (pa[i + 48]) + : "m" (p[i]), "m" (p[i + 16]), "m" (p[i + 32]), "m" (p[i + 48]), + "m" (pa[i]), "m" (pa[i + 16]), "m" (pa[i + 32]), "m" (pa[i + 48]) ); } diff --git a/raid/ssse3.c b/raid/ssse3.c index 0787c80d..394efd51 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1258,6 +1258,78 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_sse_begin(); +#ifdef CONFIG_X86_64 + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0])); + asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (p[i + 32])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (p[i + 48])); + + asm volatile ("movdqa %0,%%xmm8" : : "m" (pa[i])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (pa[i + 16])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (pa[i + 32])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (pa[i + 48])); + + asm volatile ("pxor %xmm8,%xmm0"); + asm volatile ("pxor %xmm9,%xmm1"); + asm volatile ("pxor %xmm10,%xmm2"); + asm volatile ("pxor %xmm11,%xmm3"); + + asm volatile ("movdqa %xmm0,%xmm8"); + asm volatile ("movdqa %xmm1,%xmm9"); + asm volatile ("movdqa %xmm2,%xmm10"); + asm volatile ("movdqa %xmm3,%xmm11"); + + asm volatile ("psrlw $4,%xmm8"); + asm volatile ("psrlw $4,%xmm9"); + asm volatile ("psrlw $4,%xmm10"); + asm volatile ("psrlw $4,%xmm11"); + + asm volatile ("pand %xmm7,%xmm0"); + asm volatile ("pand %xmm7,%xmm1"); + asm volatile ("pand %xmm7,%xmm2"); + asm volatile ("pand %xmm7,%xmm3"); + + asm volatile ("pand %xmm7,%xmm8"); + asm volatile ("pand %xmm7,%xmm9"); + asm volatile ("pand %xmm7,%xmm10"); + asm volatile ("pand %xmm7,%xmm11"); + + asm volatile ("movdqa %xmm4,%xmm12"); + asm volatile ("movdqa %xmm4,%xmm13"); + asm volatile ("movdqa %xmm4,%xmm14"); + asm volatile ("movdqa %xmm4,%xmm15"); + + asm volatile ("pshufb %xmm0,%xmm12"); + asm volatile ("pshufb %xmm1,%xmm13"); + asm volatile ("pshufb %xmm2,%xmm14"); + asm volatile ("pshufb %xmm3,%xmm15"); + + asm volatile ("movdqa %xmm5,%xmm0"); + asm volatile ("movdqa %xmm5,%xmm1"); + asm volatile ("movdqa %xmm5,%xmm2"); + asm volatile ("movdqa %xmm5,%xmm3"); + + asm volatile ("pshufb %xmm8,%xmm0"); + asm volatile ("pshufb %xmm9,%xmm1"); + asm volatile ("pshufb %xmm10,%xmm2"); + asm volatile ("pshufb %xmm11,%xmm3"); + + asm volatile ("pxor %xmm12,%xmm0"); + asm volatile ("pxor %xmm13,%xmm1"); + asm volatile ("pxor %xmm14,%xmm2"); + asm volatile ("pxor %xmm15,%xmm3"); + + asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48])); + } +#else /* CONFIG_X86_32 */ asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[V][0][0])); asm volatile ("movdqa %0,%%xmm5" : : "m" (raid_gfmulpshufb[V][1][0])); @@ -1277,6 +1349,7 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) asm volatile ("pxor %xmm3,%xmm2"); asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i])); } +#endif raid_sse_end(); }