diff --git a/cmdline/speed.c b/cmdline/speed.c index 841bb40c..2e1dd36e 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -1265,8 +1265,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - /* raid_rec1of1() internally calls raid_gen() */ - raid_gen_force(1, raid_gen1_neon); raid_rec1_neon(1, id, ip, nd, size, v); } SPEED_STOP @@ -1332,8 +1330,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512bw()) { SPEED_START { - /* raid_rec1of1() internally calls raid_gen() */ - raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512bw(1, id, ip, nd, size, v); } SPEED_STOP @@ -1342,8 +1338,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - /* raid_rec1of1() internally calls raid_gen() */ - raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP @@ -1352,8 +1346,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - /* raid_rec1of1() internally calls raid_gen() */ - raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP diff --git a/raid/NOTES b/raid/NOTES index b6fe2330..dccc9ad7 100644 --- a/raid/NOTES +++ b/raid/NOTES @@ -54,17 +54,19 @@ Generation: Recovery: -* The normal one-data-disk recovery with P available is already handled by - raid_rec1of1(), which is essentially an XOR generation pass with pointer - remapping. +* The normal one-data-disk recovery with P available is handled directly by the + fused 1-pass SIMD kernel on AVX512BW, GFNI, and AArch64 NEON (pure XOR + streaming), or by raid_rec1of1() on scalar and older SIMD paths. * SIMD recovery has parity-count-specific REC1..REC6 entry points. Fused paths compute exactly the selected syndromes in one scan of the surviving data and reconstruct the missing blocks without full-size temporary syndrome buffers. -* AVX512BW, GFNI, AArch64 NEON, and AArch32 NEON use fused syndrome generation - for all non-P REC1 and REC2 cases and for REC3..REC6. SSSE3 and AVX2 use it - for REC3..REC6, with additional fused cases in the extended AVX2 path. +* AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON use fused syndrome + generation for all REC1..REC6 cases, including P (rec1of1) and P+Q (rec2of2). + AArch32 NEON uses fused syndrome generation for all non-P REC1 and REC2 cases + and for REC3..REC6. SSSE3 and AVX2 use it for REC3..REC6, with additional + fused cases in the extended AVX2 path. * Scalar recovery and selected SSSE3/AVX2 REC1/REC2 paths continue to use raid_delta_gen(). @@ -82,7 +84,9 @@ Recovery: or broadcasting inside the data loop. * Specialized P+Q rec2of2 recovery reduces 4 GF multiplications to 2 GF - multiplications and 1 XOR, implemented across scalar int8 and all SIMD backends. + multiplications and 1 XOR; AVX512BW, GFNI, and AArch64 NEON handle this + directly inside the 1-pass fused kernel, while scalar int8, SSSE3, AVX2, and + AArch32 NEON backends implement dedicated rec2of2 helpers. * When P is among the selected parity blocks, optimized SIMD RECX paths reconstruct only nr - 1 missing blocks through the inverse matrix and derive @@ -184,7 +188,10 @@ Typical recovery case * raid_rec() selects the first available parity. Therefore, for a single data failure with P available, recovery uses P. -* REC1 implementations special-case ip[0] == 0 and call raid_rec1of1(). +* REC1 implementations on AVX512BW, GFNI, and AArch64 NEON execute directly + inside the fused streaming kernel (reducing to a single-pass pure XOR stream + without memory swapping), while scalar and older SIMD paths call + raid_rec1of1(). * raid_rec1of1() remaps pointers so that P is used as an XOR input and the failed data buffer is used as the GEN1 output, then calls @@ -221,22 +228,12 @@ REC1OF1: * The row uses ip[0] == 0. -* Every REC1 wrapper therefore enters raid_rec1of1() instead of executing its - normal GF recovery body. +* On AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON, REC1 executes directly + inside the fused 1-pass kernel, streaming surviving data blocks and P in a + single pass. -* The speed test explicitly selects the generation function used by the - raid_rec1of1() call with raid_gen_force(1, ...). - -* There is no GFNI GEN1 because PAR1 is only XOR. Therefore: - - rec1of1 avx2 -> raid_gen1_avx2 - rec1of1 gfni -> raid_gen1_avx2 - - rec1of1 avx512 -> raid_gen1_avx512bw - rec1of1 gfni512 -> raid_gen1_avx512bw - - This is intentional. The GFNI recovery wrapper is selected, but the fast path - naturally reduces to the best available XOR generator. +* For scalar and older SIMD wrappers, REC1 enters raid_rec1of1() to recompute P + using the generation backend. REC1OF2 and REC1OF3: @@ -249,8 +246,12 @@ REC2OF2: * The row uses ip = { 0, 1 }, meaning P and Q. -* Scalar int8 and all SIMD REC2 implementations have a specialized P+Q fast path - and call raid_rec2of2_*(). +* On AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON, REC2 executes + directly in the 1-pass fused kernel, which computes P with XOR, Q with GF + multiplication, and derives the second missing block via XOR. + +* Scalar int8, SSSE3, AVX2, and AArch32 NEON call their specialized + raid_rec2of2_*() delta helpers. * This makes the row a direct benchmark for the specialized P+Q recovery implementation. @@ -310,8 +311,9 @@ raid_delta_gen() fusion according to register pressure; they do not allocate full-size syndrome buffers. -* The common rec1of1/P case remains the direct XOR generation path and does not - enter fused GF recovery. +* The common rec1of1/P case on AVX512BW and GFNI executes as pure XOR streaming + within the fused kernel (no GF matrix multiplication), while on older backends + it uses the raid_rec1of1() pointer remapping path. * raid_delta_gen() remains in the scalar implementation and selected SSSE3/AVX2 REC1/REC2 paths. @@ -642,8 +644,9 @@ Memory layout, displacement and stride noise REC1 optimizations ------------------ -The important normal REC1 case is already raid_rec1of1() with P and is close to -the minimum possible memory traffic. +The normal REC1 case with P is handled either directly in the fused 1-pass SIMD +kernel (AVX512BW, GFNI, AArch64 NEON) as pure XOR streaming, or via +raid_rec1of1() (scalar, legacy SIMD). For generic REC1, used when P is unavailable: @@ -658,7 +661,7 @@ For generic REC1, used when P is unavailable: REC1 status: * For the current large-block SnapRAID workload and implemented backends, - no major rec1of1/P optimization is currently known. + fused single-pass streaming achieves optimal memory saturation for rec1of1/P. * Generic non-P REC1 is selected per parity count and backend, so fused and delta-based implementations are benchmarked independently. @@ -671,8 +674,9 @@ REC2 optimizations REC2 has both fused and delta-based implementations. -* AVX512BW, GFNI, AArch64 NEON, and AArch32 NEON use fused syndrome generation - for every selected parity pair. +* AVX512BW, GFNI, and AArch64 NEON use fused syndrome generation for every + selected parity pair (including P+Q). AArch32 NEON uses fused generation for + non-P pairs and REC3..REC6. * AVX2 extended uses its delta kernel for P+Q and fused recovery for other parity pairs. @@ -695,7 +699,9 @@ REC2 status: * Generic REC2 uses the fused path where available and retains local split reuse in the delta-based paths. -* Scalar int8 and all SIMD backends implement specialized rec2of2 P+Q recovery. +* AVX512BW, GFNI, and AArch64 NEON handle P+Q recovery directly inside the + 1-pass fused kernel, while scalar int8, SSSE3, AVX2, and AArch32 NEON + backends implement dedicated rec2of2 delta helpers. Specialized rec2of2 P+Q @@ -710,14 +716,16 @@ can be reduced to: This needs only two GF multiplications for Dy, followed by one XOR for Dx, instead of the generic 2x2 reconstruction with four GF multiplications. -All backends implement this specialization: +Backends implement this specialization as follows: +* AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON integrate this derivation + directly into the unified 1-pass fused kernel (P is accumulated via XOR, Q via + GF multiplication, Dy is solved with inverse coefficients, and Dx derived by + XOR with Pdelta). * Scalar int8 (raid_rec2of2_int8) * SSSE3 and SSSE3 ext (raid_rec2of2_ssse3, raid_rec2of2_ssse3ext) * AVX2 and AVX2 ext (raid_rec2of2_avx2, raid_rec2of2_avx2ext) -* AVX512BW (raid_rec2of2_avx512bw) -* AVX2 GFNI and AVX512 GFNI (RAID and AES variants) -* AArch64 NEON and AArch32 NEON (raid_rec2of2_neon, raid_rec2of2_neon32) +* AArch32 NEON (raid_rec2of2_neon32) The rec2of2 benchmark row directly measures this specialized P+Q fast path. diff --git a/raid/avx512.c b/raid/avx512.c index fdfa84fd..0afe04e7 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -911,102 +911,12 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv) void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - raid_recX_avx512bw(1, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AVX512BW implementation. - * - * Uses raid_delta_gen() and keeps both multiplication-table pairs resident - * in ZMM registers (zmm16..zmm19). - */ -static __always_inline void raid_rec2of2_avx512bw(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - uint8_t C[2]; - size_t i; - - /* get multiplication coefficients */ - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - /* compute delta parity */ - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); - - /* keep both multiplication-table pairs resident */ - asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[C[0]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[C[0]][1][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[C[1]][0][0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[C[1]][1][0])); - - for (i = 0; i < size; i += 64) { - /* Pd */ - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vpxord %0,%%zmm0,%%zmm0" : : "m" (pa[i])); - - /* Qd */ - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i])); - asm volatile ("vpxord %0,%%zmm1,%%zmm1" : : "m" (qa[i])); - - /* split Pd and Qd */ - asm volatile ("vpsrlw $4,%zmm0,%zmm2"); - asm volatile ("vpsrlw $4,%zmm1,%zmm3"); - asm volatile ("vpandq %zmm31,%zmm0,%zmm4"); - asm volatile ("vpandq %zmm31,%zmm1,%zmm5"); - asm volatile ("vpandq %zmm31,%zmm2,%zmm2"); - asm volatile ("vpandq %zmm31,%zmm3,%zmm3"); - - /* C0 * Pd */ - asm volatile ("vpshufb %zmm4,%zmm16,%zmm6"); - asm volatile ("vpshufb %zmm2,%zmm17,%zmm7"); - asm volatile ("vpxord %zmm7,%zmm6,%zmm6"); - - /* C1 * Qd */ - asm volatile ("vpshufb %zmm5,%zmm18,%zmm7"); - asm volatile ("vpxord %zmm7,%zmm6,%zmm6"); - asm volatile ("vpshufb %zmm3,%zmm19,%zmm7"); - asm volatile ("vpxord %zmm7,%zmm6,%zmm6"); - - /* Dy = zmm6, Dx = Pd ^ Dy */ - asm volatile ("vpxord %zmm6,%zmm0,%zmm0"); - - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (qa[i])); - } - - raid_avx_end(); -} - void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - - /* if recovering with P,Q use the specialized function */ - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_avx512bw(id, ip, nd, size, vv); - return; - } - raid_recX_avx512bw(2, id, ip, nd, size, vv); } diff --git a/raid/gfni.c b/raid/gfni.c index 319ca2ab..fc7da939 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -2203,94 +2203,12 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AVX2 RAID GFNI implementation. - * - * Uses raid_delta_gen() and keeps both affine matrices resident in YMM registers. - */ -static __always_inline void raid_rec2of2_avx2gfni_raid(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - const uint8_t *T[2]; - uint8_t C[2]; - size_t i; - - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - T[0] = raid_gfaffine_raid[C[0]]; - T[1] = raid_gfaffine_raid[C[1]]; - - /* compute delta parity */ - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (T[0][0])); - asm volatile ("vpbroadcastq %0,%%ymm15" : : "m" (T[1][0])); - - for (i = 0; i < size; i += 64) { - /* Pd, two 32-byte lanes */ - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); - asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i])); - asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32])); - - /* Qd, two 32-byte lanes */ - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32])); - asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i])); - asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32])); - - /* Dy = C0 * Pd ^ C1 * Qd */ - asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm4"); - asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm5"); - asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm2,%ymm6"); - asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm3,%ymm7"); - asm volatile ("vpxor %ymm6,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm7,%ymm5,%ymm5"); - - /* Dx = Pd ^ Dy */ - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); - asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i])); - asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32])); - } - - raid_avx_end(); -} - void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_avx2gfni_raid(id, ip, nd, size, vv); - return; - } - raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv); } @@ -2321,80 +2239,12 @@ void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AVX512 RAID GFNI implementation. - * - * Uses raid_delta_gen() and keeps both affine matrices resident in ZMM registers. - */ -static __always_inline void raid_rec2of2_avx512gfni_raid(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - const uint8_t *T[2]; - uint8_t C[2]; - size_t i; - - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - T[0] = raid_gfaffine_raid[C[0]]; - T[1] = raid_gfaffine_raid[C[1]]; - - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (T[0][0])); - asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (T[1][0])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i])); - - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i])); - asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i])); - - asm volatile ("vgf2p8affineqb $0,%zmm14,%zmm0,%zmm2"); - asm volatile ("vgf2p8affineqb $0,%zmm15,%zmm1,%zmm3"); - asm volatile ("vpxorq %zmm3,%zmm2,%zmm2"); - - /* Dx = Pd ^ Dy */ - asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i])); - } - - raid_avx_end(); -} - void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_avx512gfni_raid(id, ip, nd, size, vv); - return; - } - raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv); } @@ -2425,85 +2275,12 @@ void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AVX2 AES GFNI implementation. - * - * Uses raid_delta_gen() and keeps both byte multipliers resident in YMM registers. - */ -static __always_inline void raid_rec2of2_avx2gfni_aes(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - uint8_t C[2]; - size_t i; - - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (C[0])); - asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (C[1])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); - asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); - asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i])); - asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32])); - - asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i])); - asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32])); - asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i])); - asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32])); - - asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm4"); - asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm5"); - asm volatile ("vgf2p8mulb %ymm15,%ymm2,%ymm6"); - asm volatile ("vgf2p8mulb %ymm15,%ymm3,%ymm7"); - asm volatile ("vpxor %ymm6,%ymm4,%ymm4"); - asm volatile ("vpxor %ymm7,%ymm5,%ymm5"); - - asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); - asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); - - asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); - asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i])); - asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32])); - } - - raid_avx_end(); -} - void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_avx2gfni_aes(id, ip, nd, size, vv); - return; - } - raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv); } @@ -2534,75 +2311,12 @@ void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AVX512 AES GFNI implementation. - * - * Uses raid_delta_gen() and keeps both byte multipliers resident in ZMM registers. - */ -static __always_inline void raid_rec2of2_avx512gfni_aes(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - uint8_t C[2]; - size_t i; - - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_avx_begin(); - - asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (C[0])); - asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (C[1])); - - for (i = 0; i < size; i += 64) { - asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); - asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i])); - - asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i])); - asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i])); - - asm volatile ("vgf2p8mulb %zmm14,%zmm0,%zmm2"); - asm volatile ("vgf2p8mulb %zmm15,%zmm1,%zmm3"); - asm volatile ("vpxorq %zmm3,%zmm2,%zmm2"); - - asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); - - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); - asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i])); - } - - raid_avx_end(); -} - void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_avx512gfni_aes(id, ip, nd, size, vv); - return; - } - raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv); } diff --git a/raid/neon.c b/raid/neon.c index ef88c790..c8139fa6 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -1036,145 +1036,12 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv) void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - - /* if recovering with P uses the delta function */ - if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); - return; - } - raid_recX_neon(1, id, ip, nd, size, vv); } -/* - * Recover failure of two data blocks using P and Q AArch64 NEON implementation. - * - * Uses raid_delta_gen() and keeps all four multiplication tables resident - * in NEON registers (q20..q23) with a 32-byte two-lane unroll. - */ -static __always_inline void raid_rec2of2_neon(int *id, int *ip, int nd, size_t size, void **vv) -{ - uint8_t **v = (uint8_t **)vv; - uint8_t *p; - uint8_t *pa; - uint8_t *q; - uint8_t *qa; - uint8_t C[2]; - size_t i; - - C[0] = inv(powgen(id[1] - id[0]) ^ 1); - C[1] = inv(powgen(id[0]) ^ powgen(id[1])); - - raid_delta_gen(2, id, ip, nd, size, vv); - - p = v[nd]; - q = v[nd + 1]; - pa = v[id[0]]; - qa = v[id[1]]; - - raid_neon_begin(); - - asm volatile ( - "ldr q28, %0\n" - "ldr q20, %1\n" - "ldr q21, %2\n" - "ldr q22, %3\n" - "ldr q23, %4\n" - : - : "m" (gfconst16.low4[0]), - "m" (raid_gfmulpshufb[C[0]][0][0]), - "m" (raid_gfmulpshufb[C[0]][1][0]), - "m" (raid_gfmulpshufb[C[1]][0][0]), - "m" (raid_gfmulpshufb[C[1]][1][0]) - ); - - for (i = 0; i < size; i += 32) { - asm volatile ( - /* Pd, two lanes */ - "ldr q0, %4\n" - "ldr q1, %5\n" - "ldr q12, %8\n" - "ldr q13, %9\n" - "eor v0.16b, v0.16b, v12.16b\n" - "eor v1.16b, v1.16b, v13.16b\n" - - /* preserve Pd for Dx */ - "mov v8.16b, v0.16b\n" - "mov v9.16b, v1.16b\n" - - /* Qd, two lanes */ - "ldr q2, %6\n" - "ldr q3, %7\n" - "ldr q12, %10\n" - "ldr q13, %11\n" - "eor v2.16b, v2.16b, v12.16b\n" - "eor v3.16b, v3.16b, v13.16b\n" - - /* split Pd */ - "ushr v4.16b, v0.16b, #4\n" - "ushr v5.16b, v1.16b, #4\n" - "and v0.16b, v0.16b, v28.16b\n" - "and v1.16b, v1.16b, v28.16b\n" - "and v4.16b, v4.16b, v28.16b\n" - "and v5.16b, v5.16b, v28.16b\n" - - /* split Qd */ - "ushr v6.16b, v2.16b, #4\n" - "ushr v7.16b, v3.16b, #4\n" - "and v2.16b, v2.16b, v28.16b\n" - "and v3.16b, v3.16b, v28.16b\n" - "and v6.16b, v6.16b, v28.16b\n" - "and v7.16b, v7.16b, v28.16b\n" - - /* C0 * Pd */ - "tbl v10.16b, {v20.16b}, v0.16b\n" - "tbl v11.16b, {v20.16b}, v1.16b\n" - "tbl v12.16b, {v21.16b}, v4.16b\n" - "tbl v13.16b, {v21.16b}, v5.16b\n" - "eor v10.16b, v10.16b, v12.16b\n" - "eor v11.16b, v11.16b, v13.16b\n" - - /* C1 * Qd */ - "tbl v12.16b, {v22.16b}, v2.16b\n" - "tbl v13.16b, {v22.16b}, v3.16b\n" - "tbl v14.16b, {v23.16b}, v6.16b\n" - "tbl v15.16b, {v23.16b}, v7.16b\n" - "eor v12.16b, v12.16b, v14.16b\n" - "eor v13.16b, v13.16b, v15.16b\n" - - /* Dy */ - "eor v10.16b, v10.16b, v12.16b\n" - "eor v11.16b, v11.16b, v13.16b\n" - - /* Dx = Pd ^ Dy */ - "eor v8.16b, v8.16b, v10.16b\n" - "eor v9.16b, v9.16b, v11.16b\n" - - "str q8, %0\n" - "str q9, %1\n" - "str q10, %2\n" - "str q11, %3\n" - : "=m" (pa[i]), "=m" (pa[i + 16]), - "=m" (qa[i]), "=m" (qa[i + 16]) - : "m" (p[i]), "m" (p[i + 16]), - "m" (q[i]), "m" (q[i + 16]), - "m" (pa[i]), "m" (pa[i + 16]), - "m" (qa[i]), "m" (qa[i + 16]) - ); - } - - raid_neon_end(); -} - void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); - - if (ip[0] == 0 && ip[1] == 1) { - raid_rec2of2_neon(id, ip, nd, size, vv); - return; - } - raid_recX_neon(2, id, ip, nd, size, vv); } diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 19b6137f..dd3dd9db 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -1160,7 +1160,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_NEON SPEED_START { - raid_gen_force(1, raid_gen1_neon); raid_rec1_neon(1, id, ip, nd, size, v); } SPEED_STOP @@ -1224,7 +1223,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512bw()) { SPEED_START { - raid_gen_force(1, raid_gen1_avx512bw); raid_rec1_avx512bw(1, id, ip, nd, size, v); } SPEED_STOP @@ -1233,7 +1231,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP @@ -1242,7 +1239,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v); } SPEED_STOP