mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Use fused 1-pass recovery for rec1of1 and rec2of2 in NEON, AVX512 and GFNI backends
This commit is contained in:
Andrea Mazzoleni
parent
c7b8068e36
commit
35abfd81e1
6 changed files with 45 additions and 558 deletions
|
|
@ -1265,8 +1265,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
/* raid_rec1of1() internally calls raid_gen() */
|
||||
raid_gen_force(1, raid_gen1_neon);
|
||||
raid_rec1_neon(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1332,8 +1330,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
/* raid_rec1of1() internally calls raid_gen() */
|
||||
raid_gen_force(1, raid_gen1_avx512bw);
|
||||
raid_rec1_avx512bw(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1342,8 +1338,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* raid_rec1of1() internally calls raid_gen() */
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1352,8 +1346,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* raid_rec1of1() internally calls raid_gen() */
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
|
|||
82
raid/NOTES
82
raid/NOTES
|
|
@ -54,17 +54,19 @@ Generation:
|
|||
|
||||
Recovery:
|
||||
|
||||
* The normal one-data-disk recovery with P available is already handled by
|
||||
raid_rec1of1(), which is essentially an XOR generation pass with pointer
|
||||
remapping.
|
||||
* The normal one-data-disk recovery with P available is handled directly by the
|
||||
fused 1-pass SIMD kernel on AVX512BW, GFNI, and AArch64 NEON (pure XOR
|
||||
streaming), or by raid_rec1of1() on scalar and older SIMD paths.
|
||||
|
||||
* SIMD recovery has parity-count-specific REC1..REC6 entry points. Fused paths
|
||||
compute exactly the selected syndromes in one scan of the surviving data and
|
||||
reconstruct the missing blocks without full-size temporary syndrome buffers.
|
||||
|
||||
* AVX512BW, GFNI, AArch64 NEON, and AArch32 NEON use fused syndrome generation
|
||||
for all non-P REC1 and REC2 cases and for REC3..REC6. SSSE3 and AVX2 use it
|
||||
for REC3..REC6, with additional fused cases in the extended AVX2 path.
|
||||
* AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON use fused syndrome
|
||||
generation for all REC1..REC6 cases, including P (rec1of1) and P+Q (rec2of2).
|
||||
AArch32 NEON uses fused syndrome generation for all non-P REC1 and REC2 cases
|
||||
and for REC3..REC6. SSSE3 and AVX2 use it for REC3..REC6, with additional
|
||||
fused cases in the extended AVX2 path.
|
||||
|
||||
* Scalar recovery and selected SSSE3/AVX2 REC1/REC2 paths continue to use
|
||||
raid_delta_gen().
|
||||
|
|
@ -82,7 +84,9 @@ Recovery:
|
|||
or broadcasting inside the data loop.
|
||||
|
||||
* Specialized P+Q rec2of2 recovery reduces 4 GF multiplications to 2 GF
|
||||
multiplications and 1 XOR, implemented across scalar int8 and all SIMD backends.
|
||||
multiplications and 1 XOR; AVX512BW, GFNI, and AArch64 NEON handle this
|
||||
directly inside the 1-pass fused kernel, while scalar int8, SSSE3, AVX2, and
|
||||
AArch32 NEON backends implement dedicated rec2of2 helpers.
|
||||
|
||||
* When P is among the selected parity blocks, optimized SIMD RECX paths
|
||||
reconstruct only nr - 1 missing blocks through the inverse matrix and derive
|
||||
|
|
@ -184,7 +188,10 @@ Typical recovery case
|
|||
* raid_rec() selects the first available parity. Therefore, for a single data
|
||||
failure with P available, recovery uses P.
|
||||
|
||||
* REC1 implementations special-case ip[0] == 0 and call raid_rec1of1().
|
||||
* REC1 implementations on AVX512BW, GFNI, and AArch64 NEON execute directly
|
||||
inside the fused streaming kernel (reducing to a single-pass pure XOR stream
|
||||
without memory swapping), while scalar and older SIMD paths call
|
||||
raid_rec1of1().
|
||||
|
||||
* raid_rec1of1() remaps pointers so that P is used as an XOR input and the
|
||||
failed data buffer is used as the GEN1 output, then calls
|
||||
|
|
@ -221,22 +228,12 @@ REC1OF1:
|
|||
|
||||
* The row uses ip[0] == 0.
|
||||
|
||||
* Every REC1 wrapper therefore enters raid_rec1of1() instead of executing its
|
||||
normal GF recovery body.
|
||||
* On AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON, REC1 executes directly
|
||||
inside the fused 1-pass kernel, streaming surviving data blocks and P in a
|
||||
single pass.
|
||||
|
||||
* The speed test explicitly selects the generation function used by the
|
||||
raid_rec1of1() call with raid_gen_force(1, ...).
|
||||
|
||||
* There is no GFNI GEN1 because PAR1 is only XOR. Therefore:
|
||||
|
||||
rec1of1 avx2 -> raid_gen1_avx2
|
||||
rec1of1 gfni -> raid_gen1_avx2
|
||||
|
||||
rec1of1 avx512 -> raid_gen1_avx512bw
|
||||
rec1of1 gfni512 -> raid_gen1_avx512bw
|
||||
|
||||
This is intentional. The GFNI recovery wrapper is selected, but the fast path
|
||||
naturally reduces to the best available XOR generator.
|
||||
* For scalar and older SIMD wrappers, REC1 enters raid_rec1of1() to recompute P
|
||||
using the generation backend.
|
||||
|
||||
REC1OF2 and REC1OF3:
|
||||
|
||||
|
|
@ -249,8 +246,12 @@ REC2OF2:
|
|||
|
||||
* The row uses ip = { 0, 1 }, meaning P and Q.
|
||||
|
||||
* Scalar int8 and all SIMD REC2 implementations have a specialized P+Q fast path
|
||||
and call raid_rec2of2_*().
|
||||
* On AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON, REC2 executes
|
||||
directly in the 1-pass fused kernel, which computes P with XOR, Q with GF
|
||||
multiplication, and derives the second missing block via XOR.
|
||||
|
||||
* Scalar int8, SSSE3, AVX2, and AArch32 NEON call their specialized
|
||||
raid_rec2of2_*() delta helpers.
|
||||
|
||||
* This makes the row a direct benchmark for the specialized P+Q recovery
|
||||
implementation.
|
||||
|
|
@ -310,8 +311,9 @@ raid_delta_gen() fusion
|
|||
according to register pressure; they do not allocate full-size syndrome
|
||||
buffers.
|
||||
|
||||
* The common rec1of1/P case remains the direct XOR generation path and does not
|
||||
enter fused GF recovery.
|
||||
* The common rec1of1/P case on AVX512BW and GFNI executes as pure XOR streaming
|
||||
within the fused kernel (no GF matrix multiplication), while on older backends
|
||||
it uses the raid_rec1of1() pointer remapping path.
|
||||
|
||||
* raid_delta_gen() remains in the scalar implementation and selected
|
||||
SSSE3/AVX2 REC1/REC2 paths.
|
||||
|
|
@ -642,8 +644,9 @@ Memory layout, displacement and stride noise
|
|||
REC1 optimizations
|
||||
------------------
|
||||
|
||||
The important normal REC1 case is already raid_rec1of1() with P and is close to
|
||||
the minimum possible memory traffic.
|
||||
The normal REC1 case with P is handled either directly in the fused 1-pass SIMD
|
||||
kernel (AVX512BW, GFNI, AArch64 NEON) as pure XOR streaming, or via
|
||||
raid_rec1of1() (scalar, legacy SIMD).
|
||||
|
||||
For generic REC1, used when P is unavailable:
|
||||
|
||||
|
|
@ -658,7 +661,7 @@ For generic REC1, used when P is unavailable:
|
|||
REC1 status:
|
||||
|
||||
* For the current large-block SnapRAID workload and implemented backends,
|
||||
no major rec1of1/P optimization is currently known.
|
||||
fused single-pass streaming achieves optimal memory saturation for rec1of1/P.
|
||||
|
||||
* Generic non-P REC1 is selected per parity count and backend, so fused and
|
||||
delta-based implementations are benchmarked independently.
|
||||
|
|
@ -671,8 +674,9 @@ REC2 optimizations
|
|||
|
||||
REC2 has both fused and delta-based implementations.
|
||||
|
||||
* AVX512BW, GFNI, AArch64 NEON, and AArch32 NEON use fused syndrome generation
|
||||
for every selected parity pair.
|
||||
* AVX512BW, GFNI, and AArch64 NEON use fused syndrome generation for every
|
||||
selected parity pair (including P+Q). AArch32 NEON uses fused generation for
|
||||
non-P pairs and REC3..REC6.
|
||||
|
||||
* AVX2 extended uses its delta kernel for P+Q and fused recovery for other
|
||||
parity pairs.
|
||||
|
|
@ -695,7 +699,9 @@ REC2 status:
|
|||
* Generic REC2 uses the fused path where available and retains local split reuse
|
||||
in the delta-based paths.
|
||||
|
||||
* Scalar int8 and all SIMD backends implement specialized rec2of2 P+Q recovery.
|
||||
* AVX512BW, GFNI, and AArch64 NEON handle P+Q recovery directly inside the
|
||||
1-pass fused kernel, while scalar int8, SSSE3, AVX2, and AArch32 NEON
|
||||
backends implement dedicated rec2of2 delta helpers.
|
||||
|
||||
|
||||
Specialized rec2of2 P+Q
|
||||
|
|
@ -710,14 +716,16 @@ can be reduced to:
|
|||
This needs only two GF multiplications for Dy, followed by one XOR for Dx,
|
||||
instead of the generic 2x2 reconstruction with four GF multiplications.
|
||||
|
||||
All backends implement this specialization:
|
||||
Backends implement this specialization as follows:
|
||||
|
||||
* AVX512BW, GFNI (AVX2 and AVX512), and AArch64 NEON integrate this derivation
|
||||
directly into the unified 1-pass fused kernel (P is accumulated via XOR, Q via
|
||||
GF multiplication, Dy is solved with inverse coefficients, and Dx derived by
|
||||
XOR with Pdelta).
|
||||
* Scalar int8 (raid_rec2of2_int8)
|
||||
* SSSE3 and SSSE3 ext (raid_rec2of2_ssse3, raid_rec2of2_ssse3ext)
|
||||
* AVX2 and AVX2 ext (raid_rec2of2_avx2, raid_rec2of2_avx2ext)
|
||||
* AVX512BW (raid_rec2of2_avx512bw)
|
||||
* AVX2 GFNI and AVX512 GFNI (RAID and AES variants)
|
||||
* AArch64 NEON and AArch32 NEON (raid_rec2of2_neon, raid_rec2of2_neon32)
|
||||
* AArch32 NEON (raid_rec2of2_neon32)
|
||||
|
||||
The rec2of2 benchmark row directly measures this specialized P+Q fast path.
|
||||
|
||||
|
|
|
|||
|
|
@ -911,102 +911,12 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
|
|||
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512bw(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512BW implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both multiplication-table pairs resident
|
||||
* in ZMM registers (zmm16..zmm19).
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512bw(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
/* get multiplication coefficients */
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* keep both multiplication-table pairs resident */
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[C[0]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[C[0]][1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[C[1]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[C[1]][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* Pd */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxord %0,%%zmm0,%%zmm0" : : "m" (pa[i]));
|
||||
|
||||
/* Qd */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxord %0,%%zmm1,%%zmm1" : : "m" (qa[i]));
|
||||
|
||||
/* split Pd and Qd */
|
||||
asm volatile ("vpsrlw $4,%zmm0,%zmm2");
|
||||
asm volatile ("vpsrlw $4,%zmm1,%zmm3");
|
||||
asm volatile ("vpandq %zmm31,%zmm0,%zmm4");
|
||||
asm volatile ("vpandq %zmm31,%zmm1,%zmm5");
|
||||
asm volatile ("vpandq %zmm31,%zmm2,%zmm2");
|
||||
asm volatile ("vpandq %zmm31,%zmm3,%zmm3");
|
||||
|
||||
/* C0 * Pd */
|
||||
asm volatile ("vpshufb %zmm4,%zmm16,%zmm6");
|
||||
asm volatile ("vpshufb %zmm2,%zmm17,%zmm7");
|
||||
asm volatile ("vpxord %zmm7,%zmm6,%zmm6");
|
||||
|
||||
/* C1 * Qd */
|
||||
asm volatile ("vpshufb %zmm5,%zmm18,%zmm7");
|
||||
asm volatile ("vpxord %zmm7,%zmm6,%zmm6");
|
||||
asm volatile ("vpshufb %zmm3,%zmm19,%zmm7");
|
||||
asm volatile ("vpxord %zmm7,%zmm6,%zmm6");
|
||||
|
||||
/* Dy = zmm6, Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxord %zmm6,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
/* if recovering with P,Q use the specialized function */
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx512bw(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512bw(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
286
raid/gfni.c
286
raid/gfni.c
|
|
@ -2203,94 +2203,12 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv)
|
|||
void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 RAID GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both affine matrices resident in YMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2gfni_raid(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
const uint8_t *T[2];
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
T[0] = raid_gfaffine_raid[C[0]];
|
||||
T[1] = raid_gfaffine_raid[C[1]];
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (T[0][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%ymm15" : : "m" (T[1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* Pd, two 32-byte lanes */
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32]));
|
||||
|
||||
/* Qd, two 32-byte lanes */
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32]));
|
||||
|
||||
/* Dy = C0 * Pd ^ C1 * Qd */
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm5");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm2,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm3,%ymm7");
|
||||
asm volatile ("vpxor %ymm6,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm7,%ymm5,%ymm5");
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i]));
|
||||
asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx2gfni_raid(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -2321,80 +2239,12 @@ void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void
|
|||
void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512 RAID GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both affine matrices resident in ZMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512gfni_raid(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
const uint8_t *T[2];
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
T[0] = raid_gfaffine_raid[C[0]];
|
||||
T[1] = raid_gfaffine_raid[C[1]];
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (T[0][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (T[1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i]));
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i]));
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm14,%zmm0,%zmm2");
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm15,%zmm1,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm2,%zmm2");
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx512gfni_raid(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -2425,85 +2275,12 @@ void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo
|
|||
void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 AES GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both byte multipliers resident in YMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2gfni_aes(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (C[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (C[1]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32]));
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm5");
|
||||
asm volatile ("vgf2p8mulb %ymm15,%ymm2,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm15,%ymm3,%ymm7");
|
||||
asm volatile ("vpxor %ymm6,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm7,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i]));
|
||||
asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx2gfni_aes(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -2534,75 +2311,12 @@ void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void
|
|||
void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512 AES GFNI implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps both byte multipliers resident in ZMM registers.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512gfni_aes(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (C[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (C[1]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i]));
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i]));
|
||||
|
||||
asm volatile ("vgf2p8mulb %zmm14,%zmm0,%zmm2");
|
||||
asm volatile ("vgf2p8mulb %zmm15,%zmm1,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm2,%zmm2");
|
||||
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx512gfni_aes(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
133
raid/neon.c
133
raid/neon.c
|
|
@ -1036,145 +1036,12 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv)
|
|||
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_neon(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AArch64 NEON implementation.
|
||||
*
|
||||
* Uses raid_delta_gen() and keeps all four multiplication tables resident
|
||||
* in NEON registers (q20..q23) with a 32-byte two-lane unroll.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_neon(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
asm volatile (
|
||||
"ldr q28, %0\n"
|
||||
"ldr q20, %1\n"
|
||||
"ldr q21, %2\n"
|
||||
"ldr q22, %3\n"
|
||||
"ldr q23, %4\n"
|
||||
:
|
||||
: "m" (gfconst16.low4[0]),
|
||||
"m" (raid_gfmulpshufb[C[0]][0][0]),
|
||||
"m" (raid_gfmulpshufb[C[0]][1][0]),
|
||||
"m" (raid_gfmulpshufb[C[1]][0][0]),
|
||||
"m" (raid_gfmulpshufb[C[1]][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
asm volatile (
|
||||
/* Pd, two lanes */
|
||||
"ldr q0, %4\n"
|
||||
"ldr q1, %5\n"
|
||||
"ldr q12, %8\n"
|
||||
"ldr q13, %9\n"
|
||||
"eor v0.16b, v0.16b, v12.16b\n"
|
||||
"eor v1.16b, v1.16b, v13.16b\n"
|
||||
|
||||
/* preserve Pd for Dx */
|
||||
"mov v8.16b, v0.16b\n"
|
||||
"mov v9.16b, v1.16b\n"
|
||||
|
||||
/* Qd, two lanes */
|
||||
"ldr q2, %6\n"
|
||||
"ldr q3, %7\n"
|
||||
"ldr q12, %10\n"
|
||||
"ldr q13, %11\n"
|
||||
"eor v2.16b, v2.16b, v12.16b\n"
|
||||
"eor v3.16b, v3.16b, v13.16b\n"
|
||||
|
||||
/* split Pd */
|
||||
"ushr v4.16b, v0.16b, #4\n"
|
||||
"ushr v5.16b, v1.16b, #4\n"
|
||||
"and v0.16b, v0.16b, v28.16b\n"
|
||||
"and v1.16b, v1.16b, v28.16b\n"
|
||||
"and v4.16b, v4.16b, v28.16b\n"
|
||||
"and v5.16b, v5.16b, v28.16b\n"
|
||||
|
||||
/* split Qd */
|
||||
"ushr v6.16b, v2.16b, #4\n"
|
||||
"ushr v7.16b, v3.16b, #4\n"
|
||||
"and v2.16b, v2.16b, v28.16b\n"
|
||||
"and v3.16b, v3.16b, v28.16b\n"
|
||||
"and v6.16b, v6.16b, v28.16b\n"
|
||||
"and v7.16b, v7.16b, v28.16b\n"
|
||||
|
||||
/* C0 * Pd */
|
||||
"tbl v10.16b, {v20.16b}, v0.16b\n"
|
||||
"tbl v11.16b, {v20.16b}, v1.16b\n"
|
||||
"tbl v12.16b, {v21.16b}, v4.16b\n"
|
||||
"tbl v13.16b, {v21.16b}, v5.16b\n"
|
||||
"eor v10.16b, v10.16b, v12.16b\n"
|
||||
"eor v11.16b, v11.16b, v13.16b\n"
|
||||
|
||||
/* C1 * Qd */
|
||||
"tbl v12.16b, {v22.16b}, v2.16b\n"
|
||||
"tbl v13.16b, {v22.16b}, v3.16b\n"
|
||||
"tbl v14.16b, {v23.16b}, v6.16b\n"
|
||||
"tbl v15.16b, {v23.16b}, v7.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
|
||||
/* Dy */
|
||||
"eor v10.16b, v10.16b, v12.16b\n"
|
||||
"eor v11.16b, v11.16b, v13.16b\n"
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
"eor v8.16b, v8.16b, v10.16b\n"
|
||||
"eor v9.16b, v9.16b, v11.16b\n"
|
||||
|
||||
"str q8, %0\n"
|
||||
"str q9, %1\n"
|
||||
"str q10, %2\n"
|
||||
"str q11, %3\n"
|
||||
: "=m" (pa[i]), "=m" (pa[i + 16]),
|
||||
"=m" (qa[i]), "=m" (qa[i + 16])
|
||||
: "m" (p[i]), "m" (p[i + 16]),
|
||||
"m" (q[i]), "m" (q[i + 16]),
|
||||
"m" (pa[i]), "m" (pa[i + 16]),
|
||||
"m" (qa[i]), "m" (qa[i + 16])
|
||||
);
|
||||
}
|
||||
|
||||
raid_neon_end();
|
||||
}
|
||||
|
||||
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_neon(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_neon(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1160,7 +1160,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_neon);
|
||||
raid_rec1_neon(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1224,7 +1223,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw);
|
||||
raid_rec1_avx512bw(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1233,7 +1231,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
raid_rec1_avx2gfni_raid(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1242,7 +1239,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
raid_rec1_avx512gfni_raid(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue