From 46ebce4e3ca47c7d03024ff4626d81cf38bc9637 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Mon, 24 Aug 2026 14:46:18 +0200 Subject: [PATCH] Add specialized P+Q REC2OF2 recovery to all SIMD backends --- TODO | 5 - raid/avx2.c | 159 +++++++++++++++++++++++++++++++- raid/avx512.c | 80 ++++++++++++++++ raid/gfni.c | 250 ++++++++++++++++++++++++++++++++++++++++++++++++++ raid/neon.c | 123 +++++++++++++++++++++++++ raid/neon32.c | 105 +++++++++++++++++++++ raid/ssse3.c | 172 ++++++++++++++++++++++++++++++++++ 7 files changed, 885 insertions(+), 9 deletions(-) diff --git a/TODO b/TODO index a58b42eb..5c0e630f 100644 --- a/TODO +++ b/TODO @@ -24,11 +24,6 @@ All done ? * Enable MuseAir and disable random seed -* Optimize rec2of2 SIMD recovery for P+Q: add specialized AVX2, AVX512 and GFNI - fast paths for recovering two failed data disks when ip[0] == 0 && ip[1] == 1. - Exploit the RAID6 closed-form solution to reduce recovery from 4 generic GF - multiplications to 2 GF multiplications plus XOR (Dy = C0*Pd ^ C1*Qd, Dx = Pd ^ Dy). - * Check Reliability Test. We can derive Failure probability from MBTF https://github.com/linuxhw/SMART https://github.com/linuxhw/EnterpriseDrive diff --git a/raid/avx2.c b/raid/avx2.c index dfe230ac..be9364cd 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -1039,6 +1039,79 @@ static __always_inline void raid_rec2_avx2_delta(int nr, int *id, int *ip, int n raid_avx_end(); } +/* + * Recover failure of two data blocks using P and Q AVX2 implementation. + */ +static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + /* get multiplication coefficients */ + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + /* compute delta parity */ + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 32) { + /* Pd */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i])); + + /* Qd */ + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (q[i])); + asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (qa[i])); + + /* split Pd and Qd */ + asm volatile ("vpsrlw $4,%ymm0,%ymm2"); + asm volatile ("vpsrlw $4,%ymm1,%ymm3"); + asm volatile ("vpand %ymm7,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm1,%ymm5"); + asm volatile ("vpand %ymm7,%ymm2,%ymm2"); + asm volatile ("vpand %ymm7,%ymm3,%ymm3"); + + /* C0 * Pd */ + asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (raid_gfmulpshufb[C[0]][0][0])); + asm volatile ("vpshufb %ymm4,%ymm6,%ymm6"); + + asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[C[0]][1][0])); + asm volatile ("vpshufb %ymm2,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm4,%ymm6,%ymm6"); + + /* C1 * Qd */ + asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[C[1]][0][0])); + asm volatile ("vpshufb %ymm5,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm4,%ymm6,%ymm6"); + + asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[C[1]][1][0])); + asm volatile ("vpshufb %ymm3,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm4,%ymm6,%ymm6"); + + /* Dy = ymm6, Dx = Pd ^ Dy */ + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (qa[i])); + } + + raid_avx_end(); +} + /* * RAID recovering AVX2 implementation optimized for up to four failures. * @@ -1440,7 +1513,7 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz /* * RAID recovering for two disks AVX2 extended implementation */ -static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv) +static __always_inline void raid_rec2_avx2ext_delta(int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; const int N = 2; @@ -1451,8 +1524,6 @@ static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, in size_t i; int j, k; - (void)nr; /* unused, it's always 2 */ - /* setup the coefficients matrix */ for (j = 0; j < N; ++j) for (k = 0; k < N; ++k) @@ -1537,6 +1608,79 @@ static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, in raid_avx_end(); } +/* + * RAID recovering for two disks with P and Q AVX2 implementation + */ +static __always_inline void raid_rec2of2_avx2ext(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N]; + uint8_t *pa[N]; + uint8_t C[N]; + size_t i; + + /* get multiplication coefficients */ + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + /* compute delta parity */ + raid_delta_gen(N, id, ip, nd, size, vv); + + p[0] = v[nd]; + p[1] = v[nd + 1]; + pa[0] = v[id[0]]; + pa[1] = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); + + /* the multiplication tables are constant for the whole recovery */ + asm volatile ("vbroadcasti128 %0,%%ymm8" : : "m" (raid_gfmulpshufb[C[0]][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm9" : : "m" (raid_gfmulpshufb[C[0]][1][0])); + asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfmulpshufb[C[1]][0][0])); + asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfmulpshufb[C[1]][1][0])); + + for (i = 0; i < size; i += 32) { + /* d0 = p[0] ^ pa[0] */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[0][i])); + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + + /* d1 = p[1] ^ pa[1] */ + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[1][i])); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + /* C[0] * d0 */ + asm volatile ("vpsrlw $4,%ymm0,%ymm5"); + asm volatile ("vpand %ymm7,%ymm0,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm8,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm9,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm2,%ymm6"); + + /* C[1] * d1 */ + asm volatile ("vpsrlw $4,%ymm1,%ymm5"); + asm volatile ("vpand %ymm7,%ymm1,%ymm4"); + asm volatile ("vpand %ymm7,%ymm5,%ymm5"); + asm volatile ("vpshufb %ymm4,%ymm10,%ymm2"); + asm volatile ("vpshufb %ymm5,%ymm11,%ymm3"); + asm volatile ("vpxor %ymm2,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm3,%ymm6,%ymm6"); + + /* pa[1] = C[0] * d0 ^ C[1] * d1 */ + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); + + /* pa[0] = d0 ^ pa[1] */ + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[0][i])); + } + + raid_avx_end(); +} + /* * RAID recovering AVX2 implementation optimized for up to three failures. * @@ -2224,6 +2368,13 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + /* if recovering with P,Q use the specialized function */ + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_avx2(id, ip, nd, size, vv); + return; + } + raid_rec2_avx2_delta(2, id, ip, nd, size, vv); } @@ -2277,7 +2428,7 @@ void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) /* if recovering with P,Q uses the delta function */ if (ip[0] == 0 && ip[1] == 1) { - raid_rec2_avx2ext_delta(2, id, ip, nd, size, vv); + raid_rec2of2_avx2ext(id, ip, nd, size, vv); return; } diff --git a/raid/avx512.c b/raid/avx512.c index 621a34bb..56d2b86b 100644 --- a/raid/avx512.c +++ b/raid/avx512.c @@ -887,9 +887,89 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv raid_recX_avx512bw(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AVX512BW implementation. + */ +static __always_inline void raid_rec2of2_avx512bw(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + /* get multiplication coefficients */ + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + /* compute delta parity */ + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0])); + + /* keep both multiplication-table pairs resident */ + asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[C[0]][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[C[0]][1][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[C[1]][0][0])); + asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[C[1]][1][0])); + + for (i = 0; i < size; i += 64) { + /* Pd */ + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vpxord %0,%%zmm0,%%zmm0" : : "m" (pa[i])); + + /* Qd */ + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i])); + asm volatile ("vpxord %0,%%zmm1,%%zmm1" : : "m" (qa[i])); + + /* split Pd and Qd */ + asm volatile ("vpsrlw $4,%zmm0,%zmm2"); + asm volatile ("vpsrlw $4,%zmm1,%zmm3"); + asm volatile ("vpandq %zmm31,%zmm0,%zmm4"); + asm volatile ("vpandq %zmm31,%zmm1,%zmm5"); + asm volatile ("vpandq %zmm31,%zmm2,%zmm2"); + asm volatile ("vpandq %zmm31,%zmm3,%zmm3"); + + /* C0 * Pd */ + asm volatile ("vpshufb %zmm4,%zmm16,%zmm6"); + asm volatile ("vpshufb %zmm2,%zmm17,%zmm7"); + asm volatile ("vpxord %zmm7,%zmm6,%zmm6"); + + /* C1 * Qd */ + asm volatile ("vpshufb %zmm5,%zmm18,%zmm7"); + asm volatile ("vpxord %zmm7,%zmm6,%zmm6"); + asm volatile ("vpshufb %zmm3,%zmm19,%zmm7"); + asm volatile ("vpxord %zmm7,%zmm6,%zmm6"); + + /* Dy = zmm6, Dx = Pd ^ Dy */ + asm volatile ("vpxord %zmm6,%zmm0,%zmm0"); + + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (qa[i])); + } + + raid_avx_end(); +} + void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + /* if recovering with P,Q use the specialized function */ + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_avx512bw(id, ip, nd, size, vv); + return; + } + raid_recX_avx512bw(2, id, ip, nd, size, vv); } diff --git a/raid/gfni.c b/raid/gfni.c index 0d91dca8..f4717630 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -1264,9 +1264,82 @@ void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AVX2 RAID GFNI implementation. + */ +static __always_inline void raid_rec2of2_avx2gfni_raid(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + const uint8_t *T[2]; + uint8_t C[2]; + size_t i; + + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + T[0] = raid_gfaffine_raid[C[0]]; + T[1] = raid_gfaffine_raid[C[1]]; + + /* compute delta parity */ + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (T[0][0])); + asm volatile ("vpbroadcastq %0,%%ymm15" : : "m" (T[1][0])); + + for (i = 0; i < size; i += 64) { + /* Pd, two 32-byte lanes */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i])); + asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32])); + + /* Qd, two 32-byte lanes */ + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32])); + asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i])); + asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32])); + + /* Dy = C0 * Pd ^ C1 * Qd */ + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm5"); + asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm2,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm3,%ymm7"); + asm volatile ("vpxor %ymm6,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm7,%ymm5,%ymm5"); + + /* Dx = Pd ^ Dy */ + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32])); + } + + raid_avx_end(); +} + void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_avx2gfni_raid(id, ip, nd, size, vv); + return; + } + raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv); } @@ -1307,9 +1380,68 @@ void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AVX512 RAID GFNI implementation. + */ +static __always_inline void raid_rec2of2_avx512gfni_raid(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + const uint8_t *T[2]; + uint8_t C[2]; + size_t i; + + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + T[0] = raid_gfaffine_raid[C[0]]; + T[1] = raid_gfaffine_raid[C[1]]; + + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (T[0][0])); + asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (T[1][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i])); + + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i])); + asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i])); + + asm volatile ("vgf2p8affineqb $0,%zmm14,%zmm0,%zmm2"); + asm volatile ("vgf2p8affineqb $0,%zmm15,%zmm1,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm2,%zmm2"); + + /* Dx = Pd ^ Dy */ + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i])); + } + + raid_avx_end(); +} + void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_avx512gfni_raid(id, ip, nd, size, vv); + return; + } + raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv); } @@ -1350,9 +1482,73 @@ void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AVX2 AES GFNI implementation. + */ +static __always_inline void raid_rec2of2_avx2gfni_aes(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (C[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (C[1])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i])); + asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32])); + asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i])); + asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32])); + + asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm4"); + asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm5"); + asm volatile ("vgf2p8mulb %ymm15,%ymm2,%ymm6"); + asm volatile ("vgf2p8mulb %ymm15,%ymm3,%ymm7"); + asm volatile ("vpxor %ymm6,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm7,%ymm5,%ymm5"); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32])); + } + + raid_avx_end(); +} + void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_avx2gfni_aes(id, ip, nd, size, vv); + return; + } + raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv); } @@ -1393,9 +1589,63 @@ void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AVX512 AES GFNI implementation. + */ +static __always_inline void raid_rec2of2_avx512gfni_aes(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (C[0])); + asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (C[1])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i])); + + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i])); + asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i])); + + asm volatile ("vgf2p8mulb %zmm14,%zmm0,%zmm2"); + asm volatile ("vgf2p8mulb %zmm15,%zmm1,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm2,%zmm2"); + + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i])); + } + + raid_avx_end(); +} + void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_avx512gfni_aes(id, ip, nd, size, vv); + return; + } + raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv); } diff --git a/raid/neon.c b/raid/neon.c index 71ed4ecd..419392d7 100644 --- a/raid/neon.c +++ b/raid/neon.c @@ -1045,9 +1045,132 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_neon(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AArch64 NEON implementation. + */ +static __always_inline void raid_rec2of2_neon(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_neon_begin(); + + asm volatile ( + "ldr q28, %0\n" + "ldr q20, %1\n" + "ldr q21, %2\n" + "ldr q22, %3\n" + "ldr q23, %4\n" + : + : "m" (gfconst16.low4[0]), + "m" (raid_gfmulpshufb[C[0]][0][0]), + "m" (raid_gfmulpshufb[C[0]][1][0]), + "m" (raid_gfmulpshufb[C[1]][0][0]), + "m" (raid_gfmulpshufb[C[1]][1][0]) + ); + + for (i = 0; i < size; i += 32) { + asm volatile ( + /* Pd, two lanes */ + "ldr q0, %4\n" + "ldr q1, %5\n" + "ldr q12, %8\n" + "ldr q13, %9\n" + "eor v0.16b, v0.16b, v12.16b\n" + "eor v1.16b, v1.16b, v13.16b\n" + + /* preserve Pd for Dx */ + "mov v8.16b, v0.16b\n" + "mov v9.16b, v1.16b\n" + + /* Qd, two lanes */ + "ldr q2, %6\n" + "ldr q3, %7\n" + "ldr q12, %10\n" + "ldr q13, %11\n" + "eor v2.16b, v2.16b, v12.16b\n" + "eor v3.16b, v3.16b, v13.16b\n" + + /* split Pd */ + "ushr v4.16b, v0.16b, #4\n" + "ushr v5.16b, v1.16b, #4\n" + "and v0.16b, v0.16b, v28.16b\n" + "and v1.16b, v1.16b, v28.16b\n" + "and v4.16b, v4.16b, v28.16b\n" + "and v5.16b, v5.16b, v28.16b\n" + + /* split Qd */ + "ushr v6.16b, v2.16b, #4\n" + "ushr v7.16b, v3.16b, #4\n" + "and v2.16b, v2.16b, v28.16b\n" + "and v3.16b, v3.16b, v28.16b\n" + "and v6.16b, v6.16b, v28.16b\n" + "and v7.16b, v7.16b, v28.16b\n" + + /* C0 * Pd */ + "tbl v10.16b, {v20.16b}, v0.16b\n" + "tbl v11.16b, {v20.16b}, v1.16b\n" + "tbl v12.16b, {v21.16b}, v4.16b\n" + "tbl v13.16b, {v21.16b}, v5.16b\n" + "eor v10.16b, v10.16b, v12.16b\n" + "eor v11.16b, v11.16b, v13.16b\n" + + /* C1 * Qd */ + "tbl v12.16b, {v22.16b}, v2.16b\n" + "tbl v13.16b, {v22.16b}, v3.16b\n" + "tbl v14.16b, {v23.16b}, v6.16b\n" + "tbl v15.16b, {v23.16b}, v7.16b\n" + "eor v12.16b, v12.16b, v14.16b\n" + "eor v13.16b, v13.16b, v15.16b\n" + + /* Dy */ + "eor v10.16b, v10.16b, v12.16b\n" + "eor v11.16b, v11.16b, v13.16b\n" + + /* Dx = Pd ^ Dy */ + "eor v8.16b, v8.16b, v10.16b\n" + "eor v9.16b, v9.16b, v11.16b\n" + + "str q8, %0\n" + "str q9, %1\n" + "str q10, %2\n" + "str q11, %3\n" + : "=m" (pa[i]), "=m" (pa[i + 16]), + "=m" (qa[i]), "=m" (qa[i + 16]) + : "m" (p[i]), "m" (p[i + 16]), + "m" (q[i]), "m" (q[i + 16]), + "m" (pa[i]), "m" (pa[i + 16]), + "m" (qa[i]), "m" (qa[i + 16]) + ); + } + + raid_neon_end(); +} + void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_neon(id, ip, nd, size, vv); + return; + } + raid_recX_neon(2, id, ip, nd, size, vv); } diff --git a/raid/neon32.c b/raid/neon32.c index 04c839b1..d551917e 100644 --- a/raid/neon32.c +++ b/raid/neon32.c @@ -1225,9 +1225,114 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_recX_neon32_123(1, id, ip, nd, size, vv); } +/* + * Recover failure of two data blocks using P and Q AArch32 NEON implementation. + */ +static __always_inline void raid_rec2of2_neon32(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_neon32_begin(); + + /* q15 = nibble mask, q9-q12 = C0/C1 low/high tables */ + asm volatile ( + "vld1.8 {q15}, %0\n" + "vld1.8 {q9}, %1\n" + "vld1.8 {q10}, %2\n" + "vld1.8 {q11}, %3\n" + "vld1.8 {q12}, %4\n" + : + : "Q" (gfconst16.low4[0]), + "Q" (raid_gfmulpshufb[C[0]][0][0]), + "Q" (raid_gfmulpshufb[C[0]][1][0]), + "Q" (raid_gfmulpshufb[C[1]][0][0]), + "Q" (raid_gfmulpshufb[C[1]][1][0]) + ); + + for (i = 0; i < size; i += 16) { + asm volatile ( + /* Pd */ + "vld1.8 {q0}, %2\n" + "vld1.8 {q13}, %4\n" + "veor q0, q0, q13\n" + + /* Qd */ + "vld1.8 {q1}, %3\n" + "vld1.8 {q13}, %5\n" + "veor q1, q1, q13\n" + + /* split Pd */ + "vmov q2, q0\n" + "vshr.u8 q3, q0, #4\n" + "vand q2, q2, q15\n" + "vand q3, q3, q15\n" + + /* split Qd */ + "vmov q4, q1\n" + "vshr.u8 q5, q1, #4\n" + "vand q4, q4, q15\n" + "vand q5, q5, q15\n" + + /* C0 low * Pd low -> q6 */ + "vtbl.8 d12, {d18-d19}, d4\n" + "vtbl.8 d13, {d18-d19}, d5\n" + + /* C0 high * Pd high -> q7 */ + "vtbl.8 d14, {d20-d21}, d6\n" + "vtbl.8 d15, {d20-d21}, d7\n" + "veor q6, q6, q7\n" + + /* C1 low * Qd low -> q7 */ + "vtbl.8 d14, {d22-d23}, d8\n" + "vtbl.8 d15, {d22-d23}, d9\n" + + /* C1 high * Qd high -> q8 */ + "vtbl.8 d16, {d24-d25}, d10\n" + "vtbl.8 d17, {d24-d25}, d11\n" + "veor q7, q7, q8\n" + + /* Dy */ + "veor q6, q6, q7\n" + + /* Dx = Pd ^ Dy */ + "veor q0, q0, q6\n" + + "vst1.8 {q0}, %0\n" + "vst1.8 {q6}, %1\n" + : "=Q" (pa[i]), "=Q" (qa[i]) + : "Q" (p[i]), "Q" (q[i]), + "Q" (pa[i]), "Q" (qa[i]) + ); + } + + raid_neon32_end(); +} + void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_neon32(id, ip, nd, size, vv); + return; + } + raid_recX_neon32_123(2, id, ip, nd, size, vv); } diff --git a/raid/ssse3.c b/raid/ssse3.c index 922d9744..a233b55d 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1304,6 +1304,83 @@ static __always_inline void raid_rec2_ssse3_delta(int nr, int *id, int *ip, int raid_sse_end(); } +/* + * Recover failure of two data blocks using P and Q SSSE3 implementation. + */ +static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + /* get multiplication coefficients */ + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + /* compute delta parity */ + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + for (i = 0; i < size; i += 16) { + /* Pd */ + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); + asm volatile ("pxor %0,%%xmm0" : : "m" (pa[i])); + + /* Qd */ + asm volatile ("movdqa %0,%%xmm1" : : "m" (q[i])); + asm volatile ("pxor %0,%%xmm1" : : "m" (qa[i])); + + /* split Pd and Qd into low/high nibbles */ + asm volatile ("movdqa %xmm0,%xmm4"); + asm volatile ("movdqa %xmm1,%xmm5"); + asm volatile ("movdqa %xmm0,%xmm2"); + asm volatile ("movdqa %xmm1,%xmm3"); + asm volatile ("psrlw $4,%xmm2"); + asm volatile ("psrlw $4,%xmm3"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm2"); + asm volatile ("pand %xmm7,%xmm3"); + + /* C0 * Pd */ + asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfmulpshufb[C[0]][0][0])); + asm volatile ("pshufb %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[C[0]][1][0])); + asm volatile ("pshufb %xmm2,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + /* C1 * Qd */ + asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[C[1]][0][0])); + asm volatile ("pshufb %xmm5,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[C[1]][1][0])); + asm volatile ("pshufb %xmm3,%xmm4"); + asm volatile ("pxor %xmm4,%xmm6"); + + /* xmm6 = Dy, xmm0 = Pd, so Dx = Pd ^ Dy */ + asm volatile ("pxor %xmm6,%xmm0"); + + asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (qa[i])); + } + + raid_sse_end(); +} + static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip, int nd, size_t size, void **vv) { @@ -1858,6 +1935,87 @@ static __always_inline void raid_rec2_ssse3ext_delta(int nr, int *id, int *ip, i raid_sse_end(); } +/* + * Recover failure of two data blocks using P and Q SSSE3 extended implementation. + */ +static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *pa; + uint8_t *q; + uint8_t *qa; + uint8_t C[2]; + size_t i; + + /* get multiplication coefficients */ + C[0] = inv(powgen(id[1] - id[0]) ^ 1); + C[1] = inv(powgen(id[0]) ^ powgen(id[1])); + + /* compute delta parity */ + raid_delta_gen(2, id, ip, nd, size, vv); + + p = v[nd]; + q = v[nd + 1]; + pa = v[id[0]]; + qa = v[id[1]]; + + raid_sse_begin(); + + asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0])); + + /* keep the four multiplication tables resident */ + asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[C[0]][0][0])); + asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[C[0]][1][0])); + asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[C[1]][0][0])); + asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[C[1]][1][0])); + + for (i = 0; i < size; i += 16) { + /* Pd */ + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); + asm volatile ("pxor %0,%%xmm0" : : "m" (pa[i])); + + /* Qd */ + asm volatile ("movdqa %0,%%xmm1" : : "m" (q[i])); + asm volatile ("pxor %0,%%xmm1" : : "m" (qa[i])); + + /* split Pd and Qd */ + asm volatile ("movdqa %xmm0,%xmm4"); + asm volatile ("movdqa %xmm1,%xmm5"); + asm volatile ("movdqa %xmm0,%xmm2"); + asm volatile ("movdqa %xmm1,%xmm3"); + asm volatile ("psrlw $4,%xmm2"); + asm volatile ("psrlw $4,%xmm3"); + asm volatile ("pand %xmm7,%xmm4"); + asm volatile ("pand %xmm7,%xmm5"); + asm volatile ("pand %xmm7,%xmm2"); + asm volatile ("pand %xmm7,%xmm3"); + + /* C0 * Pd */ + asm volatile ("movdqa %xmm8,%xmm6"); + asm volatile ("pshufb %xmm4,%xmm6"); + asm volatile ("movdqa %xmm9,%xmm12"); + asm volatile ("pshufb %xmm2,%xmm12"); + asm volatile ("pxor %xmm12,%xmm6"); + + /* C1 * Qd */ + asm volatile ("movdqa %xmm10,%xmm12"); + asm volatile ("pshufb %xmm5,%xmm12"); + asm volatile ("pxor %xmm12,%xmm6"); + asm volatile ("movdqa %xmm11,%xmm12"); + asm volatile ("pshufb %xmm3,%xmm12"); + asm volatile ("pxor %xmm12,%xmm6"); + + /* Dy = xmm6, Dx = Pd ^ Dy */ + asm volatile ("pxor %xmm6,%xmm0"); + + asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i])); + asm volatile ("movdqa %%xmm6,%0" : "=m" (qa[i])); + } + + raid_sse_end(); +} + static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip, int nd, size_t size, void **vv) { @@ -2459,6 +2617,13 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + /* if recovering with P,Q use the specialized function */ + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_ssse3(id, ip, nd, size, vv); + return; + } + raid_rec2_ssse3_delta(2, id, ip, nd, size, vv); } @@ -2503,6 +2668,13 @@ void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 2); + + /* if recovering with P,Q use the specialized function */ + if (ip[0] == 0 && ip[1] == 1) { + raid_rec2of2_ssse3ext(id, ip, nd, size, vv); + return; + } + raid_rec2_ssse3ext_delta(2, id, ip, nd, size, vv); }