mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 02:49:02 -04:00
Add XOR-only fast path for AVX2 GFNI rec1of1
This commit is contained in:
Andrea Mazzoleni
parent
82089165b5
commit
e39c21a4c5
1 changed files with 114 additions and 95 deletions
209
raid/gfni.c
209
raid/gfni.c
|
|
@ -1352,8 +1352,8 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
|
||||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||||
* parity blocks, and generation of unused parity rows.
|
* parity blocks, and generation of unused parity rows.
|
||||||
*
|
*
|
||||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
* If recovering a single failure from P, the P delta directly contains
|
||||||
* inverse matrix and derive the last missing block from the P delta by XOR.
|
* the missing block and no inverse-matrix multiplication is required.
|
||||||
*/
|
*/
|
||||||
static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
||||||
{
|
{
|
||||||
|
|
@ -1495,55 +1495,65 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (j = 0; j < nr; ++j) {
|
/*
|
||||||
const uint8_t **t = R[j];
|
* With a single failure recovered from P, ymm0/ymm1
|
||||||
|
* already contain the missing data. Multiplication by
|
||||||
|
* the inverse coefficient 1 would be redundant.
|
||||||
|
*/
|
||||||
|
if (nr == 1 && has_p) {
|
||||||
|
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[0][i]));
|
||||||
|
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[0][i + 32]));
|
||||||
|
} else {
|
||||||
|
for (j = 0; j < nr; ++j) {
|
||||||
|
const uint8_t **t = R[j];
|
||||||
|
|
||||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0]));
|
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0]));
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm12");
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm12");
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm13");
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm13");
|
||||||
|
|
||||||
if (nr >= 2) {
|
if (nr >= 2) {
|
||||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0]));
|
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0]));
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm2,%ymm15");
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm2,%ymm15");
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm3,%ymm15");
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm3,%ymm15");
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 3) {
|
||||||
|
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0]));
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm4,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm5,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 4) {
|
||||||
|
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0]));
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm6,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm7,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 5) {
|
||||||
|
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0]));
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm8,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm9,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 6) {
|
||||||
|
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0]));
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm10,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm11,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||||
|
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (nr >= 3) {
|
|
||||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0]));
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm4,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm5,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nr >= 4) {
|
|
||||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0]));
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm6,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm7,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nr >= 5) {
|
|
||||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0]));
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm8,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm9,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nr >= 6) {
|
|
||||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0]));
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm10,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm11,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
|
||||||
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -1735,8 +1745,8 @@ static __always_inline void raid_recX_avx512gfni_raid(int nr, int has_p, int *id
|
||||||
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
* This avoids raid_delta_gen(), temporary syndrome buffers, recomputation of
|
||||||
* parity blocks, and generation of unused parity rows.
|
* parity blocks, and generation of unused parity rows.
|
||||||
*
|
*
|
||||||
* If P is available, reconstruct only nr - 1 missing blocks through the
|
* If recovering a single failure from P, the P delta directly contains
|
||||||
* inverse matrix and derive the last missing block from the P delta by XOR.
|
* the missing block and no inverse-matrix multiplication is required.
|
||||||
*/
|
*/
|
||||||
static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, int *ip, int nd, size_t size, void **vv)
|
||||||
{
|
{
|
||||||
|
|
@ -1871,53 +1881,62 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, i
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
for (j = 0; j < nr; ++j) {
|
/*
|
||||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr]));
|
* Pdelta is the missing block directly for rec1of1.
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm12");
|
* Do not multiply it by V[0] == 1.
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm13");
|
*/
|
||||||
|
if (nr == 1 && has_p) {
|
||||||
|
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[0][i]));
|
||||||
|
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[0][i + 32]));
|
||||||
|
} else {
|
||||||
|
for (j = 0; j < nr; ++j) {
|
||||||
|
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr]));
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm12");
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm13");
|
||||||
|
|
||||||
if (nr >= 2) {
|
if (nr >= 2) {
|
||||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 1]));
|
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 1]));
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm2,%ymm15");
|
asm volatile ("vgf2p8mulb %ymm14,%ymm2,%ymm15");
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm3,%ymm15");
|
asm volatile ("vgf2p8mulb %ymm14,%ymm3,%ymm15");
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 3) {
|
||||||
|
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 2]));
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm4,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm5,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 4) {
|
||||||
|
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 3]));
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm6,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm7,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 5) {
|
||||||
|
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 4]));
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm8,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm9,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
if (nr >= 6) {
|
||||||
|
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 5]));
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm10,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||||
|
asm volatile ("vgf2p8mulb %ymm14,%ymm11,%ymm15");
|
||||||
|
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||||
|
}
|
||||||
|
|
||||||
|
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||||
|
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
||||||
}
|
}
|
||||||
|
|
||||||
if (nr >= 3) {
|
|
||||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 2]));
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm4,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm5,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nr >= 4) {
|
|
||||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 3]));
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm6,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm7,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nr >= 5) {
|
|
||||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 4]));
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm8,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm9,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
if (nr >= 6) {
|
|
||||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 5]));
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm10,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
|
||||||
asm volatile ("vgf2p8mulb %ymm14,%ymm11,%ymm15");
|
|
||||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
|
||||||
}
|
|
||||||
|
|
||||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
|
||||||
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue