mirror of
https://github.com/amadvance/snapraid.git
synced 2026-11-19 22:46:36 -05:00
Add XOR-only fast path for AVX2 GFNI rec1of1
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
Build / build-freebsd (push) Waiting to run
Build / build-musl (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run
Some checks are pending
Build / build-mac (push) Waiting to run
Build / build-linux (push) Waiting to run
Build / build-arm32 (push) Waiting to run
Build / build-win (push) Waiting to run
Build / build-freebsd (push) Waiting to run
Build / build-musl (push) Waiting to run
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
CodeQL Advanced / Analyze (actions) (push) Waiting to run
This commit is contained in:
parent
58468cfb55
commit
1a15ce02d9
1 changed files with 110 additions and 91 deletions
201
raid/gfni.c
201
raid/gfni.c
|
|
@ -1495,55 +1495,65 @@ static __always_inline void raid_recX_avx2gfni_raid(int nr, int has_p, int *id,
|
|||
}
|
||||
}
|
||||
|
||||
for (j = 0; j < nr; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
/*
|
||||
* With a single failure recovered from P, ymm0/ymm1
|
||||
* already contain the missing data. Multiplication by
|
||||
* the inverse coefficient 1 would be redundant.
|
||||
*/
|
||||
if (nr == 1 && has_p) {
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[0][i + 32]));
|
||||
} else {
|
||||
for (j = 0; j < nr; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm13");
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[0][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm13");
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm2,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm3,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[1][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm2,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm3,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm4,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm5,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm6,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm7,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm8,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm9,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm10,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm11,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[2][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm4,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm5,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[3][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm6,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm7,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[4][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm8,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm9,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (t[5][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm10,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm11,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -1871,53 +1881,62 @@ static __always_inline void raid_recX_avx2gfni_aes(int nr, int has_p, int *id, i
|
|||
}
|
||||
}
|
||||
|
||||
for (j = 0; j < nr; ++j) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm13");
|
||||
/*
|
||||
* Pdelta is the missing block directly for rec1of1.
|
||||
* Do not multiply it by V[0] == 1.
|
||||
*/
|
||||
if (nr == 1 && has_p) {
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[0][i + 32]));
|
||||
} else {
|
||||
for (j = 0; j < nr; ++j) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm13");
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 1]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm2,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm3,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 1]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm2,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm3,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 2]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm4,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm5,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 3]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm6,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm7,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 4]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm8,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm9,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 5]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm10,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm11,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 2]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm4,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm5,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 3]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm6,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm7,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 4]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm8,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm9,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (V[j * nr + 5]));
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm10,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm12,%ymm12");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm11,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm13,%ymm13");
|
||||
}
|
||||
|
||||
asm volatile ("vmovdqa %%ymm12,%0" : "=m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %%ymm13,%0" : "=m" (pa[j][i + 32]));
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue