mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Add specialized P+Q REC2OF2 recovery to all SIMD backends
This commit is contained in:
Andrea Mazzoleni
parent
d4184b9aee
commit
46ebce4e3c
7 changed files with 885 additions and 9 deletions
159
raid/avx2.c
159
raid/avx2.c
|
|
@ -1039,6 +1039,79 @@ static __always_inline void raid_rec2_avx2_delta(int nr, int *id, int *ip, int n
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
/* get multiplication coefficients */
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/* Pd */
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i]));
|
||||
|
||||
/* Qd */
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (qa[i]));
|
||||
|
||||
/* split Pd and Qd */
|
||||
asm volatile ("vpsrlw $4,%ymm0,%ymm2");
|
||||
asm volatile ("vpsrlw $4,%ymm1,%ymm3");
|
||||
asm volatile ("vpand %ymm7,%ymm0,%ymm4");
|
||||
asm volatile ("vpand %ymm7,%ymm1,%ymm5");
|
||||
asm volatile ("vpand %ymm7,%ymm2,%ymm2");
|
||||
asm volatile ("vpand %ymm7,%ymm3,%ymm3");
|
||||
|
||||
/* C0 * Pd */
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm6" : : "m" (raid_gfmulpshufb[C[0]][0][0]));
|
||||
asm volatile ("vpshufb %ymm4,%ymm6,%ymm6");
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[C[0]][1][0]));
|
||||
asm volatile ("vpshufb %ymm2,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm4,%ymm6,%ymm6");
|
||||
|
||||
/* C1 * Qd */
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[C[1]][0][0]));
|
||||
asm volatile ("vpshufb %ymm5,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm4,%ymm6,%ymm6");
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (raid_gfmulpshufb[C[1]][1][0]));
|
||||
asm volatile ("vpshufb %ymm3,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm4,%ymm6,%ymm6");
|
||||
|
||||
/* Dy = ymm6, Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 implementation optimized for up to four failures.
|
||||
*
|
||||
|
|
@ -1440,7 +1513,7 @@ static __always_inline void raid_recX_avx2(int nr, int *id, int *ip, int nd, siz
|
|||
/*
|
||||
* RAID recovering for two disks AVX2 extended implementation
|
||||
*/
|
||||
static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
static __always_inline void raid_rec2_avx2ext_delta(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
|
|
@ -1451,8 +1524,6 @@ static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, in
|
|||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr; /* unused, it's always 2 */
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
|
|
@ -1537,6 +1608,79 @@ static __always_inline void raid_rec2_avx2ext_delta(int nr, int *id, int *ip, in
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks with P and Q AVX2 implementation
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2ext(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[N];
|
||||
uint8_t *pa[N];
|
||||
uint8_t C[N];
|
||||
size_t i;
|
||||
|
||||
/* get multiplication coefficients */
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
p[0] = v[nd];
|
||||
p[1] = v[nd + 1];
|
||||
pa[0] = v[id[0]];
|
||||
pa[1] = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* the multiplication tables are constant for the whole recovery */
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm8" : : "m" (raid_gfmulpshufb[C[0]][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm9" : : "m" (raid_gfmulpshufb[C[0]][1][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm10" : : "m" (raid_gfmulpshufb[C[1]][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (raid_gfmulpshufb[C[1]][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/* d0 = p[0] ^ pa[0] */
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[0][i]));
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
|
||||
/* d1 = p[1] ^ pa[1] */
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[1][i]));
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
/* C[0] * d0 */
|
||||
asm volatile ("vpsrlw $4,%ymm0,%ymm5");
|
||||
asm volatile ("vpand %ymm7,%ymm0,%ymm4");
|
||||
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
|
||||
asm volatile ("vpshufb %ymm4,%ymm8,%ymm2");
|
||||
asm volatile ("vpshufb %ymm5,%ymm9,%ymm3");
|
||||
asm volatile ("vpxor %ymm3,%ymm2,%ymm6");
|
||||
|
||||
/* C[1] * d1 */
|
||||
asm volatile ("vpsrlw $4,%ymm1,%ymm5");
|
||||
asm volatile ("vpand %ymm7,%ymm1,%ymm4");
|
||||
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
|
||||
asm volatile ("vpshufb %ymm4,%ymm10,%ymm2");
|
||||
asm volatile ("vpshufb %ymm5,%ymm11,%ymm3");
|
||||
asm volatile ("vpxor %ymm2,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm3,%ymm6,%ymm6");
|
||||
|
||||
/* pa[1] = C[0] * d0 ^ C[1] * d1 */
|
||||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
|
||||
|
||||
/* pa[0] = d0 ^ pa[1] */
|
||||
asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[0][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 implementation optimized for up to three failures.
|
||||
*
|
||||
|
|
@ -2224,6 +2368,13 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
/* if recovering with P,Q use the specialized function */
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx2(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_rec2_avx2_delta(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -2277,7 +2428,7 @@ void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
/* if recovering with P,Q uses the delta function */
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2_avx2ext_delta(2, id, ip, nd, size, vv);
|
||||
raid_rec2of2_avx2ext(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -887,9 +887,89 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
raid_recX_avx512bw(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512BW implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512bw(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
/* get multiplication coefficients */
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* keep both multiplication-table pairs resident */
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[C[0]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[C[0]][1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[C[1]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[C[1]][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* Pd */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxord %0,%%zmm0,%%zmm0" : : "m" (pa[i]));
|
||||
|
||||
/* Qd */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxord %0,%%zmm1,%%zmm1" : : "m" (qa[i]));
|
||||
|
||||
/* split Pd and Qd */
|
||||
asm volatile ("vpsrlw $4,%zmm0,%zmm2");
|
||||
asm volatile ("vpsrlw $4,%zmm1,%zmm3");
|
||||
asm volatile ("vpandq %zmm31,%zmm0,%zmm4");
|
||||
asm volatile ("vpandq %zmm31,%zmm1,%zmm5");
|
||||
asm volatile ("vpandq %zmm31,%zmm2,%zmm2");
|
||||
asm volatile ("vpandq %zmm31,%zmm3,%zmm3");
|
||||
|
||||
/* C0 * Pd */
|
||||
asm volatile ("vpshufb %zmm4,%zmm16,%zmm6");
|
||||
asm volatile ("vpshufb %zmm2,%zmm17,%zmm7");
|
||||
asm volatile ("vpxord %zmm7,%zmm6,%zmm6");
|
||||
|
||||
/* C1 * Qd */
|
||||
asm volatile ("vpshufb %zmm5,%zmm18,%zmm7");
|
||||
asm volatile ("vpxord %zmm7,%zmm6,%zmm6");
|
||||
asm volatile ("vpshufb %zmm3,%zmm19,%zmm7");
|
||||
asm volatile ("vpxord %zmm7,%zmm6,%zmm6");
|
||||
|
||||
/* Dy = zmm6, Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxord %zmm6,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
/* if recovering with P,Q use the specialized function */
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx512bw(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512bw(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
250
raid/gfni.c
250
raid/gfni.c
|
|
@ -1264,9 +1264,82 @@ void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void
|
|||
raid_recX_avx2gfni_raid(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 RAID GFNI implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2gfni_raid(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
const uint8_t *T[2];
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
T[0] = raid_gfaffine_raid[C[0]];
|
||||
T[1] = raid_gfaffine_raid[C[1]];
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (T[0][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%ymm15" : : "m" (T[1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* Pd, two 32-byte lanes */
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32]));
|
||||
|
||||
/* Qd, two 32-byte lanes */
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32]));
|
||||
|
||||
/* Dy = C0 * Pd ^ C1 * Qd */
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm0,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm1,%ymm5");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm2,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm15,%ymm3,%ymm7");
|
||||
asm volatile ("vpxor %ymm6,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm7,%ymm5,%ymm5");
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i]));
|
||||
asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx2gfni_raid(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx2gfni_raid(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -1307,9 +1380,68 @@ void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, vo
|
|||
raid_recX_avx512gfni_raid(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512 RAID GFNI implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512gfni_raid(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
const uint8_t *T[2];
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
T[0] = raid_gfaffine_raid[C[0]];
|
||||
T[1] = raid_gfaffine_raid[C[1]];
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%zmm14" : : "m" (T[0][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%zmm15" : : "m" (T[1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i]));
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i]));
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm14,%zmm0,%zmm2");
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm15,%zmm1,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm2,%zmm2");
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx512gfni_raid(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512gfni_raid(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -1350,9 +1482,73 @@ void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void
|
|||
raid_recX_avx2gfni_aes(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX2 AES GFNI implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx2gfni_aes(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (C[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (C[1]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (pa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (pa[i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (q[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (q[i + 32]));
|
||||
asm volatile ("vpxor %0,%%ymm2,%%ymm2" : : "m" (qa[i]));
|
||||
asm volatile ("vpxor %0,%%ymm3,%%ymm3" : : "m" (qa[i + 32]));
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm0,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm14,%ymm1,%ymm5");
|
||||
asm volatile ("vgf2p8mulb %ymm15,%ymm2,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm15,%ymm3,%ymm7");
|
||||
asm volatile ("vpxor %ymm6,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm7,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (qa[i]));
|
||||
asm volatile ("vmovdqa %%ymm5,%0" : "=m" (qa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx2gfni_aes(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx2gfni_aes(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -1393,9 +1589,63 @@ void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, voi
|
|||
raid_recX_avx512gfni_aes(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AVX512 AES GFNI implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_avx512gfni_aes(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (C[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (C[1]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm0,%%zmm0" : : "m" (pa[i]));
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (q[i]));
|
||||
asm volatile ("vpxorq %0,%%zmm1,%%zmm1" : : "m" (qa[i]));
|
||||
|
||||
asm volatile ("vgf2p8mulb %zmm14,%zmm0,%zmm2");
|
||||
asm volatile ("vgf2p8mulb %zmm15,%zmm1,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm2,%zmm2");
|
||||
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_avx512gfni_aes(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_avx512gfni_aes(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
123
raid/neon.c
123
raid/neon.c
|
|
@ -1045,9 +1045,132 @@ void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
raid_recX_neon(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AArch64 NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_neon(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
asm volatile (
|
||||
"ldr q28, %0\n"
|
||||
"ldr q20, %1\n"
|
||||
"ldr q21, %2\n"
|
||||
"ldr q22, %3\n"
|
||||
"ldr q23, %4\n"
|
||||
:
|
||||
: "m" (gfconst16.low4[0]),
|
||||
"m" (raid_gfmulpshufb[C[0]][0][0]),
|
||||
"m" (raid_gfmulpshufb[C[0]][1][0]),
|
||||
"m" (raid_gfmulpshufb[C[1]][0][0]),
|
||||
"m" (raid_gfmulpshufb[C[1]][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
asm volatile (
|
||||
/* Pd, two lanes */
|
||||
"ldr q0, %4\n"
|
||||
"ldr q1, %5\n"
|
||||
"ldr q12, %8\n"
|
||||
"ldr q13, %9\n"
|
||||
"eor v0.16b, v0.16b, v12.16b\n"
|
||||
"eor v1.16b, v1.16b, v13.16b\n"
|
||||
|
||||
/* preserve Pd for Dx */
|
||||
"mov v8.16b, v0.16b\n"
|
||||
"mov v9.16b, v1.16b\n"
|
||||
|
||||
/* Qd, two lanes */
|
||||
"ldr q2, %6\n"
|
||||
"ldr q3, %7\n"
|
||||
"ldr q12, %10\n"
|
||||
"ldr q13, %11\n"
|
||||
"eor v2.16b, v2.16b, v12.16b\n"
|
||||
"eor v3.16b, v3.16b, v13.16b\n"
|
||||
|
||||
/* split Pd */
|
||||
"ushr v4.16b, v0.16b, #4\n"
|
||||
"ushr v5.16b, v1.16b, #4\n"
|
||||
"and v0.16b, v0.16b, v28.16b\n"
|
||||
"and v1.16b, v1.16b, v28.16b\n"
|
||||
"and v4.16b, v4.16b, v28.16b\n"
|
||||
"and v5.16b, v5.16b, v28.16b\n"
|
||||
|
||||
/* split Qd */
|
||||
"ushr v6.16b, v2.16b, #4\n"
|
||||
"ushr v7.16b, v3.16b, #4\n"
|
||||
"and v2.16b, v2.16b, v28.16b\n"
|
||||
"and v3.16b, v3.16b, v28.16b\n"
|
||||
"and v6.16b, v6.16b, v28.16b\n"
|
||||
"and v7.16b, v7.16b, v28.16b\n"
|
||||
|
||||
/* C0 * Pd */
|
||||
"tbl v10.16b, {v20.16b}, v0.16b\n"
|
||||
"tbl v11.16b, {v20.16b}, v1.16b\n"
|
||||
"tbl v12.16b, {v21.16b}, v4.16b\n"
|
||||
"tbl v13.16b, {v21.16b}, v5.16b\n"
|
||||
"eor v10.16b, v10.16b, v12.16b\n"
|
||||
"eor v11.16b, v11.16b, v13.16b\n"
|
||||
|
||||
/* C1 * Qd */
|
||||
"tbl v12.16b, {v22.16b}, v2.16b\n"
|
||||
"tbl v13.16b, {v22.16b}, v3.16b\n"
|
||||
"tbl v14.16b, {v23.16b}, v6.16b\n"
|
||||
"tbl v15.16b, {v23.16b}, v7.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
|
||||
/* Dy */
|
||||
"eor v10.16b, v10.16b, v12.16b\n"
|
||||
"eor v11.16b, v11.16b, v13.16b\n"
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
"eor v8.16b, v8.16b, v10.16b\n"
|
||||
"eor v9.16b, v9.16b, v11.16b\n"
|
||||
|
||||
"str q8, %0\n"
|
||||
"str q9, %1\n"
|
||||
"str q10, %2\n"
|
||||
"str q11, %3\n"
|
||||
: "=m" (pa[i]), "=m" (pa[i + 16]),
|
||||
"=m" (qa[i]), "=m" (qa[i + 16])
|
||||
: "m" (p[i]), "m" (p[i + 16]),
|
||||
"m" (q[i]), "m" (q[i + 16]),
|
||||
"m" (pa[i]), "m" (pa[i + 16]),
|
||||
"m" (qa[i]), "m" (qa[i + 16])
|
||||
);
|
||||
}
|
||||
|
||||
raid_neon_end();
|
||||
}
|
||||
|
||||
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_neon(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_neon(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
105
raid/neon32.c
105
raid/neon32.c
|
|
@ -1225,9 +1225,114 @@ void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
raid_recX_neon32_123(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q AArch32 NEON implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_neon32(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_neon32_begin();
|
||||
|
||||
/* q15 = nibble mask, q9-q12 = C0/C1 low/high tables */
|
||||
asm volatile (
|
||||
"vld1.8 {q15}, %0\n"
|
||||
"vld1.8 {q9}, %1\n"
|
||||
"vld1.8 {q10}, %2\n"
|
||||
"vld1.8 {q11}, %3\n"
|
||||
"vld1.8 {q12}, %4\n"
|
||||
:
|
||||
: "Q" (gfconst16.low4[0]),
|
||||
"Q" (raid_gfmulpshufb[C[0]][0][0]),
|
||||
"Q" (raid_gfmulpshufb[C[0]][1][0]),
|
||||
"Q" (raid_gfmulpshufb[C[1]][0][0]),
|
||||
"Q" (raid_gfmulpshufb[C[1]][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
asm volatile (
|
||||
/* Pd */
|
||||
"vld1.8 {q0}, %2\n"
|
||||
"vld1.8 {q13}, %4\n"
|
||||
"veor q0, q0, q13\n"
|
||||
|
||||
/* Qd */
|
||||
"vld1.8 {q1}, %3\n"
|
||||
"vld1.8 {q13}, %5\n"
|
||||
"veor q1, q1, q13\n"
|
||||
|
||||
/* split Pd */
|
||||
"vmov q2, q0\n"
|
||||
"vshr.u8 q3, q0, #4\n"
|
||||
"vand q2, q2, q15\n"
|
||||
"vand q3, q3, q15\n"
|
||||
|
||||
/* split Qd */
|
||||
"vmov q4, q1\n"
|
||||
"vshr.u8 q5, q1, #4\n"
|
||||
"vand q4, q4, q15\n"
|
||||
"vand q5, q5, q15\n"
|
||||
|
||||
/* C0 low * Pd low -> q6 */
|
||||
"vtbl.8 d12, {d18-d19}, d4\n"
|
||||
"vtbl.8 d13, {d18-d19}, d5\n"
|
||||
|
||||
/* C0 high * Pd high -> q7 */
|
||||
"vtbl.8 d14, {d20-d21}, d6\n"
|
||||
"vtbl.8 d15, {d20-d21}, d7\n"
|
||||
"veor q6, q6, q7\n"
|
||||
|
||||
/* C1 low * Qd low -> q7 */
|
||||
"vtbl.8 d14, {d22-d23}, d8\n"
|
||||
"vtbl.8 d15, {d22-d23}, d9\n"
|
||||
|
||||
/* C1 high * Qd high -> q8 */
|
||||
"vtbl.8 d16, {d24-d25}, d10\n"
|
||||
"vtbl.8 d17, {d24-d25}, d11\n"
|
||||
"veor q7, q7, q8\n"
|
||||
|
||||
/* Dy */
|
||||
"veor q6, q6, q7\n"
|
||||
|
||||
/* Dx = Pd ^ Dy */
|
||||
"veor q0, q0, q6\n"
|
||||
|
||||
"vst1.8 {q0}, %0\n"
|
||||
"vst1.8 {q6}, %1\n"
|
||||
: "=Q" (pa[i]), "=Q" (qa[i])
|
||||
: "Q" (p[i]), "Q" (q[i]),
|
||||
"Q" (pa[i]), "Q" (qa[i])
|
||||
);
|
||||
}
|
||||
|
||||
raid_neon32_end();
|
||||
}
|
||||
|
||||
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_neon32(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_recX_neon32_123(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
172
raid/ssse3.c
172
raid/ssse3.c
|
|
@ -1304,6 +1304,83 @@ static __always_inline void raid_rec2_ssse3_delta(int nr, int *id, int *ip, int
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q SSSE3 implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_ssse3(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
/* get multiplication coefficients */
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
/* Pd */
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
||||
asm volatile ("pxor %0,%%xmm0" : : "m" (pa[i]));
|
||||
|
||||
/* Qd */
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (q[i]));
|
||||
asm volatile ("pxor %0,%%xmm1" : : "m" (qa[i]));
|
||||
|
||||
/* split Pd and Qd into low/high nibbles */
|
||||
asm volatile ("movdqa %xmm0,%xmm4");
|
||||
asm volatile ("movdqa %xmm1,%xmm5");
|
||||
asm volatile ("movdqa %xmm0,%xmm2");
|
||||
asm volatile ("movdqa %xmm1,%xmm3");
|
||||
asm volatile ("psrlw $4,%xmm2");
|
||||
asm volatile ("psrlw $4,%xmm3");
|
||||
asm volatile ("pand %xmm7,%xmm4");
|
||||
asm volatile ("pand %xmm7,%xmm5");
|
||||
asm volatile ("pand %xmm7,%xmm2");
|
||||
asm volatile ("pand %xmm7,%xmm3");
|
||||
|
||||
/* C0 * Pd */
|
||||
asm volatile ("movdqa %0,%%xmm6" : : "m" (raid_gfmulpshufb[C[0]][0][0]));
|
||||
asm volatile ("pshufb %xmm4,%xmm6");
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[C[0]][1][0]));
|
||||
asm volatile ("pshufb %xmm2,%xmm4");
|
||||
asm volatile ("pxor %xmm4,%xmm6");
|
||||
|
||||
/* C1 * Qd */
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[C[1]][0][0]));
|
||||
asm volatile ("pshufb %xmm5,%xmm4");
|
||||
asm volatile ("pxor %xmm4,%xmm6");
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm4" : : "m" (raid_gfmulpshufb[C[1]][1][0]));
|
||||
asm volatile ("pshufb %xmm3,%xmm4");
|
||||
asm volatile ("pxor %xmm4,%xmm6");
|
||||
|
||||
/* xmm6 = Dy, xmm0 = Pd, so Dx = Pd ^ Dy */
|
||||
asm volatile ("pxor %xmm6,%xmm0");
|
||||
|
||||
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("movdqa %%xmm6,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3_1234(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -1858,6 +1935,87 @@ static __always_inline void raid_rec2_ssse3ext_delta(int nr, int *id, int *ip, i
|
|||
raid_sse_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* Recover failure of two data blocks using P and Q SSSE3 extended implementation.
|
||||
*/
|
||||
static __always_inline void raid_rec2of2_ssse3ext(int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t *q;
|
||||
uint8_t *qa;
|
||||
uint8_t C[2];
|
||||
size_t i;
|
||||
|
||||
/* get multiplication coefficients */
|
||||
C[0] = inv(powgen(id[1] - id[0]) ^ 1);
|
||||
C[1] = inv(powgen(id[0]) ^ powgen(id[1]));
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(2, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd];
|
||||
q = v[nd + 1];
|
||||
pa = v[id[0]];
|
||||
qa = v[id[1]];
|
||||
|
||||
raid_sse_begin();
|
||||
|
||||
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* keep the four multiplication tables resident */
|
||||
asm volatile ("movdqa %0,%%xmm8" : : "m" (raid_gfmulpshufb[C[0]][0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm9" : : "m" (raid_gfmulpshufb[C[0]][1][0]));
|
||||
asm volatile ("movdqa %0,%%xmm10" : : "m" (raid_gfmulpshufb[C[1]][0][0]));
|
||||
asm volatile ("movdqa %0,%%xmm11" : : "m" (raid_gfmulpshufb[C[1]][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
/* Pd */
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
|
||||
asm volatile ("pxor %0,%%xmm0" : : "m" (pa[i]));
|
||||
|
||||
/* Qd */
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (q[i]));
|
||||
asm volatile ("pxor %0,%%xmm1" : : "m" (qa[i]));
|
||||
|
||||
/* split Pd and Qd */
|
||||
asm volatile ("movdqa %xmm0,%xmm4");
|
||||
asm volatile ("movdqa %xmm1,%xmm5");
|
||||
asm volatile ("movdqa %xmm0,%xmm2");
|
||||
asm volatile ("movdqa %xmm1,%xmm3");
|
||||
asm volatile ("psrlw $4,%xmm2");
|
||||
asm volatile ("psrlw $4,%xmm3");
|
||||
asm volatile ("pand %xmm7,%xmm4");
|
||||
asm volatile ("pand %xmm7,%xmm5");
|
||||
asm volatile ("pand %xmm7,%xmm2");
|
||||
asm volatile ("pand %xmm7,%xmm3");
|
||||
|
||||
/* C0 * Pd */
|
||||
asm volatile ("movdqa %xmm8,%xmm6");
|
||||
asm volatile ("pshufb %xmm4,%xmm6");
|
||||
asm volatile ("movdqa %xmm9,%xmm12");
|
||||
asm volatile ("pshufb %xmm2,%xmm12");
|
||||
asm volatile ("pxor %xmm12,%xmm6");
|
||||
|
||||
/* C1 * Qd */
|
||||
asm volatile ("movdqa %xmm10,%xmm12");
|
||||
asm volatile ("pshufb %xmm5,%xmm12");
|
||||
asm volatile ("pxor %xmm12,%xmm6");
|
||||
asm volatile ("movdqa %xmm11,%xmm12");
|
||||
asm volatile ("pshufb %xmm3,%xmm12");
|
||||
asm volatile ("pxor %xmm12,%xmm6");
|
||||
|
||||
/* Dy = xmm6, Dx = Pd ^ Dy */
|
||||
asm volatile ("pxor %xmm6,%xmm0");
|
||||
|
||||
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("movdqa %%xmm6,%0" : "=m" (qa[i]));
|
||||
}
|
||||
|
||||
raid_sse_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_ssse3ext_123(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
{
|
||||
|
|
@ -2459,6 +2617,13 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
/* if recovering with P,Q use the specialized function */
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_ssse3(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_rec2_ssse3_delta(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
@ -2503,6 +2668,13 @@ void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 2);
|
||||
|
||||
/* if recovering with P,Q use the specialized function */
|
||||
if (ip[0] == 0 && ip[1] == 1) {
|
||||
raid_rec2of2_ssse3ext(id, ip, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_rec2_ssse3ext_delta(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue