mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 02:49:02 -04:00
Optimize multi-parity recovery with fused syndrome generation
This commit is contained in:
Andrea Mazzoleni
parent
9d842b0dfb
commit
ae703ce68c
12 changed files with 6607 additions and 3934 deletions
1590
raid/avx2.c
1590
raid/avx2.c
File diff suppressed because it is too large
Load diff
671
raid/avx512.c
671
raid/avx512.c
|
|
@ -6,7 +6,6 @@
|
|||
#include "cpu.h"
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) AVX512BW implementation
|
||||
*
|
||||
|
|
@ -544,6 +543,313 @@ static __always_inline void raid_genX_avx512bw(int nd, size_t size, void **vv, i
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX512BW implementation.
|
||||
*
|
||||
* Compute only the selected syndromes directly from the stored parity
|
||||
* and surviving data. This avoids raid_delta_gen(), temporary delta
|
||||
* parity blocks, and generation of unused parity rows.
|
||||
*
|
||||
* AVX512 provides enough registers to keep all six supported syndromes
|
||||
* in registers in a single surviving-data scan.
|
||||
*/
|
||||
static __always_inline void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t *src[RAID_DATA_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
||||
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
||||
size_t i;
|
||||
int d, j, k, s;
|
||||
int ns;
|
||||
int has_p;
|
||||
|
||||
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
G[j * nr + k] = A(ip[j], id[k]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, nr);
|
||||
|
||||
/* setup selected parity and destination pointers */
|
||||
for (j = 0; j < nr; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
/* ip[] is ordered. If P is available, it is always ip[0] */
|
||||
has_p = ip[0] == 0;
|
||||
|
||||
/*
|
||||
* Build the compact list of surviving data blocks and precompute
|
||||
* the multiplication-table pointers for each selected syndrome.
|
||||
*/
|
||||
ns = 0;
|
||||
k = 0;
|
||||
|
||||
for (d = 0; d < nd; ++d) {
|
||||
if (k < nr && d == id[k]) {
|
||||
++k;
|
||||
continue;
|
||||
}
|
||||
|
||||
src[ns] = v[d];
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
||||
|
||||
++ns;
|
||||
}
|
||||
|
||||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/* precompute inverse-matrix multiplication table pointers */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
/*
|
||||
* Register allocation during syndrome generation:
|
||||
*
|
||||
* zmm0 syndrome 0
|
||||
* zmm2 syndrome 1
|
||||
* zmm4 syndrome 2
|
||||
* zmm6 syndrome 3
|
||||
* zmm8 syndrome 4
|
||||
* zmm10 syndrome 5
|
||||
*
|
||||
* zmm12 source / source low nibble
|
||||
* zmm13 source high nibble
|
||||
* zmm14 low multiplication table / result
|
||||
* zmm15 high multiplication table / result
|
||||
* zmm31 low-nibble mask
|
||||
*
|
||||
* After syndrome splitting:
|
||||
*
|
||||
* zmm0 /zmm1 syndrome 0 low/high
|
||||
* zmm2 /zmm3 syndrome 1 low/high
|
||||
* zmm4 /zmm5 syndrome 2 low/high
|
||||
* zmm6 /zmm7 syndrome 3 low/high
|
||||
* zmm8 /zmm9 syndrome 4 low/high
|
||||
* zmm10/zmm11 syndrome 5 low/high
|
||||
*/
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* start every selected syndrome from its stored parity block */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
|
||||
|
||||
if (nr >= 2)
|
||||
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (p[1][i]));
|
||||
|
||||
if (nr >= 3)
|
||||
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (p[2][i]));
|
||||
|
||||
if (nr >= 4)
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (p[3][i]));
|
||||
|
||||
if (nr >= 5)
|
||||
asm volatile ("vmovdqa64 %0,%%zmm8" : : "m" (p[4][i]));
|
||||
|
||||
if (nr >= 6)
|
||||
asm volatile ("vmovdqa64 %0,%%zmm10" : : "m" (p[5][i]));
|
||||
|
||||
/* add all surviving data contributions in one source scan */
|
||||
for (s = 0; s < ns; ++s) {
|
||||
const uint8_t **t = S[s];
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm12" : : "m" (src[s][i]));
|
||||
|
||||
/*
|
||||
* P has coefficient 1 for every data disk.
|
||||
* XOR the original source before splitting it.
|
||||
*/
|
||||
if (has_p) {
|
||||
asm volatile ("vpxorq %zmm12,%zmm0,%zmm0");
|
||||
|
||||
asm volatile ("vpsrlw $4,%zmm12,%zmm13");
|
||||
asm volatile ("vpandq %zmm31,%zmm12,%zmm12");
|
||||
asm volatile ("vpandq %zmm31,%zmm13,%zmm13");
|
||||
} else {
|
||||
asm volatile ("vpsrlw $4,%zmm12,%zmm13");
|
||||
asm volatile ("vpandq %zmm31,%zmm12,%zmm12");
|
||||
asm volatile ("vpandq %zmm31,%zmm13,%zmm13");
|
||||
|
||||
/* syndrome 0 */
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[0][16]));
|
||||
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
|
||||
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm0");
|
||||
}
|
||||
|
||||
/* syndrome 1 */
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16]));
|
||||
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
|
||||
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm2");
|
||||
}
|
||||
|
||||
/* syndrome 2 */
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16]));
|
||||
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
|
||||
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm4");
|
||||
}
|
||||
|
||||
/* syndrome 3 */
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16]));
|
||||
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
|
||||
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm6");
|
||||
}
|
||||
|
||||
/* syndrome 4 */
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16]));
|
||||
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
|
||||
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm8");
|
||||
}
|
||||
|
||||
/* syndrome 5 */
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16]));
|
||||
asm volatile ("vpshufb %zmm12,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm13,%zmm15,%zmm15");
|
||||
asm volatile ("vpternlogq $0x96,%zmm14,%zmm15,%zmm10");
|
||||
}
|
||||
}
|
||||
|
||||
/* split every completed syndrome once into low/high nibbles */
|
||||
asm volatile ("vpsrlw $4,%zmm0,%zmm1");
|
||||
asm volatile ("vpandq %zmm31,%zmm0,%zmm0");
|
||||
asm volatile ("vpandq %zmm31,%zmm1,%zmm1");
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vpsrlw $4,%zmm2,%zmm3");
|
||||
asm volatile ("vpandq %zmm31,%zmm2,%zmm2");
|
||||
asm volatile ("vpandq %zmm31,%zmm3,%zmm3");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vpsrlw $4,%zmm4,%zmm5");
|
||||
asm volatile ("vpandq %zmm31,%zmm4,%zmm4");
|
||||
asm volatile ("vpandq %zmm31,%zmm5,%zmm5");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vpsrlw $4,%zmm6,%zmm7");
|
||||
asm volatile ("vpandq %zmm31,%zmm6,%zmm6");
|
||||
asm volatile ("vpandq %zmm31,%zmm7,%zmm7");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vpsrlw $4,%zmm8,%zmm9");
|
||||
asm volatile ("vpandq %zmm31,%zmm8,%zmm8");
|
||||
asm volatile ("vpandq %zmm31,%zmm9,%zmm9");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vpsrlw $4,%zmm10,%zmm11");
|
||||
asm volatile ("vpandq %zmm31,%zmm10,%zmm10");
|
||||
asm volatile ("vpandq %zmm31,%zmm11,%zmm11");
|
||||
}
|
||||
|
||||
/*
|
||||
* Reconstruct every missing data block.
|
||||
*
|
||||
* Keep independent low/high dependency chains:
|
||||
*
|
||||
* zmm12 low accumulator
|
||||
* zmm13 high accumulator
|
||||
* zmm14 low multiplication table/result
|
||||
* zmm15 high multiplication table/result
|
||||
*
|
||||
* This is intentional. Do not serialize low/high products
|
||||
* through a single temporary register.
|
||||
*/
|
||||
for (j = 0; j < nr; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
/* coefficient 0 initializes both accumulators */
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm12" : : "m" (t[0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm13" : : "m" (t[0][16]));
|
||||
asm volatile ("vpshufb %zmm0,%zmm12,%zmm12");
|
||||
asm volatile ("vpshufb %zmm1,%zmm13,%zmm13");
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[1][16]));
|
||||
asm volatile ("vpshufb %zmm2,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm3,%zmm15,%zmm15");
|
||||
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
|
||||
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[2][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[2][16]));
|
||||
asm volatile ("vpshufb %zmm4,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm5,%zmm15,%zmm15");
|
||||
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
|
||||
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[3][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[3][16]));
|
||||
asm volatile ("vpshufb %zmm6,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm7,%zmm15,%zmm15");
|
||||
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
|
||||
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[4][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[4][16]));
|
||||
asm volatile ("vpshufb %zmm8,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm9,%zmm15,%zmm15");
|
||||
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
|
||||
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (t[5][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (t[5][16]));
|
||||
asm volatile ("vpshufb %zmm10,%zmm14,%zmm14");
|
||||
asm volatile ("vpshufb %zmm11,%zmm15,%zmm15");
|
||||
asm volatile ("vpxorq %zmm14,%zmm12,%zmm12");
|
||||
asm volatile ("vpxorq %zmm15,%zmm13,%zmm13");
|
||||
}
|
||||
|
||||
asm volatile ("vpxorq %zmm13,%zmm12,%zmm12");
|
||||
asm volatile ("vmovdqa64 %%zmm12,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX512BW implementation
|
||||
*/
|
||||
|
|
@ -568,355 +874,47 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
|
|||
raid_genX_avx512bw(nd, size, vv, 6);
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for one disk AVX512BW implementation
|
||||
*/
|
||||
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
uint8_t G, V;
|
||||
size_t i;
|
||||
|
||||
(void)nr;
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
G = A(ip[0], id[0]);
|
||||
V = inv(G);
|
||||
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (raid_gfmulpshufb[V][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (raid_gfmulpshufb[V][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
|
||||
asm volatile ("vpxord %zmm1,%zmm0,%zmm0");
|
||||
asm volatile ("vpsrlw $4,%zmm0,%zmm1");
|
||||
asm volatile ("vpandd %zmm7,%zmm0,%zmm0");
|
||||
asm volatile ("vpandd %zmm7,%zmm1,%zmm1");
|
||||
asm volatile ("vpshufb %zmm0,%zmm4,%zmm2");
|
||||
asm volatile ("vpshufb %zmm1,%zmm5,%zmm3");
|
||||
asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
|
||||
asm volatile ("vmovdqa64 %%zmm2, %0" : "=m" (pa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
raid_recX_avx512bw(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks AVX512BW implementation
|
||||
*/
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[N];
|
||||
uint8_t *pa[N];
|
||||
uint8_t G[N * N];
|
||||
uint8_t V[N * N];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr; /* unused, it's always 2 */
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, N);
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
/*
|
||||
* zmm7 = 0x0f nibble mask
|
||||
*
|
||||
* zmm16 = V[0] low table
|
||||
* zmm17 = V[0] high table
|
||||
* zmm18 = V[1] low table
|
||||
* zmm19 = V[1] high table
|
||||
* zmm20 = V[2] low table
|
||||
* zmm21 = V[2] high table
|
||||
* zmm22 = V[3] low table
|
||||
* zmm23 = V[3] high table
|
||||
*/
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* the inverse matrix V[] is constant for the whole recovery. */
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm16" : : "m" (raid_gfmulpshufb[V[0]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm17" : : "m" (raid_gfmulpshufb[V[0]][1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm18" : : "m" (raid_gfmulpshufb[V[1]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm19" : : "m" (raid_gfmulpshufb[V[1]][1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm20" : : "m" (raid_gfmulpshufb[V[2]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm21" : : "m" (raid_gfmulpshufb[V[2]][1][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm22" : : "m" (raid_gfmulpshufb[V[3]][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm23" : : "m" (raid_gfmulpshufb[V[3]][1][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* d0 = p[0] ^ pa[0] */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
|
||||
asm volatile ("vpxord %zmm2,%zmm0,%zmm0");
|
||||
|
||||
/* d1 = p[1] ^ pa[1] */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
|
||||
asm volatile ("vpxord %zmm3,%zmm1,%zmm1");
|
||||
|
||||
/*
|
||||
* Split both deltas into low/high nibbles once.
|
||||
*
|
||||
* zmm4 = d0 low
|
||||
* zmm5 = d0 high
|
||||
* zmm12 = d1 low
|
||||
* zmm13 = d1 high
|
||||
*/
|
||||
asm volatile ("vpsrlw $4,%zmm0,%zmm5");
|
||||
asm volatile ("vpsrlw $4,%zmm1,%zmm13");
|
||||
asm volatile ("vpandd %zmm7,%zmm0,%zmm4");
|
||||
asm volatile ("vpandd %zmm7,%zmm5,%zmm5");
|
||||
asm volatile ("vpandd %zmm7,%zmm1,%zmm12");
|
||||
asm volatile ("vpandd %zmm7,%zmm13,%zmm13");
|
||||
|
||||
/*
|
||||
* pa[0] = V[0] * d0 ^ V[1] * d1
|
||||
*/
|
||||
asm volatile ("vpshufb %zmm4,%zmm16,%zmm2");
|
||||
asm volatile ("vpshufb %zmm5,%zmm17,%zmm3");
|
||||
asm volatile ("vpshufb %zmm12,%zmm18,%zmm10");
|
||||
asm volatile ("vpshufb %zmm13,%zmm19,%zmm11");
|
||||
|
||||
asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
|
||||
asm volatile ("vpxord %zmm10,%zmm2,%zmm2");
|
||||
asm volatile ("vpxord %zmm11,%zmm2,%zmm2");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[0][i]));
|
||||
|
||||
/*
|
||||
* pa[1] = V[2] * d0 ^ V[3] * d1
|
||||
*
|
||||
* Reuse the already computed low/high nibbles.
|
||||
*/
|
||||
asm volatile ("vpshufb %zmm4,%zmm20,%zmm2");
|
||||
asm volatile ("vpshufb %zmm5,%zmm21,%zmm3");
|
||||
asm volatile ("vpshufb %zmm12,%zmm22,%zmm10");
|
||||
asm volatile ("vpshufb %zmm13,%zmm23,%zmm11");
|
||||
|
||||
asm volatile ("vpxord %zmm3,%zmm2,%zmm2");
|
||||
asm volatile ("vpxord %zmm10,%zmm2,%zmm2");
|
||||
asm volatile ("vpxord %zmm11,%zmm2,%zmm2");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm2,%0" : "=m" (pa[1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
BUG_ON(nr != 2);
|
||||
raid_recX_avx512bw(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX512BW implementation
|
||||
*/
|
||||
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
size_t i;
|
||||
int j, k;
|
||||
BUG_ON(nr != 3);
|
||||
raid_recX_avx512bw(3, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 4);
|
||||
raid_recX_avx512bw(4, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, N);
|
||||
void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 5);
|
||||
raid_recX_avx512bw(5, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/* precompute shuffle table pointers */
|
||||
for (j = 0; j < N * N; ++j)
|
||||
T[j] = &raid_gfmulpshufb[V[j]][0][0];
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* delta */
|
||||
asm volatile (
|
||||
"movq 0(%2), %%rax\n"
|
||||
"movq 0(%3), %%rbx\n"
|
||||
"vmovdqa64 (%%rax, %1), %%zmm12\n"
|
||||
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
|
||||
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
|
||||
"vpsrlw $4, %%zmm12, %%zmm1\n"
|
||||
"vpandq %%zmm31, %%zmm12, %%zmm0\n"
|
||||
"vpandq %%zmm31, %%zmm1, %%zmm1\n"
|
||||
"cmpq $1, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 8(%2), %%rax\n"
|
||||
"movq 8(%3), %%rbx\n"
|
||||
"vmovdqa64 (%%rax, %1), %%zmm12\n"
|
||||
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
|
||||
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
|
||||
"vpsrlw $4, %%zmm12, %%zmm3\n"
|
||||
"vpandq %%zmm31, %%zmm12, %%zmm2\n"
|
||||
"vpandq %%zmm31, %%zmm3, %%zmm3\n"
|
||||
"cmpq $2, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 16(%2), %%rax\n"
|
||||
"movq 16(%3), %%rbx\n"
|
||||
"vmovdqa64 (%%rax, %1), %%zmm12\n"
|
||||
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
|
||||
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
|
||||
"vpsrlw $4, %%zmm12, %%zmm5\n"
|
||||
"vpandq %%zmm31, %%zmm12, %%zmm4\n"
|
||||
"vpandq %%zmm31, %%zmm5, %%zmm5\n"
|
||||
"cmpq $3, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 24(%2), %%rax\n"
|
||||
"movq 24(%3), %%rbx\n"
|
||||
"vmovdqa64 (%%rax, %1), %%zmm12\n"
|
||||
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
|
||||
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
|
||||
"vpsrlw $4, %%zmm12, %%zmm7\n"
|
||||
"vpandq %%zmm31, %%zmm12, %%zmm6\n"
|
||||
"vpandq %%zmm31, %%zmm7, %%zmm7\n"
|
||||
"cmpq $4, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 32(%2), %%rax\n"
|
||||
"movq 32(%3), %%rbx\n"
|
||||
"vmovdqa64 (%%rax, %1), %%zmm12\n"
|
||||
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
|
||||
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
|
||||
"vpsrlw $4, %%zmm12, %%zmm9\n"
|
||||
"vpandq %%zmm31, %%zmm12, %%zmm8\n"
|
||||
"vpandq %%zmm31, %%zmm9, %%zmm9\n"
|
||||
"cmpq $5, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 40(%2), %%rax\n"
|
||||
"movq 40(%3), %%rbx\n"
|
||||
"vmovdqa64 (%%rax, %1), %%zmm12\n"
|
||||
"vmovdqa64 (%%rbx, %1), %%zmm13\n"
|
||||
"vpxorq %%zmm13, %%zmm12, %%zmm12\n"
|
||||
"vpsrlw $4, %%zmm12, %%zmm11\n"
|
||||
"vpandq %%zmm31, %%zmm12, %%zmm10\n"
|
||||
"vpandq %%zmm31, %%zmm11, %%zmm11\n"
|
||||
|
||||
"1:\n"
|
||||
:
|
||||
: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
|
||||
: "rax", "rbx", "cc", "memory"
|
||||
);
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile (
|
||||
"movq 0(%2), %%rcx\n"
|
||||
"vbroadcasti32x4 0(%%rcx), %%zmm14\n"
|
||||
"vbroadcasti32x4 16(%%rcx), %%zmm15\n"
|
||||
"vpshufb %%zmm0, %%zmm14, %%zmm14\n"
|
||||
"vpshufb %%zmm1, %%zmm15, %%zmm15\n"
|
||||
"cmpq $1, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 8(%2), %%rcx\n"
|
||||
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
|
||||
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
|
||||
"vpshufb %%zmm2, %%zmm12, %%zmm12\n"
|
||||
"vpshufb %%zmm3, %%zmm13, %%zmm13\n"
|
||||
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
|
||||
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
|
||||
"cmpq $2, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 16(%2), %%rcx\n"
|
||||
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
|
||||
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
|
||||
"vpshufb %%zmm4, %%zmm12, %%zmm12\n"
|
||||
"vpshufb %%zmm5, %%zmm13, %%zmm13\n"
|
||||
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
|
||||
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
|
||||
"cmpq $3, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 24(%2), %%rcx\n"
|
||||
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
|
||||
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
|
||||
"vpshufb %%zmm6, %%zmm12, %%zmm12\n"
|
||||
"vpshufb %%zmm7, %%zmm13, %%zmm13\n"
|
||||
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
|
||||
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
|
||||
"cmpq $4, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 32(%2), %%rcx\n"
|
||||
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
|
||||
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
|
||||
"vpshufb %%zmm8, %%zmm12, %%zmm12\n"
|
||||
"vpshufb %%zmm9, %%zmm13, %%zmm13\n"
|
||||
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
|
||||
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
|
||||
"cmpq $5, %0\n"
|
||||
"jbe 1f\n"
|
||||
|
||||
"movq 40(%2), %%rcx\n"
|
||||
"vbroadcasti32x4 0(%%rcx), %%zmm12\n"
|
||||
"vbroadcasti32x4 16(%%rcx), %%zmm13\n"
|
||||
"vpshufb %%zmm10, %%zmm12, %%zmm12\n"
|
||||
"vpshufb %%zmm11, %%zmm13, %%zmm13\n"
|
||||
"vpxorq %%zmm12, %%zmm14, %%zmm14\n"
|
||||
"vpxorq %%zmm13, %%zmm15, %%zmm15\n"
|
||||
|
||||
"1:\n"
|
||||
"vpxorq %%zmm15, %%zmm14, %%zmm14\n"
|
||||
"movq %3, %%rax\n"
|
||||
"vmovdqa64 %%zmm14, (%%rax, %1)\n"
|
||||
:
|
||||
: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
|
||||
: "rax", "rcx", "cc", "memory"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 6);
|
||||
raid_recX_avx512bw(6, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
void raid_register_avx512(void)
|
||||
|
|
@ -932,11 +930,12 @@ void raid_register_avx512(void)
|
|||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "avx512", raid_rec1_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "avx512", raid_rec2_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_recX_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "avx512", raid_rec3_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "avx512", raid_rec4_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "avx512", raid_rec5_avx512bw, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "avx512", raid_rec6_avx512bw, RAID_POLY_ANY);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
1869
raid/gfni.c
1869
raid/gfni.c
File diff suppressed because it is too large
Load diff
|
|
@ -533,7 +533,7 @@ void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
(void)nr; /* unused, it's always 1 */
|
||||
|
||||
/* if it's RAID5, use the faster function */
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
|
|
|
|||
|
|
@ -169,25 +169,58 @@ void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
|||
void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_gen1_neon(int nd, size_t size, void **vv);
|
||||
void raid_gen2_neon_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen2_neon_aes(int nd, size_t size, void **vv);
|
||||
|
|
@ -202,7 +235,10 @@ void raid_gen6_neon_raid(int nd, size_t size, void **vv);
|
|||
void raid_gen6_neon_aes(int nd, size_t size, void **vv);
|
||||
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_gen1_neon32(int nd, size_t size, void **vv);
|
||||
void raid_gen2_neon32_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen2_neon32_aes(int nd, size_t size, void **vv);
|
||||
|
|
@ -217,7 +253,10 @@ void raid_gen6_neon32_raid(int nd, size_t size, void **vv);
|
|||
void raid_gen6_neon32_aes(int nd, size_t size, void **vv);
|
||||
void raid_rec1_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec3_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec4_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec5_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec6_neon32(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
|
||||
/*
|
||||
* Functions for parity computation.
|
||||
|
|
|
|||
866
raid/neon.c
866
raid/neon.c
|
|
@ -6,7 +6,6 @@
|
|||
#include "cpu.h"
|
||||
|
||||
#ifdef CONFIG_NEON
|
||||
|
||||
/*
|
||||
* GEN1 (RAID5 with xor) NEON implementation
|
||||
*/
|
||||
|
|
@ -94,7 +93,7 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
if (generator == 3) {
|
||||
/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
|
||||
/* v14-v15 are otherwise unused and preserve Q across the AES xtime */
|
||||
asm volatile (
|
||||
"mov v14.16b, v2.16b\n"
|
||||
"mov v15.16b, v3.16b\n"
|
||||
|
|
@ -143,16 +142,6 @@ static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, i
|
|||
raid_neon_end();
|
||||
}
|
||||
|
||||
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon_gen(nd, size, vv, 2);
|
||||
}
|
||||
|
||||
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) NEON implementation
|
||||
*/
|
||||
|
|
@ -577,6 +566,420 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
|
|||
raid_neon_end();
|
||||
}
|
||||
|
||||
static __always_inline void raid_recX_neon(int nr, int *id, int *ip,
|
||||
int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t *src[RAID_DATA_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
const uint8_t *S[RAID_DATA_MAX][RAID_PARITY_MAX];
|
||||
const uint8_t *R[RAID_PARITY_MAX][RAID_PARITY_MAX];
|
||||
size_t i;
|
||||
int d, j, k, s;
|
||||
int ns;
|
||||
int has_p;
|
||||
|
||||
BUG_ON(nr < 1 || nr > RAID_PARITY_MAX);
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
G[j * nr + k] = A(ip[j], id[k]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, nr);
|
||||
|
||||
/* setup selected parity and destination pointers */
|
||||
for (j = 0; j < nr; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
/* ip[] is ordered. If P is available, it is always ip[0] */
|
||||
has_p = ip[0] == 0;
|
||||
|
||||
/*
|
||||
* Build the compact list of surviving data blocks and precompute
|
||||
* the multiplication-table pointers for each selected syndrome.
|
||||
*/
|
||||
ns = 0;
|
||||
k = 0;
|
||||
|
||||
for (d = 0; d < nd; ++d) {
|
||||
if (k < nr && d == id[k]) {
|
||||
++k;
|
||||
continue;
|
||||
}
|
||||
|
||||
src[ns] = v[d];
|
||||
|
||||
for (j = 0; j < nr; ++j)
|
||||
S[ns][j] = &raid_gfmulpshufb[A(ip[j], d)][0][0];
|
||||
|
||||
++ns;
|
||||
}
|
||||
|
||||
BUG_ON(k != nr);
|
||||
BUG_ON(ns != nd - nr);
|
||||
|
||||
/* precompute inverse-matrix multiplication table pointers */
|
||||
for (j = 0; j < nr; ++j)
|
||||
for (k = 0; k < nr; ++k)
|
||||
R[j][k] = &raid_gfmulpshufb[V[j * nr + k]][0][0];
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/*
|
||||
* V31 is reused during reconstruction, so reload the nibble mask
|
||||
* at the beginning of every iteration.
|
||||
*/
|
||||
asm volatile ("ldr q31, %0" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
/* start all selected syndromes from the stored parity */
|
||||
asm volatile ("ldr q0, %0" : : "m" (p[0][i]));
|
||||
asm volatile ("ldr q1, %0" : : "m" (p[0][i + 16]));
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("ldr q2, %0" : : "m" (p[1][i]));
|
||||
asm volatile ("ldr q3, %0" : : "m" (p[1][i + 16]));
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("ldr q4, %0" : : "m" (p[2][i]));
|
||||
asm volatile ("ldr q5, %0" : : "m" (p[2][i + 16]));
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("ldr q6, %0" : : "m" (p[3][i]));
|
||||
asm volatile ("ldr q7, %0" : : "m" (p[3][i + 16]));
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("ldr q8, %0" : : "m" (p[4][i]));
|
||||
asm volatile ("ldr q9, %0" : : "m" (p[4][i + 16]));
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("ldr q10, %0" : : "m" (p[5][i]));
|
||||
asm volatile ("ldr q11, %0" : : "m" (p[5][i + 16]));
|
||||
}
|
||||
|
||||
/*
|
||||
* Add all surviving data contributions.
|
||||
* All source reads for this chunk occur before any destination
|
||||
* write.
|
||||
*/
|
||||
for (s = 0; s < ns; ++s) {
|
||||
const uint8_t **t = S[s];
|
||||
|
||||
asm volatile ("ldr q12, %0" : : "m" (src[s][i]));
|
||||
asm volatile ("ldr q13, %0" : : "m" (src[s][i + 16]));
|
||||
|
||||
/*
|
||||
* P has coefficient 1. Use the original source before
|
||||
* destructively splitting it into nibbles.
|
||||
*/
|
||||
if (has_p) {
|
||||
asm volatile ("eor v0.16b, v0.16b, v12.16b");
|
||||
asm volatile ("eor v1.16b, v1.16b, v13.16b");
|
||||
|
||||
/* split the source lanes */
|
||||
asm volatile ("ushr v14.16b, v12.16b, #4");
|
||||
asm volatile ("ushr v15.16b, v13.16b, #4");
|
||||
asm volatile ("and v12.16b, v12.16b, v31.16b");
|
||||
asm volatile ("and v13.16b, v13.16b, v31.16b");
|
||||
asm volatile ("and v14.16b, v14.16b, v31.16b");
|
||||
asm volatile ("and v15.16b, v15.16b, v31.16b");
|
||||
} else {
|
||||
/* split the source lanes */
|
||||
asm volatile ("ushr v14.16b, v12.16b, #4");
|
||||
asm volatile ("ushr v15.16b, v13.16b, #4");
|
||||
asm volatile ("and v12.16b, v12.16b, v31.16b");
|
||||
asm volatile ("and v13.16b, v13.16b, v31.16b");
|
||||
asm volatile ("and v14.16b, v14.16b, v31.16b");
|
||||
asm volatile ("and v15.16b, v15.16b, v31.16b");
|
||||
|
||||
/* syndrome 0 */
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[0][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[0][16]));
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v0.16b, v0.16b, v26.16b");
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v1.16b, v1.16b, v26.16b");
|
||||
}
|
||||
|
||||
/* syndrome 1 */
|
||||
if (nr >= 2) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[1][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[1][16]));
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v2.16b, v2.16b, v26.16b");
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v3.16b, v3.16b, v26.16b");
|
||||
}
|
||||
|
||||
/* syndrome 2 */
|
||||
if (nr >= 3) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[2][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[2][16]));
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v4.16b, v4.16b, v26.16b");
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v5.16b, v5.16b, v26.16b");
|
||||
}
|
||||
|
||||
/* syndrome 3 */
|
||||
if (nr >= 4) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[3][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[3][16]));
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v6.16b, v6.16b, v26.16b");
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v7.16b, v7.16b, v26.16b");
|
||||
}
|
||||
|
||||
/* syndrome 4 */
|
||||
if (nr >= 5) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[4][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[4][16]));
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v8.16b, v8.16b, v26.16b");
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v9.16b, v9.16b, v26.16b");
|
||||
}
|
||||
|
||||
/* syndrome 5 */
|
||||
if (nr >= 6) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[5][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[5][16]));
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v14.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v10.16b, v10.16b, v26.16b");
|
||||
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v13.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v11.16b, v11.16b, v26.16b");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Expand raw syndromes backwards into:
|
||||
*
|
||||
* S0 = v0..v3
|
||||
* S1 = v4..v7
|
||||
* S2 = v8..v11
|
||||
* S3 = v12..v15
|
||||
* S4 = v16..v19
|
||||
* S5 = v20..v23
|
||||
*/
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("ushr v21.16b, v10.16b, #4");
|
||||
asm volatile ("and v20.16b, v10.16b, v31.16b");
|
||||
asm volatile ("and v21.16b, v21.16b, v31.16b");
|
||||
asm volatile ("ushr v23.16b, v11.16b, #4");
|
||||
asm volatile ("and v22.16b, v11.16b, v31.16b");
|
||||
asm volatile ("and v23.16b, v23.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("ushr v17.16b, v8.16b, #4");
|
||||
asm volatile ("and v16.16b, v8.16b, v31.16b");
|
||||
asm volatile ("and v17.16b, v17.16b, v31.16b");
|
||||
asm volatile ("ushr v19.16b, v9.16b, #4");
|
||||
asm volatile ("and v18.16b, v9.16b, v31.16b");
|
||||
asm volatile ("and v19.16b, v19.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("ushr v13.16b, v6.16b, #4");
|
||||
asm volatile ("and v12.16b, v6.16b, v31.16b");
|
||||
asm volatile ("and v13.16b, v13.16b, v31.16b");
|
||||
asm volatile ("ushr v15.16b, v7.16b, #4");
|
||||
asm volatile ("and v14.16b, v7.16b, v31.16b");
|
||||
asm volatile ("and v15.16b, v15.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("ushr v9.16b, v4.16b, #4");
|
||||
asm volatile ("and v8.16b, v4.16b, v31.16b");
|
||||
asm volatile ("and v9.16b, v9.16b, v31.16b");
|
||||
asm volatile ("ushr v11.16b, v5.16b, #4");
|
||||
asm volatile ("and v10.16b, v5.16b, v31.16b");
|
||||
asm volatile ("and v11.16b, v11.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("ushr v5.16b, v2.16b, #4");
|
||||
asm volatile ("and v4.16b, v2.16b, v31.16b");
|
||||
asm volatile ("and v5.16b, v5.16b, v31.16b");
|
||||
asm volatile ("ushr v7.16b, v3.16b, #4");
|
||||
asm volatile ("and v6.16b, v3.16b, v31.16b");
|
||||
asm volatile ("and v7.16b, v7.16b, v31.16b");
|
||||
}
|
||||
|
||||
/*
|
||||
* S0 overlaps its original raw lane registers.
|
||||
* Process raw lane 1 before overwriting v1.
|
||||
*/
|
||||
asm volatile ("ushr v3.16b, v1.16b, #4");
|
||||
asm volatile ("and v2.16b, v1.16b, v31.16b");
|
||||
asm volatile ("and v3.16b, v3.16b, v31.16b");
|
||||
asm volatile ("ushr v1.16b, v0.16b, #4");
|
||||
asm volatile ("and v0.16b, v0.16b, v31.16b");
|
||||
asm volatile ("and v1.16b, v1.16b, v31.16b");
|
||||
|
||||
/*
|
||||
* Reconstruct all missing blocks.
|
||||
*
|
||||
* v24/v25 = low/high tables
|
||||
* v26/v27 = lane-0 low/high accumulators
|
||||
* v28/v29 = lane-1 low/high accumulators
|
||||
* v30/v31 = low/high multiplication temporaries
|
||||
*/
|
||||
for (j = 0; j < nr; ++j) {
|
||||
const uint8_t **t = R[j];
|
||||
|
||||
/* coefficient 0 initializes both lane accumulators */
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[0][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[0][16]));
|
||||
asm volatile ("tbl v26.16b, {v24.16b}, v0.16b");
|
||||
asm volatile ("tbl v27.16b, {v25.16b}, v1.16b");
|
||||
asm volatile ("tbl v28.16b, {v24.16b}, v2.16b");
|
||||
asm volatile ("tbl v29.16b, {v25.16b}, v3.16b");
|
||||
|
||||
if (nr >= 2) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[1][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[1][16]));
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v4.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v5.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v30.16b");
|
||||
asm volatile ("eor v27.16b, v27.16b, v31.16b");
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v6.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v7.16b");
|
||||
asm volatile ("eor v28.16b, v28.16b, v30.16b");
|
||||
asm volatile ("eor v29.16b, v29.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 3) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[2][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[2][16]));
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v8.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v9.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v30.16b");
|
||||
asm volatile ("eor v27.16b, v27.16b, v31.16b");
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v10.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v11.16b");
|
||||
asm volatile ("eor v28.16b, v28.16b, v30.16b");
|
||||
asm volatile ("eor v29.16b, v29.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 4) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[3][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[3][16]));
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v12.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v13.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v30.16b");
|
||||
asm volatile ("eor v27.16b, v27.16b, v31.16b");
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v14.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v15.16b");
|
||||
asm volatile ("eor v28.16b, v28.16b, v30.16b");
|
||||
asm volatile ("eor v29.16b, v29.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 5) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[4][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[4][16]));
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v16.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v17.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v30.16b");
|
||||
asm volatile ("eor v27.16b, v27.16b, v31.16b");
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v18.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v19.16b");
|
||||
asm volatile ("eor v28.16b, v28.16b, v30.16b");
|
||||
asm volatile ("eor v29.16b, v29.16b, v31.16b");
|
||||
}
|
||||
|
||||
if (nr >= 6) {
|
||||
asm volatile ("ldr q24, %0" : : "m" (t[5][0]));
|
||||
asm volatile ("ldr q25, %0" : : "m" (t[5][16]));
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v20.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v21.16b");
|
||||
asm volatile ("eor v26.16b, v26.16b, v30.16b");
|
||||
asm volatile ("eor v27.16b, v27.16b, v31.16b");
|
||||
|
||||
asm volatile ("tbl v30.16b, {v24.16b}, v22.16b");
|
||||
asm volatile ("tbl v31.16b, {v25.16b}, v23.16b");
|
||||
asm volatile ("eor v28.16b, v28.16b, v30.16b");
|
||||
asm volatile ("eor v29.16b, v29.16b, v31.16b");
|
||||
}
|
||||
|
||||
asm volatile ("eor v26.16b, v26.16b, v27.16b");
|
||||
asm volatile ("eor v28.16b, v28.16b, v29.16b");
|
||||
asm volatile ("str q26, %0" : "=m" (pa[j][i]));
|
||||
asm volatile ("str q28, %0" : "=m" (pa[j][i + 16]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_neon_end();
|
||||
}
|
||||
|
||||
void raid_gen2_neon_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon_gen(nd, size, vv, 2);
|
||||
}
|
||||
|
||||
void raid_gen2_neon_aes(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_gen2_neon_gen(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) NEON implementation
|
||||
*/
|
||||
|
|
@ -629,431 +1032,47 @@ void raid_gen6_neon_aes(int nd, size_t size, void **vv)
|
|||
raid_genX_neon(nd, size, vv, 6, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for one disk NEON implementation
|
||||
*/
|
||||
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *pa;
|
||||
uint8_t G;
|
||||
uint8_t V;
|
||||
size_t i;
|
||||
BUG_ON(nr != 1);
|
||||
|
||||
(void)nr; /* unused, it's always 1 */
|
||||
|
||||
/* if it's RAID5 uses the faster function */
|
||||
/* if recovering with P uses the delta function */
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
G = A(ip[0], id[0]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
V = inv(G);
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
/* preload tables */
|
||||
asm volatile (
|
||||
"ldr q28, %0\n" /* low4 */
|
||||
"ldr q24, %1\n" /* v low table */
|
||||
"ldr q25, %2\n" /* v high table */
|
||||
:
|
||||
: "m" (gfconst16.low4[0]),
|
||||
"m" (raid_gfmulpshufb[V][0][0]), "m" (raid_gfmulpshufb[V][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile (
|
||||
"ldr q0, %4\n"
|
||||
"ldr q1, %5\n"
|
||||
"ldr q2, %6\n"
|
||||
"ldr q3, %7\n"
|
||||
|
||||
"ldr q8, %8\n"
|
||||
"ldr q9, %9\n"
|
||||
"ldr q10, %10\n"
|
||||
"ldr q11, %11\n"
|
||||
|
||||
"eor v0.16b, v0.16b, v8.16b\n"
|
||||
"eor v1.16b, v1.16b, v9.16b\n"
|
||||
"eor v2.16b, v2.16b, v10.16b\n"
|
||||
"eor v3.16b, v3.16b, v11.16b\n"
|
||||
|
||||
"ushr v8.16b, v0.16b, #4\n"
|
||||
"ushr v9.16b, v1.16b, #4\n"
|
||||
"ushr v10.16b, v2.16b, #4\n"
|
||||
"ushr v11.16b, v3.16b, #4\n"
|
||||
|
||||
"and v0.16b, v0.16b, v28.16b\n"
|
||||
"and v1.16b, v1.16b, v28.16b\n"
|
||||
"and v2.16b, v2.16b, v28.16b\n"
|
||||
"and v3.16b, v3.16b, v28.16b\n"
|
||||
|
||||
"and v8.16b, v8.16b, v28.16b\n"
|
||||
"and v9.16b, v9.16b, v28.16b\n"
|
||||
"and v10.16b, v10.16b, v28.16b\n"
|
||||
"and v11.16b, v11.16b, v28.16b\n"
|
||||
|
||||
"tbl v12.16b, {v24.16b}, v0.16b\n"
|
||||
"tbl v13.16b, {v24.16b}, v1.16b\n"
|
||||
"tbl v14.16b, {v24.16b}, v2.16b\n"
|
||||
"tbl v15.16b, {v24.16b}, v3.16b\n"
|
||||
|
||||
"tbl v8.16b, {v25.16b}, v8.16b\n"
|
||||
"tbl v9.16b, {v25.16b}, v9.16b\n"
|
||||
"tbl v10.16b, {v25.16b}, v10.16b\n"
|
||||
"tbl v11.16b, {v25.16b}, v11.16b\n"
|
||||
|
||||
"eor v0.16b, v12.16b, v8.16b\n"
|
||||
"eor v1.16b, v13.16b, v9.16b\n"
|
||||
"eor v2.16b, v14.16b, v10.16b\n"
|
||||
"eor v3.16b, v15.16b, v11.16b\n"
|
||||
|
||||
"str q0, %0\n"
|
||||
"str q1, %1\n"
|
||||
"str q2, %2\n"
|
||||
"str q3, %3\n"
|
||||
: "=m" (pa[i]), "=m" (pa[i + 16]), "=m" (pa[i + 32]), "=m" (pa[i + 48])
|
||||
: "m" (p[i]), "m" (p[i + 16]), "m" (p[i + 32]), "m" (p[i + 48]),
|
||||
"m" (pa[i]), "m" (pa[i + 16]), "m" (pa[i + 32]), "m" (pa[i + 48])
|
||||
);
|
||||
}
|
||||
|
||||
raid_neon_end();
|
||||
raid_recX_neon(1, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks NEON implementation
|
||||
*/
|
||||
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[2];
|
||||
uint8_t *pa[2];
|
||||
uint8_t G[2 * 2];
|
||||
uint8_t V[2 * 2];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr; /* unused, it's always 2 */
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, N);
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
/* preload tables */
|
||||
asm volatile (
|
||||
"ldr q28, %0\n" /* low4 */
|
||||
"ldr q20, %1\n" /* v[0] low table */
|
||||
"ldr q21, %2\n" /* v[0] high table */
|
||||
"ldr q22, %3\n" /* v[1] low table */
|
||||
"ldr q23, %4\n" /* v[1] high table */
|
||||
"ldr q24, %5\n" /* v[2] low table */
|
||||
"ldr q25, %6\n" /* v[2] high table */
|
||||
"ldr q26, %7\n" /* v[3] low table */
|
||||
"ldr q27, %8\n" /* v[3] high table */
|
||||
:
|
||||
: "m" (gfconst16.low4[0]),
|
||||
"m" (raid_gfmulpshufb[V[0]][0][0]), "m" (raid_gfmulpshufb[V[0]][1][0]),
|
||||
"m" (raid_gfmulpshufb[V[1]][0][0]), "m" (raid_gfmulpshufb[V[1]][1][0]),
|
||||
"m" (raid_gfmulpshufb[V[2]][0][0]), "m" (raid_gfmulpshufb[V[2]][1][0]),
|
||||
"m" (raid_gfmulpshufb[V[3]][0][0]), "m" (raid_gfmulpshufb[V[3]][1][0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
asm volatile (
|
||||
/* load delta 0 block 0 & 1 */
|
||||
"ldr q0, %4\n"
|
||||
"ldr q2, %5\n"
|
||||
"ldr q8, %8\n"
|
||||
"ldr q9, %9\n"
|
||||
"eor v0.16b, v0.16b, v8.16b\n"
|
||||
"eor v2.16b, v2.16b, v9.16b\n"
|
||||
|
||||
/* load delta 1 block 0 & 1 */
|
||||
"ldr q4, %6\n"
|
||||
"ldr q6, %7\n"
|
||||
"ldr q10, %10\n"
|
||||
"ldr q11, %11\n"
|
||||
"eor v4.16b, v4.16b, v10.16b\n"
|
||||
"eor v6.16b, v6.16b, v11.16b\n"
|
||||
|
||||
/* split delta 0 (v0, v2 -> v0 low, v1 high, v2 low, v3 high) */
|
||||
"ushr v1.16b, v0.16b, #4\n"
|
||||
"ushr v3.16b, v2.16b, #4\n"
|
||||
"and v0.16b, v0.16b, v28.16b\n"
|
||||
"and v1.16b, v1.16b, v28.16b\n"
|
||||
"and v2.16b, v2.16b, v28.16b\n"
|
||||
"and v3.16b, v3.16b, v28.16b\n"
|
||||
|
||||
/* split delta 1 (v4, v6 -> v4 low, v5 high, v6 low, v7 high) */
|
||||
"ushr v5.16b, v4.16b, #4\n"
|
||||
"ushr v7.16b, v6.16b, #4\n"
|
||||
"and v4.16b, v4.16b, v28.16b\n"
|
||||
"and v5.16b, v5.16b, v28.16b\n"
|
||||
"and v6.16b, v6.16b, v28.16b\n"
|
||||
"and v7.16b, v7.16b, v28.16b\n"
|
||||
|
||||
/* reconstruct pa[0] (v8 = block 0, v9 = block 1) */
|
||||
/* V[0] * delta 0 */
|
||||
"tbl v8.16b, {v20.16b}, v0.16b\n"
|
||||
"tbl v9.16b, {v20.16b}, v2.16b\n"
|
||||
"tbl v12.16b, {v21.16b}, v1.16b\n"
|
||||
"tbl v13.16b, {v21.16b}, v3.16b\n"
|
||||
"eor v8.16b, v8.16b, v12.16b\n"
|
||||
"eor v9.16b, v9.16b, v13.16b\n"
|
||||
|
||||
/* V[1] * delta 1 */
|
||||
"tbl v12.16b, {v22.16b}, v4.16b\n"
|
||||
"tbl v13.16b, {v22.16b}, v6.16b\n"
|
||||
"tbl v14.16b, {v23.16b}, v5.16b\n"
|
||||
"tbl v15.16b, {v23.16b}, v7.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
"eor v8.16b, v8.16b, v12.16b\n"
|
||||
"eor v9.16b, v9.16b, v13.16b\n"
|
||||
"str q8, %0\n"
|
||||
"str q9, %1\n"
|
||||
|
||||
/* reconstruct pa[1] (v10 = block 0, v11 = block 1) */
|
||||
/* V[2] * delta 0 */
|
||||
"tbl v10.16b, {v24.16b}, v0.16b\n"
|
||||
"tbl v11.16b, {v24.16b}, v2.16b\n"
|
||||
"tbl v12.16b, {v25.16b}, v1.16b\n"
|
||||
"tbl v13.16b, {v25.16b}, v3.16b\n"
|
||||
"eor v10.16b, v10.16b, v12.16b\n"
|
||||
"eor v11.16b, v11.16b, v13.16b\n"
|
||||
|
||||
/* V[3] * delta 1 */
|
||||
"tbl v12.16b, {v26.16b}, v4.16b\n"
|
||||
"tbl v13.16b, {v26.16b}, v6.16b\n"
|
||||
"tbl v14.16b, {v27.16b}, v5.16b\n"
|
||||
"tbl v15.16b, {v27.16b}, v7.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
"eor v10.16b, v10.16b, v12.16b\n"
|
||||
"eor v11.16b, v11.16b, v13.16b\n"
|
||||
"str q10, %2\n"
|
||||
"str q11, %3\n"
|
||||
: "=m" (pa[0][i]), "=m" (pa[0][i + 16]),
|
||||
"=m" (pa[1][i]), "=m" (pa[1][i + 16])
|
||||
: "m" (p[0][i]), "m" (p[0][i + 16]),
|
||||
"m" (p[1][i]), "m" (p[1][i + 16]),
|
||||
"m" (pa[0][i]), "m" (pa[0][i + 16]),
|
||||
"m" (pa[1][i]), "m" (pa[1][i + 16])
|
||||
);
|
||||
}
|
||||
|
||||
raid_neon_end();
|
||||
BUG_ON(nr != 2);
|
||||
raid_recX_neon(2, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering NEON implementation
|
||||
*/
|
||||
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_rec3_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
const uint8_t *T[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
size_t i;
|
||||
int j, k;
|
||||
BUG_ON(nr != 3);
|
||||
raid_recX_neon(3, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/* setup the coefficients matrix */
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
void raid_rec4_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 4);
|
||||
raid_recX_neon(4, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/* invert it to solve the system of linear equations */
|
||||
raid_invert(G, V, N);
|
||||
void raid_rec5_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 5);
|
||||
raid_recX_neon(5, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
/* precompute shuffle table pointers */
|
||||
for (j = 0; j < N * N; ++j)
|
||||
T[j] = &raid_gfmulpshufb[V[j]][0][0];
|
||||
|
||||
/* compute delta parity */
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_neon_begin();
|
||||
|
||||
/* preload mask */
|
||||
asm volatile (
|
||||
"ldr q28, %0\n" /* low4 */
|
||||
:
|
||||
: "m" (gfconst16.low4[0])
|
||||
);
|
||||
|
||||
for (i = 0; i < size; i += 16) {
|
||||
/* delta */
|
||||
asm volatile (
|
||||
"ldr x4, [%2, #0]\n"
|
||||
"ldr x5, [%3, #0]\n"
|
||||
"ldr q0, [x4, %1]\n"
|
||||
"ldr q14, [x5, %1]\n"
|
||||
"eor v0.16b, v0.16b, v14.16b\n"
|
||||
"ushr v1.16b, v0.16b, #4\n"
|
||||
"and v0.16b, v0.16b, v28.16b\n"
|
||||
"and v1.16b, v1.16b, v28.16b\n"
|
||||
"cmp %0, #1\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x4, [%2, #8]\n"
|
||||
"ldr x5, [%3, #8]\n"
|
||||
"ldr q2, [x4, %1]\n"
|
||||
"ldr q14, [x5, %1]\n"
|
||||
"eor v2.16b, v2.16b, v14.16b\n"
|
||||
"ushr v3.16b, v2.16b, #4\n"
|
||||
"and v2.16b, v2.16b, v28.16b\n"
|
||||
"and v3.16b, v3.16b, v28.16b\n"
|
||||
"cmp %0, #2\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x4, [%2, #16]\n"
|
||||
"ldr x5, [%3, #16]\n"
|
||||
"ldr q4, [x4, %1]\n"
|
||||
"ldr q14, [x5, %1]\n"
|
||||
"eor v4.16b, v4.16b, v14.16b\n"
|
||||
"ushr v5.16b, v4.16b, #4\n"
|
||||
"and v4.16b, v4.16b, v28.16b\n"
|
||||
"and v5.16b, v5.16b, v28.16b\n"
|
||||
"cmp %0, #3\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x4, [%2, #24]\n"
|
||||
"ldr x5, [%3, #24]\n"
|
||||
"ldr q6, [x4, %1]\n"
|
||||
"ldr q14, [x5, %1]\n"
|
||||
"eor v6.16b, v6.16b, v14.16b\n"
|
||||
"ushr v7.16b, v6.16b, #4\n"
|
||||
"and v6.16b, v6.16b, v28.16b\n"
|
||||
"and v7.16b, v7.16b, v28.16b\n"
|
||||
"cmp %0, #4\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x4, [%2, #32]\n"
|
||||
"ldr x5, [%3, #32]\n"
|
||||
"ldr q8, [x4, %1]\n"
|
||||
"ldr q14, [x5, %1]\n"
|
||||
"eor v8.16b, v8.16b, v14.16b\n"
|
||||
"ushr v9.16b, v8.16b, #4\n"
|
||||
"and v8.16b, v8.16b, v28.16b\n"
|
||||
"and v9.16b, v9.16b, v28.16b\n"
|
||||
"cmp %0, #5\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x4, [%2, #40]\n"
|
||||
"ldr x5, [%3, #40]\n"
|
||||
"ldr q10, [x4, %1]\n"
|
||||
"ldr q14, [x5, %1]\n"
|
||||
"eor v10.16b, v10.16b, v14.16b\n"
|
||||
"ushr v11.16b, v10.16b, #4\n"
|
||||
"and v10.16b, v10.16b, v28.16b\n"
|
||||
"and v11.16b, v11.16b, v28.16b\n"
|
||||
|
||||
"1:\n"
|
||||
:
|
||||
: "r" ((uint64_t)N), "r" (i), "r" (p), "r" (pa)
|
||||
: "x4", "x5", "cc", "memory"
|
||||
);
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile (
|
||||
"ldr x5, [%2, #0]\n"
|
||||
"ldp q24, q25, [x5]\n"
|
||||
"tbl v12.16b, {v24.16b}, v0.16b\n"
|
||||
"tbl v13.16b, {v25.16b}, v1.16b\n"
|
||||
"cmp %0, #1\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x5, [%2, #8]\n"
|
||||
"ldp q24, q25, [x5]\n"
|
||||
"tbl v14.16b, {v24.16b}, v2.16b\n"
|
||||
"tbl v15.16b, {v25.16b}, v3.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
"cmp %0, #2\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x5, [%2, #16]\n"
|
||||
"ldp q24, q25, [x5]\n"
|
||||
"tbl v14.16b, {v24.16b}, v4.16b\n"
|
||||
"tbl v15.16b, {v25.16b}, v5.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
"cmp %0, #3\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x5, [%2, #24]\n"
|
||||
"ldp q24, q25, [x5]\n"
|
||||
"tbl v14.16b, {v24.16b}, v6.16b\n"
|
||||
"tbl v15.16b, {v25.16b}, v7.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
"cmp %0, #4\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x5, [%2, #32]\n"
|
||||
"ldp q24, q25, [x5]\n"
|
||||
"tbl v14.16b, {v24.16b}, v8.16b\n"
|
||||
"tbl v15.16b, {v25.16b}, v9.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
"cmp %0, #5\n"
|
||||
"b.ls 1f\n"
|
||||
|
||||
"ldr x5, [%2, #40]\n"
|
||||
"ldp q24, q25, [x5]\n"
|
||||
"tbl v14.16b, {v24.16b}, v10.16b\n"
|
||||
"tbl v15.16b, {v25.16b}, v11.16b\n"
|
||||
"eor v12.16b, v12.16b, v14.16b\n"
|
||||
"eor v13.16b, v13.16b, v15.16b\n"
|
||||
|
||||
"1:\n"
|
||||
"eor v12.16b, v12.16b, v13.16b\n"
|
||||
"str q12, [%3, %1]\n"
|
||||
:
|
||||
: "r" ((uint64_t)N), "r" (i), "r" (&T[j * N]), "r" (pa[j])
|
||||
: "x4", "x5", "cc", "memory"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
raid_neon_end();
|
||||
void raid_rec6_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
BUG_ON(nr != 6);
|
||||
raid_recX_neon(6, id, ip, nd, size, vv);
|
||||
}
|
||||
|
||||
void raid_register_neon(void)
|
||||
|
|
@ -1073,9 +1092,10 @@ void raid_register_neon(void)
|
|||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_recX_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_recX_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_recX_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_recX_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_rec3_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_rec4_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_rec5_neon, RAID_POLY_ANY);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_rec6_neon, RAID_POLY_ANY);
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
1004
raid/neon32.c
1004
raid/neon32.c
File diff suppressed because it is too large
Load diff
2666
raid/ssse3.c
2666
raid/ssse3.c
File diff suppressed because it is too large
Load diff
653
raid/test.c
653
raid/test.c
|
|
@ -70,6 +70,19 @@ int raid_test_gfaffine(void)
|
|||
if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x])
|
||||
return -1;
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
int d, p;
|
||||
|
||||
for (d = 0; d < RAID_DATA_MAX; ++d) {
|
||||
for (p = 1; p < RAID_PARITY_MAX; ++p) {
|
||||
if (memcmp(raid_gfcauchyaffine_raid[d][p - 1],
|
||||
raid_gfaffine_raid[raid_gfcauchy_raid[p][d]],
|
||||
8) != 0)
|
||||
return -1;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
|
@ -391,101 +404,245 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
|
||||
/* setup recov functions */
|
||||
for (i = 1; i <= np; ++i) {
|
||||
if (i == 1) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_int8;
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
test_setup(i) = raid_rec1_ssse3;
|
||||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_rec1_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec1_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec1_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec1_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec1_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
test_setup(i) = raid_rec1_neon;
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
test_setup(i) = raid_rec1_neon32;
|
||||
#endif
|
||||
} else if (i == 2) {
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_int8;
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
test_setup(i) = raid_rec2_ssse3;
|
||||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_rec2_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
test_setup(i) = raid_rec2_avx512bw;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec2_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec2_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec2_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec2_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
test_setup(i) = raid_rec2_neon;
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
test_setup(i) = raid_rec2_neon32;
|
||||
#endif
|
||||
} else {
|
||||
break;
|
||||
default:
|
||||
test_setup(i) = raid_recX_int8;
|
||||
break;
|
||||
}
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
test_setup(i) = raid_recX_ssse3;
|
||||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_recX_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
test_setup(i) = raid_recX_avx512bw;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_recX_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_recX_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_recX_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_recX_avx512gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_ssse3;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_ssse3;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_ssse3;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_ssse3;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_ssse3;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_ssse3;
|
||||
break;
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_ssse3ext;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_ssse3ext;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_ssse3ext;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_ssse3ext;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_ssse3ext;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_ssse3ext;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
test_setup(i) = raid_recX_neon;
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
test_setup(i) = raid_recX_neon32;
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx2;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx2;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx2;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx2;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx2;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx512bw;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx512bw;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx512bw;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx512bw;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx512bw;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx512bw;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx2gfni_aes;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx2gfni_aes;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx2gfni_aes;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx2gfni_aes;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx2gfni_aes;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx2gfni_aes;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx2gfni_raid;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx2gfni_raid;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx2gfni_raid;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx2gfni_raid;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx2gfni_raid;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx2gfni_raid;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx512gfni_aes;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx512gfni_aes;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx512gfni_aes;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx512gfni_aes;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx512gfni_aes;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx512gfni_aes;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx512gfni_raid;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx512gfni_raid;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx512gfni_raid;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx512gfni_raid;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx512gfni_raid;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx512gfni_raid;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_neon;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_neon;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_neon;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_neon;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_neon;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_neon;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_neon32;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_neon32;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_neon32;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_neon32;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_neon32;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_neon32;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/* compute the parity */
|
||||
|
|
@ -607,101 +764,245 @@ int raid_test_tail(int mode, int nd_max, size_t size)
|
|||
|
||||
/* setup recov functions */
|
||||
for (i = 1; i <= np; ++i) {
|
||||
if (i == 1) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_int8;
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
test_setup(i) = raid_rec1_ssse3;
|
||||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_rec1_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec1_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec1_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec1_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec1_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
test_setup(i) = raid_rec1_neon;
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
test_setup(i) = raid_rec1_neon32;
|
||||
#endif
|
||||
} else if (i == 2) {
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_int8;
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
test_setup(i) = raid_rec2_ssse3;
|
||||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_rec2_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
test_setup(i) = raid_rec2_avx512bw;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec2_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec2_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec2_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec2_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
test_setup(i) = raid_rec2_neon;
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
test_setup(i) = raid_rec2_neon32;
|
||||
#endif
|
||||
} else {
|
||||
break;
|
||||
default:
|
||||
test_setup(i) = raid_recX_int8;
|
||||
break;
|
||||
}
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
test_setup(i) = raid_recX_ssse3;
|
||||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_recX_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
test_setup(i) = raid_recX_avx512bw;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_recX_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_recX_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_recX_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_recX_avx512gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_ssse3;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_ssse3;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_ssse3;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_ssse3;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_ssse3;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_ssse3;
|
||||
break;
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_ssse3ext;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_ssse3ext;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_ssse3ext;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_ssse3ext;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_ssse3ext;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_ssse3ext;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
test_setup(i) = raid_recX_neon;
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
test_setup(i) = raid_recX_neon32;
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx2;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx2;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx2;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx2;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx2;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx2;
|
||||
break;
|
||||
}
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx512bw;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx512bw;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx512bw;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx512bw;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx512bw;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx512bw;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx2gfni_aes;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx2gfni_aes;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx2gfni_aes;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx2gfni_aes;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx2gfni_aes;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx2gfni_aes;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx2gfni_raid;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx2gfni_raid;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx2gfni_raid;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx2gfni_raid;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx2gfni_raid;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx2gfni_raid;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx512gfni_aes;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx512gfni_aes;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx512gfni_aes;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx512gfni_aes;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx512gfni_aes;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx512gfni_aes;
|
||||
break;
|
||||
}
|
||||
} else {
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_avx512gfni_raid;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_avx512gfni_raid;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_avx512gfni_raid;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_avx512gfni_raid;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_avx512gfni_raid;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_avx512gfni_raid;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_NEON
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_neon;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_neon;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_neon;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_neon;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_neon;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_neon;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
switch (i) {
|
||||
case 1:
|
||||
test_setup(i) = raid_rec1_neon32;
|
||||
break;
|
||||
case 2:
|
||||
test_setup(i) = raid_rec2_neon32;
|
||||
break;
|
||||
case 3:
|
||||
test_setup(i) = raid_rec3_neon32;
|
||||
break;
|
||||
case 4:
|
||||
test_setup(i) = raid_rec4_neon32;
|
||||
break;
|
||||
case 5:
|
||||
test_setup(i) = raid_rec5_neon32;
|
||||
break;
|
||||
case 6:
|
||||
test_setup(i) = raid_rec6_neon32;
|
||||
break;
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
/* test all disk counts from 1 to nd_max */
|
||||
|
|
|
|||
|
|
@ -103,6 +103,11 @@ sdecovtest:
|
|||
|
||||
# Use this target to run a complete test on Ice Lake simulator
|
||||
sdetest:
|
||||
$(SDE) -p4p -- ./selftest
|
||||
$(SDE) -mrm -- ./selftest
|
||||
$(SDE) -nhm -- ./selftest
|
||||
$(SDE) -hsw -- ./selftest
|
||||
$(SDE) -skx -- ./selftest
|
||||
$(SDE) -icl -- ./selftest
|
||||
$(SDE) -icl -- ./fulltest
|
||||
|
||||
|
|
|
|||
|
|
@ -1122,10 +1122,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
printf("%8s", "neon32");
|
||||
#endif
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3())
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
printf("%8s", "ssse3");
|
||||
if (raid_cpu_has_avx2())
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "ssse3e");
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx2e");
|
||||
#endif
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
printf("%8s", "avx512");
|
||||
|
|
@ -1172,12 +1180,24 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
/* raid_rec1of1() internally calls raid_gen() */
|
||||
raid_gen_force(1, raid_gen1_sse2);
|
||||
raid_rec1_ssse3(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* raid_rec1of1() internally calls raid_gen() */
|
||||
raid_gen_force(1, raid_gen1_sse2);
|
||||
raid_rec1_ssse3ext(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1189,6 +1209,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
/*
|
||||
* raid_rec1_avx2ext() uses raid_rec1of1() for P,
|
||||
* which internally calls raid_gen().
|
||||
*/
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
raid_rec1_avx2ext(1, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw);
|
||||
|
|
@ -1236,7 +1269,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_neon);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_neon(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1246,7 +1278,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_neon32);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_neon32(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1258,6 +1289,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_sse2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_ssse3(1, id, ip + 1, nd, size, v);
|
||||
|
|
@ -1265,6 +1297,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_sse2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_ssse3ext(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1277,10 +1321,20 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx2ext(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
/* +1 to avoid RAID5 optimized case */
|
||||
raid_rec1_avx512bw(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1289,8 +1343,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
/* +1 to avoid RAID5 optimized case */
|
||||
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1299,8 +1352,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
/* +1 to avoid RAID5 optimized case */
|
||||
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1311,6 +1363,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
printf("\n");
|
||||
|
||||
/* REC1OF3 */
|
||||
printf("%8s", "rec1of3");
|
||||
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR1));
|
||||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
raid_gen_force(1, sizeof(void *) == 8 ? raid_gen1_int64 : raid_gen1_int32);
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_int8(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_neon(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_neon32(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_sse2);
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_ssse3(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_sse2);
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_ssse3ext(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_avx2(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_avx2ext(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_avx512bw(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_avx2gfni_raid(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* +2 to select R instead of Q */
|
||||
raid_rec1_avx512gfni_raid(1, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
/* REC2OF2 */
|
||||
printf("%8s", "rec2of2");
|
||||
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
|
||||
|
|
@ -1326,7 +1488,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_neon_raid);
|
||||
raid_rec2_neon(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1335,7 +1496,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_neon32_raid);
|
||||
raid_rec2_neon32(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1346,24 +1506,28 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(2, raid_gen2_sse2ext_raid);
|
||||
#else
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_sse2_raid);
|
||||
#endif
|
||||
raid_rec2_ssse3(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_sse2ext_raid);
|
||||
raid_rec2_ssse3ext(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(2, raid_gen2_avx2ext_raid);
|
||||
#else
|
||||
raid_gen_force(2, raid_gen2_avx2_raid);
|
||||
#endif
|
||||
raid_rec2_avx2(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1371,9 +1535,18 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx2ext_raid);
|
||||
raid_rec2_avx2ext(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx512bw);
|
||||
raid_rec2_avx512bw(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1382,7 +1555,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx2gfni_raid);
|
||||
raid_rec2_avx2gfni_raid(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1391,7 +1563,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx512gfni_raid);
|
||||
raid_rec2_avx512gfni_raid(2, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1418,7 +1589,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_neon_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_neon(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1428,7 +1598,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_neon32_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_neon32(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1440,25 +1609,30 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(2, raid_gen2_sse2ext_raid);
|
||||
#else
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_sse2_raid);
|
||||
#endif
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_ssse3(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_sse2ext_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_ssse3ext(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(2, raid_gen2_avx2ext_raid);
|
||||
#else
|
||||
raid_gen_force(2, raid_gen2_avx2_raid);
|
||||
#endif
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1467,9 +1641,19 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx2ext_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2ext(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx512bw);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx512bw(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1479,7 +1663,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx2gfni_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1489,7 +1672,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx512gfni_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -1501,6 +1683,116 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
printf("\n");
|
||||
|
||||
/* REC2OF4 */
|
||||
printf("%8s", "rec2of4");
|
||||
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR2));
|
||||
fflush(stdout);
|
||||
|
||||
SPEED_START {
|
||||
raid_gen_force(2, sizeof(void *) == 8 ? raid_gen2_int64_raid : raid_gen2_int32_raid);
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_int8(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_neon(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_neon32(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_sse2_raid);
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_ssse3(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_sse2ext_raid);
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_ssse3ext(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx2_raid);
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_avx2(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx2ext_raid);
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_avx2ext(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_avx512bw(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_avx2gfni_raid(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* +2 to select R,S instead of Q,R */
|
||||
raid_rec2_avx512gfni_raid(2, id, ip + 2, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
/* REC3OF3 */
|
||||
printf("%8s", "rec3of3");
|
||||
printf("%8s", raid_rec_tag(RAID_ALGO_CAUCHY_PAR3));
|
||||
|
|
@ -1516,8 +1808,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_neon_raid);
|
||||
raid_recX_neon(3, id, ip, nd, size, v);
|
||||
raid_rec3_neon(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1525,8 +1816,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_neon32_raid);
|
||||
raid_recX_neon32(3, id, ip, nd, size, v);
|
||||
raid_rec3_neon32(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1536,36 +1826,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(3, raid_gen3_ssse3ext_raid);
|
||||
#else
|
||||
raid_gen_force(3, raid_gen3_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_ssse3(3, id, ip, nd, size, v);
|
||||
raid_rec3_ssse3(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
raid_rec3_ssse3ext(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(3, raid_gen3_avx2ext_raid);
|
||||
#else
|
||||
raid_gen_force(3, raid_gen3_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_avx2(3, id, ip, nd, size, v);
|
||||
raid_rec3_avx2(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_rec3_avx2ext(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx512bw);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512bw(3, id, ip, nd, size, v);
|
||||
raid_rec3_avx512bw(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1573,8 +1868,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
|
||||
raid_rec3_avx2gfni_raid(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1582,8 +1876,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
|
||||
raid_rec3_avx512gfni_raid(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1608,8 +1901,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_neon_raid);
|
||||
raid_recX_neon(4, id, ip, nd, size, v);
|
||||
raid_rec4_neon(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1617,8 +1909,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_neon32_raid);
|
||||
raid_recX_neon32(4, id, ip, nd, size, v);
|
||||
raid_rec4_neon32(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1628,36 +1919,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(4, raid_gen4_ssse3ext_raid);
|
||||
#else
|
||||
raid_gen_force(4, raid_gen4_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_ssse3(4, id, ip, nd, size, v);
|
||||
raid_rec4_ssse3(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
raid_rec4_ssse3ext(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(4, raid_gen4_avx2ext_raid);
|
||||
#else
|
||||
raid_gen_force(4, raid_gen4_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_avx2(4, id, ip, nd, size, v);
|
||||
raid_rec4_avx2(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_rec4_avx2ext(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx512bw);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512bw(4, id, ip, nd, size, v);
|
||||
raid_rec4_avx512bw(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1665,8 +1961,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
|
||||
raid_rec4_avx2gfni_raid(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1674,8 +1969,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
|
||||
raid_rec4_avx512gfni_raid(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1700,8 +1994,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_neon_raid);
|
||||
raid_recX_neon(5, id, ip, nd, size, v);
|
||||
raid_rec5_neon(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1709,8 +2002,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_neon32_raid);
|
||||
raid_recX_neon32(5, id, ip, nd, size, v);
|
||||
raid_rec5_neon32(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1720,36 +2012,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(5, raid_gen5_ssse3ext_raid);
|
||||
#else
|
||||
raid_gen_force(5, raid_gen5_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_ssse3(5, id, ip, nd, size, v);
|
||||
raid_rec5_ssse3(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
raid_rec5_ssse3ext(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(5, raid_gen5_avx2ext_raid);
|
||||
#else
|
||||
raid_gen_force(5, raid_gen5_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_avx2(5, id, ip, nd, size, v);
|
||||
raid_rec5_avx2(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_rec5_avx2ext(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx512bw);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512bw(5, id, ip, nd, size, v);
|
||||
raid_rec5_avx512bw(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1757,8 +2054,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
|
||||
raid_rec5_avx2gfni_raid(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1766,8 +2062,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
|
||||
raid_rec5_avx512gfni_raid(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1792,8 +2087,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
|
||||
#ifdef CONFIG_NEON
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_neon_raid);
|
||||
raid_recX_neon(6, id, ip, nd, size, v);
|
||||
raid_rec6_neon(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1801,8 +2095,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#endif
|
||||
#ifdef CONFIG_NEON32
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_neon32_raid);
|
||||
raid_recX_neon32(6, id, ip, nd, size, v);
|
||||
raid_rec6_neon32(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1812,36 +2105,41 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
#ifdef CONFIG_X86
|
||||
if (raid_cpu_has_ssse3()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(6, raid_gen6_ssse3ext_raid);
|
||||
#else
|
||||
raid_gen_force(6, raid_gen6_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_ssse3(6, id, ip, nd, size, v);
|
||||
raid_rec6_ssse3(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
SPEED_START {
|
||||
raid_rec6_ssse3ext(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen_force(6, raid_gen6_avx2ext_raid);
|
||||
#else
|
||||
raid_gen_force(6, raid_gen6_ssse3_raid);
|
||||
#endif
|
||||
raid_recX_avx2(6, id, ip, nd, size, v);
|
||||
raid_rec6_avx2(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_rec6_avx2ext(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx512bw);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512bw(6, id, ip, nd, size, v);
|
||||
raid_rec6_avx512bw(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1849,8 +2147,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
|
||||
raid_rec6_avx2gfni_raid(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1858,8 +2155,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
|
||||
raid_rec6_avx512gfni_raid(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue