mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-25 18:40:19 -04:00
Add GFNI affine parity generation for RAID polynomial
This commit is contained in:
Andrea Mazzoleni
parent
b5d507fbe9
commit
17591ad932
13 changed files with 1523 additions and 142 deletions
|
|
@ -1151,6 +1151,12 @@ void selftest(void)
|
|||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (raid_test_gfaffine() != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed GFNI AFFINE test\n");
|
||||
exit(EXIT_FAILURE);
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
if (raid_test_par(RAID_MODE_VANDERMONDE_RAID, 32, 256) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
log_fatal(EINTERNAL, "Failed GEN Vandermonde RAID test\n");
|
||||
|
|
|
|||
108
cmdline/speed.c
108
cmdline/speed.c
|
|
@ -306,6 +306,10 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
printf("%8s", "");
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -449,7 +453,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen2_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -461,7 +469,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen2_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -570,7 +582,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen3_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -582,7 +598,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen3_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -691,7 +711,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen4_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -703,7 +727,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen4_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -809,7 +837,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen5_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -821,7 +853,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen5_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -927,7 +963,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen6_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -939,7 +979,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen6_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -1168,7 +1212,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v);
|
||||
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1179,7 +1223,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v);
|
||||
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1261,9 +1305,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx2gfni_aes);
|
||||
raid_gen_force(2, raid_gen2_avx2gfni_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v);
|
||||
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1272,9 +1316,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx512gfni_aes);
|
||||
raid_gen_force(2, raid_gen2_avx512gfni_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v);
|
||||
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1352,8 +1396,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(3, raid_gen3_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(3, id, ip, nd, size, v);
|
||||
raid_gen_force(3, raid_gen3_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1362,8 +1406,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(3, raid_gen3_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(3, id, ip, nd, size, v);
|
||||
raid_gen_force(3, raid_gen3_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1442,8 +1486,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(4, raid_gen4_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(4, id, ip, nd, size, v);
|
||||
raid_gen_force(4, raid_gen4_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1452,8 +1496,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(4, raid_gen4_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(4, id, ip, nd, size, v);
|
||||
raid_gen_force(4, raid_gen4_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1531,8 +1575,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(5, raid_gen5_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(5, id, ip, nd, size, v);
|
||||
raid_gen_force(5, raid_gen5_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1541,8 +1585,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(5, raid_gen5_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(5, id, ip, nd, size, v);
|
||||
raid_gen_force(5, raid_gen5_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1620,8 +1664,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(6, raid_gen6_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(6, id, ip, nd, size, v);
|
||||
raid_gen_force(6, raid_gen6_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1630,8 +1674,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period)
|
|||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(6, raid_gen6_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(6, id, ip, nd, size, v);
|
||||
raid_gen_force(6, raid_gen6_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
|
|||
20
raid/README
20
raid/README
|
|
@ -90,7 +90,9 @@ Operating Modes and Polynomials
|
|||
Uses Cauchy matrix generator with standard RAID polynomial 0x1d
|
||||
(x^8 + x^4 + x^3 + x^2 + 1, decimal 285, 0x11d) and primitive generator
|
||||
g = 2. Supports parities 1 to 6. Matches standard Linux RAID-6 (Anvin
|
||||
method) for 2-parity arrays and ensures on-disk compatibility.
|
||||
method) for 2-parity arrays and ensures on-disk compatibility. On CPUs
|
||||
supporting Intel GFNI, generation and recovery are accelerated using
|
||||
VGF2P8AFFINEQB with precomputed affine matrices for the RAID field.
|
||||
|
||||
RAID_MODE_VANDERMONDE_RAID (Mode = 1)
|
||||
Legacy mode supporting up to 3 parities using standard polynomial 0x1d.
|
||||
|
|
@ -99,12 +101,16 @@ Operating Modes and Polynomials
|
|||
|
||||
RAID_MODE_CAUCHY_AES (Mode = 2)
|
||||
Uses Cauchy matrix with AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1,
|
||||
0x11b) and primitive generator g = 3. Enables hardware acceleration via
|
||||
Intel GFNI instructions (vgf2p8mulb) on supported CPUs (Ice Lake or newer).
|
||||
Parity data is incompatible with standard 0x1d polynomial arrays.
|
||||
0x11b) and primitive generator g = 3. On CPUs supporting Intel GFNI,
|
||||
generation and recovery are accelerated directly using VGF2P8MULB.
|
||||
This native GFNI multiplication is typically faster than the affine GFNI
|
||||
implementation required by the standard RAID polynomial. Parity data is
|
||||
incompatible with standard 0x11d polynomial arrays.
|
||||
|
||||
Both Cauchy modes use the same Extended Cauchy construction with coefficients
|
||||
based on powers of their respective primitive generator.
|
||||
based on powers of their respective primitive generator. GFNI acceleration
|
||||
is available for both fields: the RAID field uses affine transformations,
|
||||
while the AES field uses the native GFNI multiplication instruction.
|
||||
|
||||
|
||||
Implementation Structure
|
||||
|
|
@ -115,7 +121,7 @@ Implementation Structure
|
|||
module.c Algorithm registration, CPU detection, and dynamic dispatch.
|
||||
int.c Portable C fallback implementation (8/32/64-bit word loops).
|
||||
tables.c Precomputed GF(2^8) log/exp, multiplication, Cauchy tables.
|
||||
mktables.c Table generator script for GF(2^8) field matrices.
|
||||
mktables.c Generator for GF(2^8), Cauchy, and GFNI affine tables.
|
||||
internal.h Private header, assertion macros (BUG_ON), alignment helpers.
|
||||
|
||||
Platform-Specific Acceleration:
|
||||
|
|
@ -123,7 +129,7 @@ Implementation Structure
|
|||
ssse3.c x86 SSSE3 (128-bit pshufb nibble-split multiplication)
|
||||
avx2.c x86 AVX2 (256-bit vectorization)
|
||||
avx512.c x86 AVX-512BW (512-bit vectorization)
|
||||
gfni.c x86 GFNI hardware Galois field vector multiplication
|
||||
gfni.c x86 GFNI generation/recovery (RAID affine and AES multiply)
|
||||
neon.c ARM NEON (128-bit SIMD vectorization)
|
||||
|
||||
|
||||
|
|
|
|||
886
raid/gfni.c
886
raid/gfni.c
|
|
@ -6,6 +6,385 @@
|
|||
#include "cpu.h"
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GENX AVX2 GFNI implementation for the RAID polynomial with Horner evaluation.
|
||||
*
|
||||
* Q is evaluated with Horner's rule using the fixed multiply-by-2 affine
|
||||
* matrix. The remaining Cauchy parities use their per-disk coefficients.
|
||||
*
|
||||
* Only supported up to GEN5 (np <= 5), where ymm11 remains available for the
|
||||
* fixed Q multiply-by-2 matrix.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx2gfni_horner_raid(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
size_t i;
|
||||
int d, l;
|
||||
|
||||
if (nd == 1) {
|
||||
for (d = 0; d < np; ++d)
|
||||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
l = nd - 1;
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
/* Fixed GF(2^8) multiply-by-2 matrix used by the Q Horner recurrence. */
|
||||
asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[2][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* Start P and Q from the last data disk, as in Horner evaluation. */
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[l][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %ymm12,%ymm0");
|
||||
asm volatile ("vmovdqa %ymm13,%ymm1");
|
||||
asm volatile ("vmovdqa %ymm12,%ymm2");
|
||||
asm volatile ("vmovdqa %ymm13,%ymm3");
|
||||
|
||||
/*
|
||||
* Initialize R/S/T with the contribution of the last disk.
|
||||
* Unlike P and Q, their coefficients are not powers forming a
|
||||
* fixed Horner recurrence.
|
||||
*/
|
||||
if (np >= 3) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm5");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm7");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm8");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm9");
|
||||
}
|
||||
|
||||
/* Process all remaining disks except D0. */
|
||||
for (d = l - 1; d >= 1; --d) {
|
||||
/* Q = 2 * Q. */
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm2");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32]));
|
||||
|
||||
/* P ^= D[d], Q ^= D[d]. */
|
||||
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
|
||||
|
||||
if (np >= 3) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Final Horner step with D0.
|
||||
*
|
||||
* All Cauchy rows are normalized to coefficient 1 for D0,
|
||||
* so R/S/T require only XOR and no affine multiplication.
|
||||
*/
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm2");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
|
||||
|
||||
if (np >= 3) {
|
||||
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vpxor %ymm12,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm13,%ymm7,%ymm7");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
}
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i]));
|
||||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i]));
|
||||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32]));
|
||||
if (np >= 3) {
|
||||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i]));
|
||||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32]));
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i]));
|
||||
asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32]));
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i]));
|
||||
asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GENX AVX2 GFNI implementation
|
||||
*/
|
||||
static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
size_t i;
|
||||
int d;
|
||||
|
||||
if (nd == 1) {
|
||||
for (d = 0; d < np; ++d)
|
||||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %ymm0,%ymm2");
|
||||
asm volatile ("vmovdqa %ymm1,%ymm3");
|
||||
if (np >= 3) {
|
||||
asm volatile ("vmovdqa %ymm0,%ymm4");
|
||||
asm volatile ("vmovdqa %ymm1,%ymm5");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vmovdqa %ymm0,%ymm6");
|
||||
asm volatile ("vmovdqa %ymm1,%ymm7");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vmovdqa %ymm0,%ymm8");
|
||||
asm volatile ("vmovdqa %ymm1,%ymm9");
|
||||
}
|
||||
if (np >= 6) {
|
||||
asm volatile ("vmovdqa %ymm0,%ymm10");
|
||||
asm volatile ("vmovdqa %ymm1,%ymm11");
|
||||
}
|
||||
|
||||
/* all other disks */
|
||||
for (d = 1; d < nd; ++d) {
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[1][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
|
||||
if (np >= 3) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
|
||||
}
|
||||
if (np >= 6) {
|
||||
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
|
||||
}
|
||||
}
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i]));
|
||||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i]));
|
||||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32]));
|
||||
if (np >= 3) {
|
||||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i]));
|
||||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32]));
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i]));
|
||||
asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32]));
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i]));
|
||||
asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32]));
|
||||
}
|
||||
if (np >= 6) {
|
||||
asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i]));
|
||||
asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GENX AVX512 GFNI implementation for the RAID polynomial.
|
||||
*
|
||||
* Q is evaluated with Horner's rule using the fixed multiply-by-2 affine
|
||||
* matrix. The remaining Cauchy parities use their per-disk coefficients.
|
||||
*/
|
||||
static __always_inline void raid_genX_avx512gfni_horner_raid(int nd, size_t size, void **vv, int np)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
size_t i;
|
||||
int d, l;
|
||||
|
||||
if (nd == 1) {
|
||||
for (d = 0; d < np; ++d)
|
||||
memcpy(v[1 + d], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
l = nd - 1;
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
/* Fixed GF(2^8) multiply-by-2 matrix used by the Q Horner recurrence. */
|
||||
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfaffine_raid[2][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* Start P and Q from the last data disk. */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa64 %zmm6,%zmm0");
|
||||
asm volatile ("vmovdqa64 %zmm6,%zmm1");
|
||||
|
||||
/* Initialize R/S/T/U with the contribution of the last disk. */
|
||||
if (np >= 3) {
|
||||
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm6,%zmm2");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm6,%zmm3");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm6,%zmm4");
|
||||
}
|
||||
if (np >= 6) {
|
||||
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][l]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm6,%zmm5");
|
||||
}
|
||||
|
||||
/* Process all remaining disks except D0. */
|
||||
for (d = l - 1; d >= 1; --d) {
|
||||
/* Q = 2 * Q. */
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm1,%zmm1");
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
|
||||
|
||||
/* P ^= D[d], Q ^= D[d]. */
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm6,%zmm1,%zmm1");
|
||||
|
||||
if (np >= 3)
|
||||
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0]));
|
||||
if (np >= 4)
|
||||
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0]));
|
||||
if (np >= 5)
|
||||
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0]));
|
||||
if (np >= 6)
|
||||
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0]));
|
||||
|
||||
if (np >= 3) {
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm6,%zmm12");
|
||||
asm volatile ("vpxorq %zmm12,%zmm2,%zmm2");
|
||||
}
|
||||
if (np >= 4) {
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm6,%zmm13");
|
||||
asm volatile ("vpxorq %zmm13,%zmm3,%zmm3");
|
||||
}
|
||||
if (np >= 5) {
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm6,%zmm14");
|
||||
asm volatile ("vpxorq %zmm14,%zmm4,%zmm4");
|
||||
}
|
||||
if (np >= 6) {
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm6,%zmm15");
|
||||
asm volatile ("vpxorq %zmm15,%zmm5,%zmm5");
|
||||
}
|
||||
}
|
||||
|
||||
/*
|
||||
* Final Horner step with D0.
|
||||
*
|
||||
* All Cauchy rows have coefficient 1 for D0.
|
||||
*/
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm1,%zmm1");
|
||||
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm6,%zmm1,%zmm1");
|
||||
|
||||
if (np >= 3)
|
||||
asm volatile ("vpxorq %zmm6,%zmm2,%zmm2");
|
||||
if (np >= 4)
|
||||
asm volatile ("vpxorq %zmm6,%zmm3,%zmm3");
|
||||
if (np >= 5)
|
||||
asm volatile ("vpxorq %zmm6,%zmm4,%zmm4");
|
||||
if (np >= 6)
|
||||
asm volatile ("vpxorq %zmm6,%zmm5,%zmm5");
|
||||
|
||||
asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i]));
|
||||
asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i]));
|
||||
if (np >= 3)
|
||||
asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i]));
|
||||
if (np >= 4)
|
||||
asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i]));
|
||||
if (np >= 5)
|
||||
asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i]));
|
||||
if (np >= 6)
|
||||
asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GENX AVX2 GFNI implementation
|
||||
|
|
@ -198,6 +577,86 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
|
|||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_horner_raid(nd, size, vv, 2);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_horner_raid(nd, size, vv, 2);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_horner_raid(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_horner_raid(nd, size, vv, 3);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_horner_raid(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_horner_raid(nd, size, vv, 4);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_horner_raid(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_horner_raid(nd, size, vv, 5);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx2gfni_raid(nd, size, vv, 6);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv)
|
||||
{
|
||||
raid_genX_avx512gfni_horner_raid(nd, size, vv, 6);
|
||||
}
|
||||
|
||||
/*
|
||||
* GEN2 Cauchy AVX2 GFNI implementation
|
||||
*/
|
||||
|
|
@ -281,7 +740,362 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv)
|
|||
/*
|
||||
* RAID recovering for one disk AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
uint8_t G, V;
|
||||
size_t i;
|
||||
|
||||
(void)nr;
|
||||
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
G = A(ip[0], id[0]);
|
||||
V = inv(G);
|
||||
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%ymm4" : : "m" (raid_gfaffine_raid[V][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for one disk GFNI implementation
|
||||
*/
|
||||
void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
uint8_t G, V;
|
||||
size_t i;
|
||||
|
||||
(void)nr;
|
||||
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
G = A(ip[0], id[0]);
|
||||
V = inv(G);
|
||||
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfaffine_raid[V][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
|
||||
asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm0,%zmm0");
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[N], *pa[N];
|
||||
uint8_t G[N * N], V[N * N];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
asm volatile ("vpbroadcastq %0,%%ymm10" : : "m" (raid_gfaffine_raid[V[0]][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[V[1]][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%ymm12" : : "m" (raid_gfaffine_raid[V[2]][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%ymm13" : : "m" (raid_gfaffine_raid[V[3]][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm7,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm0,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm1,%ymm5");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm4,%ymm4");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32]));
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm0,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm1,%ymm7");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm2,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm6,%ymm6");
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm3,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm7,%ymm7");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
|
||||
asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks GFNI implementation
|
||||
*/
|
||||
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[N], *pa[N];
|
||||
uint8_t G[N * N], V[N * N];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[V[0]][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[V[1]][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[V[2]][0]));
|
||||
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[V[3]][0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm3,%zmm1,%zmm1");
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm0,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm1,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm0,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm1,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t buffer[RAID_PARITY_MAX * 64 + 64];
|
||||
uint8_t *pd = __align_ptr(buffer, 64);
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* delta */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32]));
|
||||
}
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vpxor %ymm0,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm1,%ymm1,%ymm1");
|
||||
for (k = 0; k < N; ++k) {
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64]));
|
||||
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastq %0,%%ymm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0]));
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm4,%ymm3");
|
||||
asm volatile ("vpxor %ymm3,%ymm0,%ymm0");
|
||||
|
||||
asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm5,%ymm3");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
}
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* RAID recovering GFNI implementation
|
||||
*/
|
||||
void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t buffer[RAID_PARITY_MAX * 64 + 64];
|
||||
uint8_t *pd = __align_ptr(buffer, 64);
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* delta */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i]));
|
||||
asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64]));
|
||||
}
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vpxorq %zmm0,%zmm0,%zmm0");
|
||||
for (k = 0; k < N; ++k) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64]));
|
||||
asm volatile ("vpbroadcastq %0,%%zmm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0]));
|
||||
asm volatile ("vgf2p8affineqb $0,%zmm2,%zmm4,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
}
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
|
||||
/*
|
||||
* AES recovering for one disk AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
|
|
@ -328,9 +1142,9 @@ void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for one disk GFNI implementation
|
||||
* AES recovering for one disk GFNI implementation
|
||||
*/
|
||||
void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
|
|
@ -368,9 +1182,9 @@ void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks AVX2 GFNI implementation
|
||||
* AES recovering for two disks AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
|
|
@ -445,9 +1259,9 @@ void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering for two disks GFNI implementation
|
||||
* AES recovering for two disks GFNI implementation
|
||||
*/
|
||||
void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
|
|
@ -509,9 +1323,9 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering AVX2 GFNI implementation
|
||||
* AES recovering AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
|
|
@ -579,9 +1393,9 @@ void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
}
|
||||
|
||||
/*
|
||||
* RAID recovering GFNI implementation
|
||||
* AES recovering GFNI implementation
|
||||
*/
|
||||
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
|
|
@ -636,18 +1450,31 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
void raid_register_avx2gfni(void)
|
||||
{
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_raid, RAID_POLY_RAID);
|
||||
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_aes, RAID_POLY_AES);
|
||||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID);
|
||||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES);
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -655,18 +1482,31 @@ void raid_register_avx512gfni(void)
|
|||
{
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (!raid_cpu_has_slow_avx512()) {
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_raid, RAID_POLY_RAID);
|
||||
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_aes, RAID_POLY_AES);
|
||||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID);
|
||||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -104,6 +104,8 @@ void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv);
|
|||
void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_genz_int32_raid(int nd, size_t size, void **vv);
|
||||
|
|
@ -119,6 +121,8 @@ void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv);
|
|||
void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen4_int8(int nd, size_t size, void **vv);
|
||||
|
|
@ -129,6 +133,8 @@ void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv);
|
|||
void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen5_int8(int nd, size_t size, void **vv);
|
||||
|
|
@ -138,6 +144,8 @@ void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv);
|
|||
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen6_int8(int nd, size_t size, void **vv);
|
||||
|
|
@ -147,6 +155,8 @@ void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv);
|
|||
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv);
|
||||
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
|
|
@ -161,12 +171,18 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
|||
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_gen1_neon(int nd, size_t size, void **vv);
|
||||
void raid_gen2_neon_raid(int nd, size_t size, void **vv);
|
||||
void raid_gen2_neon_aes(int nd, size_t size, void **vv);
|
||||
|
|
@ -325,6 +341,7 @@ extern const uint8_t raid_gfexp_raid[256] __aligned(256);
|
|||
extern const uint8_t raid_gfinv_raid[256] __aligned(256);
|
||||
extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256);
|
||||
extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256);
|
||||
extern const uint8_t raid_gfaffine_raid[256][8] __aligned(256);
|
||||
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
|
||||
extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256);
|
||||
extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256);
|
||||
|
|
|
|||
|
|
@ -38,6 +38,34 @@ static uint8_t raid_gfmul(uint8_t poly, uint8_t a, uint8_t b)
|
|||
return v;
|
||||
}
|
||||
|
||||
/**
|
||||
* Setup the GFNI affine matrix for multiplication by a constant.
|
||||
*
|
||||
* VGF2P8MULB is fixed to the AES polynomial 0x11b, while RAID parity is
|
||||
* defined over GF(2^8) modulo 0x11d. VGF2P8AFFINEQB can implement the
|
||||
* multiplication because multiplication by a constant is GF(2)-linear.
|
||||
*
|
||||
* VGF2P8AFFINEQB stores the row producing output bit j in byte 7-j.
|
||||
* Within each row, bit k selects input bit k.
|
||||
*/
|
||||
static void set_affine(uint8_t poly, uint8_t c, uint8_t *matrix)
|
||||
{
|
||||
int j, k;
|
||||
|
||||
for (j = 0; j < 8; ++j) {
|
||||
uint8_t row = 0;
|
||||
|
||||
for (k = 0; k < 8; ++k) {
|
||||
uint8_t v = raid_gfmul(poly, c, (uint8_t)(1U << k));
|
||||
|
||||
if ((v & (1U << j)) != 0)
|
||||
row |= (uint8_t)(1U << k);
|
||||
}
|
||||
|
||||
matrix[7 - j] = row;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Inversion (1/a) in GF(2^8).
|
||||
*/
|
||||
|
|
@ -231,6 +259,31 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
|
|||
}
|
||||
printf("};\n\n");
|
||||
|
||||
if (poly == RAID_POLY_RAID) {
|
||||
uint8_t affine[8];
|
||||
|
||||
printf("/**\n");
|
||||
printf(" * GFNI affine matrices for multiplication in GF(2^8)/0x11d.\n");
|
||||
printf(" *\n");
|
||||
printf(" * VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane.\n");
|
||||
printf(" * Byte 7-j contains the row producing output bit j, and bit k\n");
|
||||
printf(" * of that byte selects input bit k. The instruction uses imm8=0.\n");
|
||||
printf(" */\n");
|
||||
printf("const uint8_t __aligned(256) raid_gfaffine_raid[256][8] =\n");
|
||||
printf("{\n");
|
||||
for (i = 0; i < 256; ++i) {
|
||||
set_affine(poly, (uint8_t)i, affine);
|
||||
printf("\t{ ");
|
||||
for (j = 0; j < 8; ++j) {
|
||||
printf("0x%02x", (unsigned)affine[j]);
|
||||
if (j != 7)
|
||||
printf(", ");
|
||||
}
|
||||
printf(" },\n");
|
||||
}
|
||||
printf("};\n\n");
|
||||
}
|
||||
|
||||
/* generator^a */
|
||||
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
|
||||
printf("{\n");
|
||||
|
|
|
|||
48
raid/raid.h
48
raid/raid.h
|
|
@ -21,26 +21,32 @@
|
|||
* The polynomial used by the original RAID-6 specification and the
|
||||
* Linux kernel RAID implementation, used with primitive generator g=2.
|
||||
* Choosing this polynomial ensures on-disk compatibility with existing
|
||||
* RAID arrays and interchange with other RAID implementations. Use this
|
||||
* when reading or writing data that may be accessed by other software or
|
||||
* hardware RAID controllers.
|
||||
* RAID arrays and interchange with other RAID implementations.
|
||||
*
|
||||
* On CPUs supporting Intel GFNI (Galois Field New Instructions),
|
||||
* generation and recovery are accelerated with vgf2p8affineqb.
|
||||
* Multiplication by each GF(2^8) coefficient is represented as a
|
||||
* precomputed GF(2) 8x8 affine matrix for the 0x11d field.
|
||||
*
|
||||
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial
|
||||
*
|
||||
* The polynomial used by the AES encryption standard (0x11b including
|
||||
* the x^8 term), used with primitive generator g=3. Choosing this
|
||||
* polynomial enables the use of the Intel GFNI (Galois Field New
|
||||
* Instructions) extension, specifically the vgf2p8mulb instruction,
|
||||
* which performs GF(2^8) multiplication natively in hardware using
|
||||
* exactly this polynomial. This eliminates the nibble-split pshufb
|
||||
* multiply sequence (vpsrlw + vpandq + 2x vpshufb + vpxorq) and the
|
||||
* conditional multiply-by-2 sequence (vpcmpgtb + vpmovm2b + vpaddb +
|
||||
* vpandq + vpxorq) used in the AVX-512BW path, replacing each with a
|
||||
* single vgf2p8mulb instruction. On CPUs supporting GFNI (Ice Lake and
|
||||
* later), this yields a significant throughput improvement. The
|
||||
* trade-off is that parity data is not compatible with arrays using
|
||||
* the standard 0x1d polynomial (g=2), so this option requires that all
|
||||
* data be generated and recovered using the same operating mode.
|
||||
* the x^8 term), used with primitive generator g=3.
|
||||
*
|
||||
* On CPUs supporting Intel GFNI, generation and recovery can use the
|
||||
* vgf2p8mulb instruction, which performs GF(2^8) multiplication directly
|
||||
* in hardware using this polynomial. This native multiplication is
|
||||
* typically faster than the affine GFNI implementation required for the
|
||||
* standard RAID polynomial.
|
||||
*
|
||||
* The trade-off is that parity data is not compatible with arrays using
|
||||
* the standard 0x11d polynomial (g=2), so all data must be generated and
|
||||
* recovered using the same operating mode.
|
||||
*
|
||||
* Both Cauchy modes can therefore use GFNI acceleration. The standard RAID
|
||||
* mode preserves compatibility by using affine GFNI transformations, while
|
||||
* the AES mode trades compatibility for the typically faster native GFNI
|
||||
* multiplication instruction.
|
||||
*/
|
||||
|
||||
/**
|
||||
|
|
@ -51,7 +57,8 @@
|
|||
/**
|
||||
* Default RAID mode supporting up to 6 parities using standard polynomial 0x1d and generator g=2.
|
||||
*
|
||||
* Provides high performance on modern CPUs with SSSE3, AVX2, AVX-512, or NEON support.
|
||||
* Provides high performance on modern CPUs with SSSE3, AVX2, AVX-512, GFNI, or NEON support.
|
||||
* On GFNI CPUs, the standard RAID field is accelerated using affine transformations.
|
||||
* Ensures maximum compatibility with standard Linux RAID arrays.
|
||||
*
|
||||
* This is the default mode set after calling raid_init().
|
||||
|
|
@ -72,8 +79,11 @@
|
|||
* RAID mode supporting up to 6 parities using AES polynomial 0x1b (0x11b) and primitive generator g=3.
|
||||
*
|
||||
* It has slightly lower performance than RAID_MODE_CAUCHY_RAID on CPUs without GFNI due
|
||||
* to the additional cost of generator g=3, but achieves significantly higher throughput on
|
||||
* CPUs supporting GFNI hardware acceleration.
|
||||
* to the additional cost of generator g=3. On CPUs supporting GFNI, it uses native
|
||||
* VGF2P8MULB multiplication and is typically faster than the affine GFNI implementation
|
||||
* required by RAID_MODE_CAUCHY_RAID.
|
||||
*
|
||||
* Parity data is incompatible with RAID_MODE_CAUCHY_RAID.
|
||||
*/
|
||||
#define RAID_MODE_CAUCHY_AES 2
|
||||
|
||||
|
|
|
|||
267
raid/tables.c
267
raid/tables.c
|
|
@ -8713,6 +8713,273 @@ const uint8_t __aligned(256) raid_gfmul_raid[256][256] =
|
|||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* GFNI affine matrices for multiplication in GF(2^8)/0x11d.
|
||||
*
|
||||
* VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane.
|
||||
* Byte 7-j contains the row producing output bit j, and bit k
|
||||
* of that byte selects input bit k. The instruction uses imm8=0.
|
||||
*/
|
||||
const uint8_t __aligned(256) raid_gfaffine_raid[256][8] =
|
||||
{
|
||||
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
|
||||
{ 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 },
|
||||
{ 0x40, 0x20, 0x10, 0x88, 0x84, 0x82, 0x01, 0x80 },
|
||||
{ 0xc0, 0x60, 0x30, 0x98, 0x8c, 0x86, 0x03, 0x81 },
|
||||
{ 0x20, 0x10, 0x88, 0xc4, 0xc2, 0x41, 0x80, 0x40 },
|
||||
{ 0xa0, 0x50, 0xa8, 0xd4, 0xca, 0x45, 0x82, 0x41 },
|
||||
{ 0x60, 0x30, 0x98, 0x4c, 0x46, 0xc3, 0x81, 0xc0 },
|
||||
{ 0xe0, 0x70, 0xb8, 0x5c, 0x4e, 0xc7, 0x83, 0xc1 },
|
||||
{ 0x10, 0x88, 0xc4, 0xe2, 0x61, 0xa0, 0x40, 0x20 },
|
||||
{ 0x90, 0xc8, 0xe4, 0xf2, 0x69, 0xa4, 0x42, 0x21 },
|
||||
{ 0x50, 0xa8, 0xd4, 0x6a, 0xe5, 0x22, 0x41, 0xa0 },
|
||||
{ 0xd0, 0xe8, 0xf4, 0x7a, 0xed, 0x26, 0x43, 0xa1 },
|
||||
{ 0x30, 0x98, 0x4c, 0x26, 0xa3, 0xe1, 0xc0, 0x60 },
|
||||
{ 0xb0, 0xd8, 0x6c, 0x36, 0xab, 0xe5, 0xc2, 0x61 },
|
||||
{ 0x70, 0xb8, 0x5c, 0xae, 0x27, 0x63, 0xc1, 0xe0 },
|
||||
{ 0xf0, 0xf8, 0x7c, 0xbe, 0x2f, 0x67, 0xc3, 0xe1 },
|
||||
{ 0x88, 0xc4, 0xe2, 0x71, 0xb0, 0x50, 0x20, 0x10 },
|
||||
{ 0x08, 0x84, 0xc2, 0x61, 0xb8, 0x54, 0x22, 0x11 },
|
||||
{ 0xc8, 0xe4, 0xf2, 0xf9, 0x34, 0xd2, 0x21, 0x90 },
|
||||
{ 0x48, 0xa4, 0xd2, 0xe9, 0x3c, 0xd6, 0x23, 0x91 },
|
||||
{ 0xa8, 0xd4, 0x6a, 0xb5, 0x72, 0x11, 0xa0, 0x50 },
|
||||
{ 0x28, 0x94, 0x4a, 0xa5, 0x7a, 0x15, 0xa2, 0x51 },
|
||||
{ 0xe8, 0xf4, 0x7a, 0x3d, 0xf6, 0x93, 0xa1, 0xd0 },
|
||||
{ 0x68, 0xb4, 0x5a, 0x2d, 0xfe, 0x97, 0xa3, 0xd1 },
|
||||
{ 0x98, 0x4c, 0x26, 0x93, 0xd1, 0xf0, 0x60, 0x30 },
|
||||
{ 0x18, 0x0c, 0x06, 0x83, 0xd9, 0xf4, 0x62, 0x31 },
|
||||
{ 0xd8, 0x6c, 0x36, 0x1b, 0x55, 0x72, 0x61, 0xb0 },
|
||||
{ 0x58, 0x2c, 0x16, 0x0b, 0x5d, 0x76, 0x63, 0xb1 },
|
||||
{ 0xb8, 0x5c, 0xae, 0x57, 0x13, 0xb1, 0xe0, 0x70 },
|
||||
{ 0x38, 0x1c, 0x8e, 0x47, 0x1b, 0xb5, 0xe2, 0x71 },
|
||||
{ 0xf8, 0x7c, 0xbe, 0xdf, 0x97, 0x33, 0xe1, 0xf0 },
|
||||
{ 0x78, 0x3c, 0x9e, 0xcf, 0x9f, 0x37, 0xe3, 0xf1 },
|
||||
{ 0xc4, 0xe2, 0x71, 0x38, 0xd8, 0xa8, 0x10, 0x88 },
|
||||
{ 0x44, 0xa2, 0x51, 0x28, 0xd0, 0xac, 0x12, 0x89 },
|
||||
{ 0x84, 0xc2, 0x61, 0xb0, 0x5c, 0x2a, 0x11, 0x08 },
|
||||
{ 0x04, 0x82, 0x41, 0xa0, 0x54, 0x2e, 0x13, 0x09 },
|
||||
{ 0xe4, 0xf2, 0xf9, 0xfc, 0x1a, 0xe9, 0x90, 0xc8 },
|
||||
{ 0x64, 0xb2, 0xd9, 0xec, 0x12, 0xed, 0x92, 0xc9 },
|
||||
{ 0xa4, 0xd2, 0xe9, 0x74, 0x9e, 0x6b, 0x91, 0x48 },
|
||||
{ 0x24, 0x92, 0xc9, 0x64, 0x96, 0x6f, 0x93, 0x49 },
|
||||
{ 0xd4, 0x6a, 0xb5, 0xda, 0xb9, 0x08, 0x50, 0xa8 },
|
||||
{ 0x54, 0x2a, 0x95, 0xca, 0xb1, 0x0c, 0x52, 0xa9 },
|
||||
{ 0x94, 0x4a, 0xa5, 0x52, 0x3d, 0x8a, 0x51, 0x28 },
|
||||
{ 0x14, 0x0a, 0x85, 0x42, 0x35, 0x8e, 0x53, 0x29 },
|
||||
{ 0xf4, 0x7a, 0x3d, 0x1e, 0x7b, 0x49, 0xd0, 0xe8 },
|
||||
{ 0x74, 0x3a, 0x1d, 0x0e, 0x73, 0x4d, 0xd2, 0xe9 },
|
||||
{ 0xb4, 0x5a, 0x2d, 0x96, 0xff, 0xcb, 0xd1, 0x68 },
|
||||
{ 0x34, 0x1a, 0x0d, 0x86, 0xf7, 0xcf, 0xd3, 0x69 },
|
||||
{ 0x4c, 0x26, 0x93, 0x49, 0x68, 0xf8, 0x30, 0x98 },
|
||||
{ 0xcc, 0x66, 0xb3, 0x59, 0x60, 0xfc, 0x32, 0x99 },
|
||||
{ 0x0c, 0x06, 0x83, 0xc1, 0xec, 0x7a, 0x31, 0x18 },
|
||||
{ 0x8c, 0x46, 0xa3, 0xd1, 0xe4, 0x7e, 0x33, 0x19 },
|
||||
{ 0x6c, 0x36, 0x1b, 0x8d, 0xaa, 0xb9, 0xb0, 0xd8 },
|
||||
{ 0xec, 0x76, 0x3b, 0x9d, 0xa2, 0xbd, 0xb2, 0xd9 },
|
||||
{ 0x2c, 0x16, 0x0b, 0x05, 0x2e, 0x3b, 0xb1, 0x58 },
|
||||
{ 0xac, 0x56, 0x2b, 0x15, 0x26, 0x3f, 0xb3, 0x59 },
|
||||
{ 0x5c, 0xae, 0x57, 0xab, 0x09, 0x58, 0x70, 0xb8 },
|
||||
{ 0xdc, 0xee, 0x77, 0xbb, 0x01, 0x5c, 0x72, 0xb9 },
|
||||
{ 0x1c, 0x8e, 0x47, 0x23, 0x8d, 0xda, 0x71, 0x38 },
|
||||
{ 0x9c, 0xce, 0x67, 0x33, 0x85, 0xde, 0x73, 0x39 },
|
||||
{ 0x7c, 0xbe, 0xdf, 0x6f, 0xcb, 0x19, 0xf0, 0xf8 },
|
||||
{ 0xfc, 0xfe, 0xff, 0x7f, 0xc3, 0x1d, 0xf2, 0xf9 },
|
||||
{ 0x3c, 0x9e, 0xcf, 0xe7, 0x4f, 0x9b, 0xf1, 0x78 },
|
||||
{ 0xbc, 0xde, 0xef, 0xf7, 0x47, 0x9f, 0xf3, 0x79 },
|
||||
{ 0xe2, 0x71, 0x38, 0x1c, 0x6c, 0xd4, 0x88, 0xc4 },
|
||||
{ 0x62, 0x31, 0x18, 0x0c, 0x64, 0xd0, 0x8a, 0xc5 },
|
||||
{ 0xa2, 0x51, 0x28, 0x94, 0xe8, 0x56, 0x89, 0x44 },
|
||||
{ 0x22, 0x11, 0x08, 0x84, 0xe0, 0x52, 0x8b, 0x45 },
|
||||
{ 0xc2, 0x61, 0xb0, 0xd8, 0xae, 0x95, 0x08, 0x84 },
|
||||
{ 0x42, 0x21, 0x90, 0xc8, 0xa6, 0x91, 0x0a, 0x85 },
|
||||
{ 0x82, 0x41, 0xa0, 0x50, 0x2a, 0x17, 0x09, 0x04 },
|
||||
{ 0x02, 0x01, 0x80, 0x40, 0x22, 0x13, 0x0b, 0x05 },
|
||||
{ 0xf2, 0xf9, 0xfc, 0xfe, 0x0d, 0x74, 0xc8, 0xe4 },
|
||||
{ 0x72, 0xb9, 0xdc, 0xee, 0x05, 0x70, 0xca, 0xe5 },
|
||||
{ 0xb2, 0xd9, 0xec, 0x76, 0x89, 0xf6, 0xc9, 0x64 },
|
||||
{ 0x32, 0x99, 0xcc, 0x66, 0x81, 0xf2, 0xcb, 0x65 },
|
||||
{ 0xd2, 0xe9, 0x74, 0x3a, 0xcf, 0x35, 0x48, 0xa4 },
|
||||
{ 0x52, 0xa9, 0x54, 0x2a, 0xc7, 0x31, 0x4a, 0xa5 },
|
||||
{ 0x92, 0xc9, 0x64, 0xb2, 0x4b, 0xb7, 0x49, 0x24 },
|
||||
{ 0x12, 0x89, 0x44, 0xa2, 0x43, 0xb3, 0x4b, 0x25 },
|
||||
{ 0x6a, 0xb5, 0xda, 0x6d, 0xdc, 0x84, 0xa8, 0xd4 },
|
||||
{ 0xea, 0xf5, 0xfa, 0x7d, 0xd4, 0x80, 0xaa, 0xd5 },
|
||||
{ 0x2a, 0x95, 0xca, 0xe5, 0x58, 0x06, 0xa9, 0x54 },
|
||||
{ 0xaa, 0xd5, 0xea, 0xf5, 0x50, 0x02, 0xab, 0x55 },
|
||||
{ 0x4a, 0xa5, 0x52, 0xa9, 0x1e, 0xc5, 0x28, 0x94 },
|
||||
{ 0xca, 0xe5, 0x72, 0xb9, 0x16, 0xc1, 0x2a, 0x95 },
|
||||
{ 0x0a, 0x85, 0x42, 0x21, 0x9a, 0x47, 0x29, 0x14 },
|
||||
{ 0x8a, 0xc5, 0x62, 0x31, 0x92, 0x43, 0x2b, 0x15 },
|
||||
{ 0x7a, 0x3d, 0x1e, 0x8f, 0xbd, 0x24, 0xe8, 0xf4 },
|
||||
{ 0xfa, 0x7d, 0x3e, 0x9f, 0xb5, 0x20, 0xea, 0xf5 },
|
||||
{ 0x3a, 0x1d, 0x0e, 0x07, 0x39, 0xa6, 0xe9, 0x74 },
|
||||
{ 0xba, 0x5d, 0x2e, 0x17, 0x31, 0xa2, 0xeb, 0x75 },
|
||||
{ 0x5a, 0x2d, 0x96, 0x4b, 0x7f, 0x65, 0x68, 0xb4 },
|
||||
{ 0xda, 0x6d, 0xb6, 0x5b, 0x77, 0x61, 0x6a, 0xb5 },
|
||||
{ 0x1a, 0x0d, 0x86, 0xc3, 0xfb, 0xe7, 0x69, 0x34 },
|
||||
{ 0x9a, 0x4d, 0xa6, 0xd3, 0xf3, 0xe3, 0x6b, 0x35 },
|
||||
{ 0x26, 0x93, 0x49, 0x24, 0xb4, 0x7c, 0x98, 0x4c },
|
||||
{ 0xa6, 0xd3, 0x69, 0x34, 0xbc, 0x78, 0x9a, 0x4d },
|
||||
{ 0x66, 0xb3, 0x59, 0xac, 0x30, 0xfe, 0x99, 0xcc },
|
||||
{ 0xe6, 0xf3, 0x79, 0xbc, 0x38, 0xfa, 0x9b, 0xcd },
|
||||
{ 0x06, 0x83, 0xc1, 0xe0, 0x76, 0x3d, 0x18, 0x0c },
|
||||
{ 0x86, 0xc3, 0xe1, 0xf0, 0x7e, 0x39, 0x1a, 0x0d },
|
||||
{ 0x46, 0xa3, 0xd1, 0x68, 0xf2, 0xbf, 0x19, 0x8c },
|
||||
{ 0xc6, 0xe3, 0xf1, 0x78, 0xfa, 0xbb, 0x1b, 0x8d },
|
||||
{ 0x36, 0x1b, 0x8d, 0xc6, 0xd5, 0xdc, 0xd8, 0x6c },
|
||||
{ 0xb6, 0x5b, 0xad, 0xd6, 0xdd, 0xd8, 0xda, 0x6d },
|
||||
{ 0x76, 0x3b, 0x9d, 0x4e, 0x51, 0x5e, 0xd9, 0xec },
|
||||
{ 0xf6, 0x7b, 0xbd, 0x5e, 0x59, 0x5a, 0xdb, 0xed },
|
||||
{ 0x16, 0x0b, 0x05, 0x02, 0x17, 0x9d, 0x58, 0x2c },
|
||||
{ 0x96, 0x4b, 0x25, 0x12, 0x1f, 0x99, 0x5a, 0x2d },
|
||||
{ 0x56, 0x2b, 0x15, 0x8a, 0x93, 0x1f, 0x59, 0xac },
|
||||
{ 0xd6, 0x6b, 0x35, 0x9a, 0x9b, 0x1b, 0x5b, 0xad },
|
||||
{ 0xae, 0x57, 0xab, 0x55, 0x04, 0x2c, 0xb8, 0x5c },
|
||||
{ 0x2e, 0x17, 0x8b, 0x45, 0x0c, 0x28, 0xba, 0x5d },
|
||||
{ 0xee, 0x77, 0xbb, 0xdd, 0x80, 0xae, 0xb9, 0xdc },
|
||||
{ 0x6e, 0x37, 0x9b, 0xcd, 0x88, 0xaa, 0xbb, 0xdd },
|
||||
{ 0x8e, 0x47, 0x23, 0x91, 0xc6, 0x6d, 0x38, 0x1c },
|
||||
{ 0x0e, 0x07, 0x03, 0x81, 0xce, 0x69, 0x3a, 0x1d },
|
||||
{ 0xce, 0x67, 0x33, 0x19, 0x42, 0xef, 0x39, 0x9c },
|
||||
{ 0x4e, 0x27, 0x13, 0x09, 0x4a, 0xeb, 0x3b, 0x9d },
|
||||
{ 0xbe, 0xdf, 0x6f, 0xb7, 0x65, 0x8c, 0xf8, 0x7c },
|
||||
{ 0x3e, 0x9f, 0x4f, 0xa7, 0x6d, 0x88, 0xfa, 0x7d },
|
||||
{ 0xfe, 0xff, 0x7f, 0x3f, 0xe1, 0x0e, 0xf9, 0xfc },
|
||||
{ 0x7e, 0xbf, 0x5f, 0x2f, 0xe9, 0x0a, 0xfb, 0xfd },
|
||||
{ 0x9e, 0xcf, 0xe7, 0x73, 0xa7, 0xcd, 0x78, 0x3c },
|
||||
{ 0x1e, 0x8f, 0xc7, 0x63, 0xaf, 0xc9, 0x7a, 0x3d },
|
||||
{ 0xde, 0xef, 0xf7, 0xfb, 0x23, 0x4f, 0x79, 0xbc },
|
||||
{ 0x5e, 0xaf, 0xd7, 0xeb, 0x2b, 0x4b, 0x7b, 0xbd },
|
||||
{ 0x71, 0x38, 0x1c, 0x8e, 0x36, 0x6a, 0xc4, 0xe2 },
|
||||
{ 0xf1, 0x78, 0x3c, 0x9e, 0x3e, 0x6e, 0xc6, 0xe3 },
|
||||
{ 0x31, 0x18, 0x0c, 0x06, 0xb2, 0xe8, 0xc5, 0x62 },
|
||||
{ 0xb1, 0x58, 0x2c, 0x16, 0xba, 0xec, 0xc7, 0x63 },
|
||||
{ 0x51, 0x28, 0x94, 0x4a, 0xf4, 0x2b, 0x44, 0xa2 },
|
||||
{ 0xd1, 0x68, 0xb4, 0x5a, 0xfc, 0x2f, 0x46, 0xa3 },
|
||||
{ 0x11, 0x08, 0x84, 0xc2, 0x70, 0xa9, 0x45, 0x22 },
|
||||
{ 0x91, 0x48, 0xa4, 0xd2, 0x78, 0xad, 0x47, 0x23 },
|
||||
{ 0x61, 0xb0, 0xd8, 0x6c, 0x57, 0xca, 0x84, 0xc2 },
|
||||
{ 0xe1, 0xf0, 0xf8, 0x7c, 0x5f, 0xce, 0x86, 0xc3 },
|
||||
{ 0x21, 0x90, 0xc8, 0xe4, 0xd3, 0x48, 0x85, 0x42 },
|
||||
{ 0xa1, 0xd0, 0xe8, 0xf4, 0xdb, 0x4c, 0x87, 0x43 },
|
||||
{ 0x41, 0xa0, 0x50, 0xa8, 0x95, 0x8b, 0x04, 0x82 },
|
||||
{ 0xc1, 0xe0, 0x70, 0xb8, 0x9d, 0x8f, 0x06, 0x83 },
|
||||
{ 0x01, 0x80, 0x40, 0x20, 0x11, 0x09, 0x05, 0x02 },
|
||||
{ 0x81, 0xc0, 0x60, 0x30, 0x19, 0x0d, 0x07, 0x03 },
|
||||
{ 0xf9, 0xfc, 0xfe, 0xff, 0x86, 0x3a, 0xe4, 0xf2 },
|
||||
{ 0x79, 0xbc, 0xde, 0xef, 0x8e, 0x3e, 0xe6, 0xf3 },
|
||||
{ 0xb9, 0xdc, 0xee, 0x77, 0x02, 0xb8, 0xe5, 0x72 },
|
||||
{ 0x39, 0x9c, 0xce, 0x67, 0x0a, 0xbc, 0xe7, 0x73 },
|
||||
{ 0xd9, 0xec, 0x76, 0x3b, 0x44, 0x7b, 0x64, 0xb2 },
|
||||
{ 0x59, 0xac, 0x56, 0x2b, 0x4c, 0x7f, 0x66, 0xb3 },
|
||||
{ 0x99, 0xcc, 0x66, 0xb3, 0xc0, 0xf9, 0x65, 0x32 },
|
||||
{ 0x19, 0x8c, 0x46, 0xa3, 0xc8, 0xfd, 0x67, 0x33 },
|
||||
{ 0xe9, 0x74, 0x3a, 0x1d, 0xe7, 0x9a, 0xa4, 0xd2 },
|
||||
{ 0x69, 0x34, 0x1a, 0x0d, 0xef, 0x9e, 0xa6, 0xd3 },
|
||||
{ 0xa9, 0x54, 0x2a, 0x95, 0x63, 0x18, 0xa5, 0x52 },
|
||||
{ 0x29, 0x14, 0x0a, 0x85, 0x6b, 0x1c, 0xa7, 0x53 },
|
||||
{ 0xc9, 0x64, 0xb2, 0xd9, 0x25, 0xdb, 0x24, 0x92 },
|
||||
{ 0x49, 0x24, 0x92, 0xc9, 0x2d, 0xdf, 0x26, 0x93 },
|
||||
{ 0x89, 0x44, 0xa2, 0x51, 0xa1, 0x59, 0x25, 0x12 },
|
||||
{ 0x09, 0x04, 0x82, 0x41, 0xa9, 0x5d, 0x27, 0x13 },
|
||||
{ 0xb5, 0xda, 0x6d, 0xb6, 0xee, 0xc2, 0xd4, 0x6a },
|
||||
{ 0x35, 0x9a, 0x4d, 0xa6, 0xe6, 0xc6, 0xd6, 0x6b },
|
||||
{ 0xf5, 0xfa, 0x7d, 0x3e, 0x6a, 0x40, 0xd5, 0xea },
|
||||
{ 0x75, 0xba, 0x5d, 0x2e, 0x62, 0x44, 0xd7, 0xeb },
|
||||
{ 0x95, 0xca, 0xe5, 0x72, 0x2c, 0x83, 0x54, 0x2a },
|
||||
{ 0x15, 0x8a, 0xc5, 0x62, 0x24, 0x87, 0x56, 0x2b },
|
||||
{ 0xd5, 0xea, 0xf5, 0xfa, 0xa8, 0x01, 0x55, 0xaa },
|
||||
{ 0x55, 0xaa, 0xd5, 0xea, 0xa0, 0x05, 0x57, 0xab },
|
||||
{ 0xa5, 0x52, 0xa9, 0x54, 0x8f, 0x62, 0x94, 0x4a },
|
||||
{ 0x25, 0x12, 0x89, 0x44, 0x87, 0x66, 0x96, 0x4b },
|
||||
{ 0xe5, 0x72, 0xb9, 0xdc, 0x0b, 0xe0, 0x95, 0xca },
|
||||
{ 0x65, 0x32, 0x99, 0xcc, 0x03, 0xe4, 0x97, 0xcb },
|
||||
{ 0x85, 0x42, 0x21, 0x90, 0x4d, 0x23, 0x14, 0x0a },
|
||||
{ 0x05, 0x02, 0x01, 0x80, 0x45, 0x27, 0x16, 0x0b },
|
||||
{ 0xc5, 0x62, 0x31, 0x18, 0xc9, 0xa1, 0x15, 0x8a },
|
||||
{ 0x45, 0x22, 0x11, 0x08, 0xc1, 0xa5, 0x17, 0x8b },
|
||||
{ 0x3d, 0x1e, 0x8f, 0xc7, 0x5e, 0x92, 0xf4, 0x7a },
|
||||
{ 0xbd, 0x5e, 0xaf, 0xd7, 0x56, 0x96, 0xf6, 0x7b },
|
||||
{ 0x7d, 0x3e, 0x9f, 0x4f, 0xda, 0x10, 0xf5, 0xfa },
|
||||
{ 0xfd, 0x7e, 0xbf, 0x5f, 0xd2, 0x14, 0xf7, 0xfb },
|
||||
{ 0x1d, 0x0e, 0x07, 0x03, 0x9c, 0xd3, 0x74, 0x3a },
|
||||
{ 0x9d, 0x4e, 0x27, 0x13, 0x94, 0xd7, 0x76, 0x3b },
|
||||
{ 0x5d, 0x2e, 0x17, 0x8b, 0x18, 0x51, 0x75, 0xba },
|
||||
{ 0xdd, 0x6e, 0x37, 0x9b, 0x10, 0x55, 0x77, 0xbb },
|
||||
{ 0x2d, 0x96, 0x4b, 0x25, 0x3f, 0x32, 0xb4, 0x5a },
|
||||
{ 0xad, 0xd6, 0x6b, 0x35, 0x37, 0x36, 0xb6, 0x5b },
|
||||
{ 0x6d, 0xb6, 0x5b, 0xad, 0xbb, 0xb0, 0xb5, 0xda },
|
||||
{ 0xed, 0xf6, 0x7b, 0xbd, 0xb3, 0xb4, 0xb7, 0xdb },
|
||||
{ 0x0d, 0x86, 0xc3, 0xe1, 0xfd, 0x73, 0x34, 0x1a },
|
||||
{ 0x8d, 0xc6, 0xe3, 0xf1, 0xf5, 0x77, 0x36, 0x1b },
|
||||
{ 0x4d, 0xa6, 0xd3, 0x69, 0x79, 0xf1, 0x35, 0x9a },
|
||||
{ 0xcd, 0xe6, 0xf3, 0x79, 0x71, 0xf5, 0x37, 0x9b },
|
||||
{ 0x93, 0x49, 0x24, 0x92, 0x5a, 0xbe, 0x4c, 0x26 },
|
||||
{ 0x13, 0x09, 0x04, 0x82, 0x52, 0xba, 0x4e, 0x27 },
|
||||
{ 0xd3, 0x69, 0x34, 0x1a, 0xde, 0x3c, 0x4d, 0xa6 },
|
||||
{ 0x53, 0x29, 0x14, 0x0a, 0xd6, 0x38, 0x4f, 0xa7 },
|
||||
{ 0xb3, 0x59, 0xac, 0x56, 0x98, 0xff, 0xcc, 0x66 },
|
||||
{ 0x33, 0x19, 0x8c, 0x46, 0x90, 0xfb, 0xce, 0x67 },
|
||||
{ 0xf3, 0x79, 0xbc, 0xde, 0x1c, 0x7d, 0xcd, 0xe6 },
|
||||
{ 0x73, 0x39, 0x9c, 0xce, 0x14, 0x79, 0xcf, 0xe7 },
|
||||
{ 0x83, 0xc1, 0xe0, 0x70, 0x3b, 0x1e, 0x0c, 0x06 },
|
||||
{ 0x03, 0x81, 0xc0, 0x60, 0x33, 0x1a, 0x0e, 0x07 },
|
||||
{ 0xc3, 0xe1, 0xf0, 0xf8, 0xbf, 0x9c, 0x0d, 0x86 },
|
||||
{ 0x43, 0xa1, 0xd0, 0xe8, 0xb7, 0x98, 0x0f, 0x87 },
|
||||
{ 0xa3, 0xd1, 0x68, 0xb4, 0xf9, 0x5f, 0x8c, 0x46 },
|
||||
{ 0x23, 0x91, 0x48, 0xa4, 0xf1, 0x5b, 0x8e, 0x47 },
|
||||
{ 0xe3, 0xf1, 0x78, 0x3c, 0x7d, 0xdd, 0x8d, 0xc6 },
|
||||
{ 0x63, 0xb1, 0x58, 0x2c, 0x75, 0xd9, 0x8f, 0xc7 },
|
||||
{ 0x1b, 0x8d, 0xc6, 0xe3, 0xea, 0xee, 0x6c, 0x36 },
|
||||
{ 0x9b, 0xcd, 0xe6, 0xf3, 0xe2, 0xea, 0x6e, 0x37 },
|
||||
{ 0x5b, 0xad, 0xd6, 0x6b, 0x6e, 0x6c, 0x6d, 0xb6 },
|
||||
{ 0xdb, 0xed, 0xf6, 0x7b, 0x66, 0x68, 0x6f, 0xb7 },
|
||||
{ 0x3b, 0x9d, 0x4e, 0x27, 0x28, 0xaf, 0xec, 0x76 },
|
||||
{ 0xbb, 0xdd, 0x6e, 0x37, 0x20, 0xab, 0xee, 0x77 },
|
||||
{ 0x7b, 0xbd, 0x5e, 0xaf, 0xac, 0x2d, 0xed, 0xf6 },
|
||||
{ 0xfb, 0xfd, 0x7e, 0xbf, 0xa4, 0x29, 0xef, 0xf7 },
|
||||
{ 0x0b, 0x05, 0x02, 0x01, 0x8b, 0x4e, 0x2c, 0x16 },
|
||||
{ 0x8b, 0x45, 0x22, 0x11, 0x83, 0x4a, 0x2e, 0x17 },
|
||||
{ 0x4b, 0x25, 0x12, 0x89, 0x0f, 0xcc, 0x2d, 0x96 },
|
||||
{ 0xcb, 0x65, 0x32, 0x99, 0x07, 0xc8, 0x2f, 0x97 },
|
||||
{ 0x2b, 0x15, 0x8a, 0xc5, 0x49, 0x0f, 0xac, 0x56 },
|
||||
{ 0xab, 0x55, 0xaa, 0xd5, 0x41, 0x0b, 0xae, 0x57 },
|
||||
{ 0x6b, 0x35, 0x9a, 0x4d, 0xcd, 0x8d, 0xad, 0xd6 },
|
||||
{ 0xeb, 0x75, 0xba, 0x5d, 0xc5, 0x89, 0xaf, 0xd7 },
|
||||
{ 0x57, 0xab, 0x55, 0xaa, 0x82, 0x16, 0x5c, 0xae },
|
||||
{ 0xd7, 0xeb, 0x75, 0xba, 0x8a, 0x12, 0x5e, 0xaf },
|
||||
{ 0x17, 0x8b, 0x45, 0x22, 0x06, 0x94, 0x5d, 0x2e },
|
||||
{ 0x97, 0xcb, 0x65, 0x32, 0x0e, 0x90, 0x5f, 0x2f },
|
||||
{ 0x77, 0xbb, 0xdd, 0x6e, 0x40, 0x57, 0xdc, 0xee },
|
||||
{ 0xf7, 0xfb, 0xfd, 0x7e, 0x48, 0x53, 0xde, 0xef },
|
||||
{ 0x37, 0x9b, 0xcd, 0xe6, 0xc4, 0xd5, 0xdd, 0x6e },
|
||||
{ 0xb7, 0xdb, 0xed, 0xf6, 0xcc, 0xd1, 0xdf, 0x6f },
|
||||
{ 0x47, 0x23, 0x91, 0x48, 0xe3, 0xb6, 0x1c, 0x8e },
|
||||
{ 0xc7, 0x63, 0xb1, 0x58, 0xeb, 0xb2, 0x1e, 0x8f },
|
||||
{ 0x07, 0x03, 0x81, 0xc0, 0x67, 0x34, 0x1d, 0x0e },
|
||||
{ 0x87, 0x43, 0xa1, 0xd0, 0x6f, 0x30, 0x1f, 0x0f },
|
||||
{ 0x67, 0x33, 0x19, 0x8c, 0x21, 0xf7, 0x9c, 0xce },
|
||||
{ 0xe7, 0x73, 0x39, 0x9c, 0x29, 0xf3, 0x9e, 0xcf },
|
||||
{ 0x27, 0x13, 0x09, 0x04, 0xa5, 0x75, 0x9d, 0x4e },
|
||||
{ 0xa7, 0x53, 0x29, 0x14, 0xad, 0x71, 0x9f, 0x4f },
|
||||
{ 0xdf, 0x6f, 0xb7, 0xdb, 0x32, 0x46, 0x7c, 0xbe },
|
||||
{ 0x5f, 0x2f, 0x97, 0xcb, 0x3a, 0x42, 0x7e, 0xbf },
|
||||
{ 0x9f, 0x4f, 0xa7, 0x53, 0xb6, 0xc4, 0x7d, 0x3e },
|
||||
{ 0x1f, 0x0f, 0x87, 0x43, 0xbe, 0xc0, 0x7f, 0x3f },
|
||||
{ 0xff, 0x7f, 0x3f, 0x1f, 0xf0, 0x07, 0xfc, 0xfe },
|
||||
{ 0x7f, 0x3f, 0x1f, 0x0f, 0xf8, 0x03, 0xfe, 0xff },
|
||||
{ 0xbf, 0x5f, 0x2f, 0x97, 0x74, 0x85, 0xfd, 0x7e },
|
||||
{ 0x3f, 0x1f, 0x0f, 0x87, 0x7c, 0x81, 0xff, 0x7f },
|
||||
{ 0xcf, 0xe7, 0x73, 0x39, 0x53, 0xe6, 0x3c, 0x9e },
|
||||
{ 0x4f, 0xa7, 0x53, 0x29, 0x5b, 0xe2, 0x3e, 0x9f },
|
||||
{ 0x8f, 0xc7, 0x63, 0xb1, 0xd7, 0x64, 0x3d, 0x1e },
|
||||
{ 0x0f, 0x87, 0x43, 0xa1, 0xdf, 0x60, 0x3f, 0x1f },
|
||||
{ 0xef, 0xf7, 0xfb, 0xfd, 0x91, 0xa7, 0xbc, 0xde },
|
||||
{ 0x6f, 0xb7, 0xdb, 0xed, 0x99, 0xa3, 0xbe, 0xdf },
|
||||
{ 0xaf, 0xd7, 0xeb, 0x75, 0x15, 0x25, 0xbd, 0x5e },
|
||||
{ 0x2f, 0x97, 0xcb, 0x65, 0x1d, 0x21, 0xbf, 0x5f },
|
||||
};
|
||||
|
||||
const uint8_t __aligned(256) raid_gfexp_raid[256] =
|
||||
{
|
||||
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,
|
||||
|
|
|
|||
119
raid/test.c
119
raid/test.c
|
|
@ -40,6 +40,39 @@ static void meminc(void *dst, size_t size)
|
|||
++p[i];
|
||||
}
|
||||
|
||||
/**
|
||||
* Simulate the bit-level semantics of VGF2P8AFFINEQB with imm8=0.
|
||||
*/
|
||||
static uint8_t raid_gfaffine_mul(uint8_t c, uint8_t x)
|
||||
{
|
||||
uint8_t v = 0;
|
||||
int j;
|
||||
|
||||
for (j = 0; j < 8; ++j) {
|
||||
uint8_t row = raid_gfaffine_raid[c][7 - j] & x;
|
||||
|
||||
row ^= row >> 4;
|
||||
row ^= row >> 2;
|
||||
row ^= row >> 1;
|
||||
v |= (row & 1U) << j;
|
||||
}
|
||||
|
||||
return v;
|
||||
}
|
||||
|
||||
int raid_test_gfaffine(void)
|
||||
{
|
||||
int c, x;
|
||||
|
||||
/* all coefficients, including 0, 1, 2, and 0xff, are mandatory */
|
||||
for (c = 0; c < 256; ++c)
|
||||
for (x = 0; x < 256; ++x)
|
||||
if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x])
|
||||
return -1;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int raid_test_combo(void)
|
||||
{
|
||||
int r;
|
||||
|
|
@ -366,11 +399,19 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
if (raid_cpu_has_avx2())
|
||||
test_setup(i) = raid_rec1_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
test_setup(i) = raid_rec1_avx2gfni;
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
test_setup(i) = raid_rec1_avx512gfni;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec1_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec1_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec1_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec1_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
@ -387,11 +428,19 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
test_setup(i) = raid_rec2_avx512bw;
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
test_setup(i) = raid_rec2_avx2gfni;
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
test_setup(i) = raid_rec2_avx512gfni;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec2_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec2_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_rec2_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_rec2_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
@ -408,11 +457,19 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512bw())
|
||||
test_setup(i) = raid_recX_avx512bw;
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
test_setup(i) = raid_recX_avx2gfni;
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
test_setup(i) = raid_recX_avx512gfni;
|
||||
if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_recX_avx2gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_recX_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(i) = raid_recX_avx512gfni_aes;
|
||||
else
|
||||
test_setup(i) = raid_recX_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
@ -582,13 +639,17 @@ int raid_test_par(int mode, int nd, size_t size)
|
|||
test_setup(1) = raid_gen1_avx512bw;
|
||||
test_setup(2) = raid_gen2_avx512bw;
|
||||
}
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(2) = raid_gen2_avx2gfni_aes;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
else
|
||||
test_setup(2) = raid_gen2_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES)
|
||||
test_setup(2) = raid_gen2_avx512gfni_aes;
|
||||
}
|
||||
else
|
||||
test_setup(2) = raid_gen2_avx512gfni_raid;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
|
@ -658,18 +719,30 @@ int raid_test_par(int mode, int nd, size_t size)
|
|||
test_setup(5) = raid_gen5_avx512bw;
|
||||
test_setup(6) = raid_gen6_avx512bw;
|
||||
}
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
test_setup(3) = raid_gen3_avx2gfni_aes;
|
||||
test_setup(4) = raid_gen4_avx2gfni_aes;
|
||||
test_setup(5) = raid_gen5_avx2gfni_aes;
|
||||
test_setup(6) = raid_gen6_avx2gfni_aes;
|
||||
} else {
|
||||
test_setup(3) = raid_gen3_avx2gfni_raid;
|
||||
test_setup(4) = raid_gen4_avx2gfni_raid;
|
||||
test_setup(5) = raid_gen5_avx2gfni_raid;
|
||||
test_setup(6) = raid_gen6_avx2gfni_raid;
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (mode == RAID_MODE_CAUCHY_AES) {
|
||||
test_setup(3) = raid_gen3_avx512gfni_aes;
|
||||
test_setup(4) = raid_gen4_avx512gfni_aes;
|
||||
test_setup(5) = raid_gen5_avx512gfni_aes;
|
||||
test_setup(6) = raid_gen6_avx512gfni_aes;
|
||||
} else {
|
||||
test_setup(3) = raid_gen3_avx512gfni_raid;
|
||||
test_setup(4) = raid_gen4_avx512gfni_raid;
|
||||
test_setup(5) = raid_gen5_avx512gfni_raid;
|
||||
test_setup(6) = raid_gen6_avx512gfni_raid;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -31,6 +31,13 @@ int raid_test_sort(void);
|
|||
*/
|
||||
int raid_test_combo(void);
|
||||
|
||||
/**
|
||||
* Tests all RAID affine GFNI matrices against scalar GF multiplication.
|
||||
*
|
||||
* Returns 0 on success.
|
||||
*/
|
||||
int raid_test_gfaffine(void);
|
||||
|
||||
/**
|
||||
* Tests the Galois field and Cauchy matrix properties.
|
||||
*
|
||||
|
|
|
|||
|
|
@ -73,6 +73,13 @@ int main(void)
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("Test GFNI affine matrices...\n");
|
||||
if (raid_test_gfaffine() != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("Test Cauchy RAID parity generation with 1-%u data disks...\n", RAID_DATA_MAX);
|
||||
for (int i = 1; i <= RAID_DATA_MAX; ++i) {
|
||||
if (raid_test_par(RAID_MODE_CAUCHY_RAID, i, TEST_SIZE) != 0) {
|
||||
|
|
|
|||
|
|
@ -16,6 +16,13 @@ int main(void)
|
|||
|
||||
raid_init();
|
||||
|
||||
printf("Test GFNI affine matrices\n");
|
||||
if (raid_test_gfaffine() != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("Test Cauchy RAID\n");
|
||||
if (raid_test_poly(RAID_MODE_CAUCHY_RAID) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
|
|
|
|||
|
|
@ -208,6 +208,10 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
printf("%8s", "");
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -351,7 +355,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen2_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -363,7 +371,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen2_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -470,7 +482,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen3_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -482,7 +498,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen3_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -589,7 +609,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen4_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -601,7 +625,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen4_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -706,7 +734,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen5_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -718,7 +750,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen5_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -823,7 +859,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen6_avx2gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -835,7 +875,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
|
|||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
} else {
|
||||
printf("%8s", "");
|
||||
SPEED_START {
|
||||
raid_gen6_avx512gfni_raid(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
}
|
||||
fflush(stdout);
|
||||
}
|
||||
|
|
@ -1058,7 +1102,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v);
|
||||
raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1068,7 +1112,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v);
|
||||
raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1145,9 +1189,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx2gfni_aes);
|
||||
raid_gen_force(2, raid_gen2_avx2gfni_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v);
|
||||
raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1155,9 +1199,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx512gfni_aes);
|
||||
raid_gen_force(2, raid_gen2_avx512gfni_raid);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v);
|
||||
raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1224,8 +1268,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(3, id, ip, nd, size, v);
|
||||
raid_gen_force(3, raid_gen3_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1233,8 +1277,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(3, id, ip, nd, size, v);
|
||||
raid_gen_force(3, raid_gen3_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1305,8 +1349,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(4, id, ip, nd, size, v);
|
||||
raid_gen_force(4, raid_gen4_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1314,8 +1358,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(4, id, ip, nd, size, v);
|
||||
raid_gen_force(4, raid_gen4_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1386,8 +1430,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(5, id, ip, nd, size, v);
|
||||
raid_gen_force(5, raid_gen5_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1395,8 +1439,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(5, id, ip, nd, size, v);
|
||||
raid_gen_force(5, raid_gen5_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1467,8 +1511,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx2gfni_aes);
|
||||
raid_recX_avx2gfni(6, id, ip, nd, size, v);
|
||||
raid_gen_force(6, raid_gen6_avx2gfni_raid);
|
||||
raid_recX_avx2gfni_raid(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
@ -1476,8 +1520,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
|
|||
}
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx512gfni_aes);
|
||||
raid_recX_avx512gfni(6, id, ip, nd, size, v);
|
||||
raid_gen_force(6, raid_gen6_avx512gfni_raid);
|
||||
raid_recX_avx512gfni_raid(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue