From 17591ad932d5440fbaa44d09d005897c333bb3f9 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Wed, 19 Aug 2026 17:59:02 +0200 Subject: [PATCH] Add GFNI affine parity generation for RAID polynomial --- cmdline/selftest.c | 6 + cmdline/speed.c | 108 +++-- raid/README | 20 +- raid/gfni.c | 886 ++++++++++++++++++++++++++++++++++++++++-- raid/internal.h | 29 +- raid/mktables.c | 53 +++ raid/raid.h | 48 ++- raid/tables.c | 267 +++++++++++++ raid/test.c | 119 ++++-- raid/test.h | 7 + raid/test/fulltest.c | 7 + raid/test/selftest.c | 7 + raid/test/speedtest.c | 108 +++-- 13 files changed, 1523 insertions(+), 142 deletions(-) diff --git a/cmdline/selftest.c b/cmdline/selftest.c index d8436c4a..30b2db4d 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -1151,6 +1151,12 @@ void selftest(void) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } + if (raid_test_gfaffine() != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed GFNI AFFINE test\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } if (raid_test_par(RAID_MODE_VANDERMONDE_RAID, 32, 256) != 0) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Failed GEN Vandermonde RAID test\n"); diff --git a/cmdline/speed.c b/cmdline/speed.c index 4e94f8a0..5fbbda24 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -306,6 +306,10 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); } + if (raid_cpu_has_avx2gfni()) + printf("%8s", ""); + if (raid_cpu_has_avx512gfni()) + printf("%8s", ""); #endif #endif printf("\n"); @@ -449,7 +453,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -461,7 +469,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -570,7 +582,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -582,7 +598,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -691,7 +711,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -703,7 +727,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -809,7 +837,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -821,7 +853,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -927,7 +963,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -939,7 +979,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -1168,7 +1212,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) /* ensure to use same hardware in the delta step */ raid_gen_force(1, raid_gen1_avx2); /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1179,7 +1223,7 @@ void speed_rec(int nd, void** v, int size, int delta, int period) /* ensure to use same hardware in the delta step */ raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1261,9 +1305,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx2gfni_aes); + raid_gen_force(2, raid_gen2_avx2gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1272,9 +1316,9 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(2, raid_gen2_avx512gfni_aes); + raid_gen_force(2, raid_gen2_avx512gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1352,8 +1396,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx2gfni_aes); - raid_recX_avx2gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx2gfni_raid); + raid_recX_avx2gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1362,8 +1406,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(3, raid_gen3_avx512gfni_aes); - raid_recX_avx512gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx512gfni_raid); + raid_recX_avx512gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1442,8 +1486,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx2gfni_aes); - raid_recX_avx2gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx2gfni_raid); + raid_recX_avx2gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1452,8 +1496,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(4, raid_gen4_avx512gfni_aes); - raid_recX_avx512gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx512gfni_raid); + raid_recX_avx512gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1531,8 +1575,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx2gfni_aes); - raid_recX_avx2gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx2gfni_raid); + raid_recX_avx2gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1541,8 +1585,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(5, raid_gen5_avx512gfni_aes); - raid_recX_avx512gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx512gfni_raid); + raid_recX_avx512gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1620,8 +1664,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx2gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx2gfni_aes); - raid_recX_avx2gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx2gfni_raid); + raid_recX_avx2gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1630,8 +1674,8 @@ void speed_rec(int nd, void** v, int size, int delta, int period) if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ - raid_gen_force(6, raid_gen6_avx512gfni_aes); - raid_recX_avx512gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx512gfni_raid); + raid_recX_avx512gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); diff --git a/raid/README b/raid/README index 99584f5f..e39eb4b6 100644 --- a/raid/README +++ b/raid/README @@ -90,7 +90,9 @@ Operating Modes and Polynomials Uses Cauchy matrix generator with standard RAID polynomial 0x1d (x^8 + x^4 + x^3 + x^2 + 1, decimal 285, 0x11d) and primitive generator g = 2. Supports parities 1 to 6. Matches standard Linux RAID-6 (Anvin - method) for 2-parity arrays and ensures on-disk compatibility. + method) for 2-parity arrays and ensures on-disk compatibility. On CPUs + supporting Intel GFNI, generation and recovery are accelerated using + VGF2P8AFFINEQB with precomputed affine matrices for the RAID field. RAID_MODE_VANDERMONDE_RAID (Mode = 1) Legacy mode supporting up to 3 parities using standard polynomial 0x1d. @@ -99,12 +101,16 @@ Operating Modes and Polynomials RAID_MODE_CAUCHY_AES (Mode = 2) Uses Cauchy matrix with AES polynomial 0x1b (x^8 + x^4 + x^3 + x + 1, - 0x11b) and primitive generator g = 3. Enables hardware acceleration via - Intel GFNI instructions (vgf2p8mulb) on supported CPUs (Ice Lake or newer). - Parity data is incompatible with standard 0x1d polynomial arrays. + 0x11b) and primitive generator g = 3. On CPUs supporting Intel GFNI, + generation and recovery are accelerated directly using VGF2P8MULB. + This native GFNI multiplication is typically faster than the affine GFNI + implementation required by the standard RAID polynomial. Parity data is + incompatible with standard 0x11d polynomial arrays. Both Cauchy modes use the same Extended Cauchy construction with coefficients - based on powers of their respective primitive generator. + based on powers of their respective primitive generator. GFNI acceleration + is available for both fields: the RAID field uses affine transformations, + while the AES field uses the native GFNI multiplication instruction. Implementation Structure @@ -115,7 +121,7 @@ Implementation Structure module.c Algorithm registration, CPU detection, and dynamic dispatch. int.c Portable C fallback implementation (8/32/64-bit word loops). tables.c Precomputed GF(2^8) log/exp, multiplication, Cauchy tables. - mktables.c Table generator script for GF(2^8) field matrices. + mktables.c Generator for GF(2^8), Cauchy, and GFNI affine tables. internal.h Private header, assertion macros (BUG_ON), alignment helpers. Platform-Specific Acceleration: @@ -123,7 +129,7 @@ Implementation Structure ssse3.c x86 SSSE3 (128-bit pshufb nibble-split multiplication) avx2.c x86 AVX2 (256-bit vectorization) avx512.c x86 AVX-512BW (512-bit vectorization) - gfni.c x86 GFNI hardware Galois field vector multiplication + gfni.c x86 GFNI generation/recovery (RAID affine and AES multiply) neon.c ARM NEON (128-bit SIMD vectorization) diff --git a/raid/gfni.c b/raid/gfni.c index bdea56c8..ba323293 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -6,6 +6,385 @@ #include "cpu.h" #ifdef CONFIG_X86_64 +/* + * GENX AVX2 GFNI implementation for the RAID polynomial with Horner evaluation. + * + * Q is evaluated with Horner's rule using the fixed multiply-by-2 affine + * matrix. The remaining Cauchy parities use their per-disk coefficients. + * + * Only supported up to GEN5 (np <= 5), where ymm11 remains available for the + * fixed Q multiply-by-2 matrix. + */ +static __always_inline void raid_genX_avx2gfni_horner_raid(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + l = nd - 1; + + raid_avx_begin(); + + /* Fixed GF(2^8) multiply-by-2 matrix used by the Q Horner recurrence. */ + asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[2][0])); + + for (i = 0; i < size; i += 64) { + /* Start P and Q from the last data disk, as in Horner evaluation. */ + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm12,%ymm0"); + asm volatile ("vmovdqa %ymm13,%ymm1"); + asm volatile ("vmovdqa %ymm12,%ymm2"); + asm volatile ("vmovdqa %ymm13,%ymm3"); + + /* + * Initialize R/S/T with the contribution of the last disk. + * Unlike P and Q, their coefficients are not powers forming a + * fixed Horner recurrence. + */ + if (np >= 3) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][l]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm5"); + } + if (np >= 4) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][l]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm7"); + } + if (np >= 5) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][l]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm8"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm9"); + } + + /* Process all remaining disks except D0. */ + for (d = l - 1; d >= 1; --d) { + /* Q = 2 * Q. */ + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm2"); + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm3"); + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32])); + + /* P ^= D[d], Q ^= D[d]. */ + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + + if (np >= 3) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + } + if (np >= 4) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + } + if (np >= 5) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + } + } + + /* + * Final Horner step with D0. + * + * All Cauchy rows are normalized to coefficient 1 for D0, + * so R/S/T require only XOR and no affine multiplication. + */ + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm2"); + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm3"); + + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + + if (np >= 3) { + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + } + if (np >= 4) { + asm volatile ("vpxor %ymm12,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm13,%ymm7,%ymm7"); + } + if (np >= 5) { + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (np >= 3) { + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + if (np >= 4) { + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + if (np >= 5) { + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + } + + raid_avx_end(); +} + +/* + * GENX AVX2 GFNI implementation + */ +static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + if (np >= 3) { + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); + } + if (np >= 4) { + asm volatile ("vmovdqa %ymm0,%ymm6"); + asm volatile ("vmovdqa %ymm1,%ymm7"); + } + if (np >= 5) { + asm volatile ("vmovdqa %ymm0,%ymm8"); + asm volatile ("vmovdqa %ymm1,%ymm9"); + } + if (np >= 6) { + asm volatile ("vmovdqa %ymm0,%ymm10"); + asm volatile ("vmovdqa %ymm1,%ymm11"); + } + + /* all other disks */ + for (d = 1; d < nd; ++d) { + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[1][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + if (np >= 3) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + } + if (np >= 4) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + } + if (np >= 5) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + } + if (np >= 6) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (np >= 3) { + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + if (np >= 4) { + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + if (np >= 5) { + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + if (np >= 6) { + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } + } + + raid_avx_end(); +} + +/* + * GENX AVX512 GFNI implementation for the RAID polynomial. + * + * Q is evaluated with Horner's rule using the fixed multiply-by-2 affine + * matrix. The remaining Cauchy parities use their per-disk coefficients. + */ +static __always_inline void raid_genX_avx512gfni_horner_raid(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d, l; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + l = nd - 1; + + raid_avx_begin(); + + /* Fixed GF(2^8) multiply-by-2 matrix used by the Q Horner recurrence. */ + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfaffine_raid[2][0])); + + for (i = 0; i < size; i += 64) { + /* Start P and Q from the last data disk. */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i])); + asm volatile ("vmovdqa64 %zmm6,%zmm0"); + asm volatile ("vmovdqa64 %zmm6,%zmm1"); + + /* Initialize R/S/T/U with the contribution of the last disk. */ + if (np >= 3) { + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][l]][0])); + asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm6,%zmm2"); + } + if (np >= 4) { + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][l]][0])); + asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm6,%zmm3"); + } + if (np >= 5) { + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][l]][0])); + asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm6,%zmm4"); + } + if (np >= 6) { + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][l]][0])); + asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm6,%zmm5"); + } + + /* Process all remaining disks except D0. */ + for (d = l - 1; d >= 1; --d) { + /* Q = 2 * Q. */ + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm1,%zmm1"); + + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + /* P ^= D[d], Q ^= D[d]. */ + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); + + if (np >= 3) + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0])); + if (np >= 4) + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0])); + if (np >= 5) + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0])); + if (np >= 6) + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0])); + + if (np >= 3) { + asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm6,%zmm12"); + asm volatile ("vpxorq %zmm12,%zmm2,%zmm2"); + } + if (np >= 4) { + asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm6,%zmm13"); + asm volatile ("vpxorq %zmm13,%zmm3,%zmm3"); + } + if (np >= 5) { + asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm6,%zmm14"); + asm volatile ("vpxorq %zmm14,%zmm4,%zmm4"); + } + if (np >= 6) { + asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm6,%zmm15"); + asm volatile ("vpxorq %zmm15,%zmm5,%zmm5"); + } + } + + /* + * Final Horner step with D0. + * + * All Cauchy rows have coefficient 1 for D0. + */ + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm1,%zmm1"); + + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); + + if (np >= 3) + asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); + if (np >= 4) + asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); + if (np >= 5) + asm volatile ("vpxorq %zmm6,%zmm4,%zmm4"); + if (np >= 6) + asm volatile ("vpxorq %zmm6,%zmm5,%zmm5"); + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } + + raid_avx_end(); +} /* * GENX AVX2 GFNI implementation @@ -198,6 +577,86 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void * raid_avx_end(); } +/* + * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation + */ +void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_horner_raid(nd, size, vv, 2); +} + +/* + * GEN2 (RAID6 with powers of 2) GFNI implementation + */ +void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_horner_raid(nd, size, vv, 2); +} + +/* + * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_horner_raid(nd, size, vv, 3); +} + +/* + * GEN3 (triple parity with Cauchy matrix) GFNI implementation + */ +void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_horner_raid(nd, size, vv, 3); +} + +/* + * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_horner_raid(nd, size, vv, 4); +} + +/* + * GEN4 (quad parity with Cauchy matrix) GFNI implementation + */ +void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_horner_raid(nd, size, vv, 4); +} + +/* + * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_horner_raid(nd, size, vv, 5); +} + +/* + * GEN5 (penta parity with Cauchy matrix) GFNI implementation + */ +void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_horner_raid(nd, size, vv, 5); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_raid(nd, size, vv, 6); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) GFNI implementation + */ +void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_horner_raid(nd, size, vv, 6); +} + /* * GEN2 Cauchy AVX2 GFNI implementation */ @@ -281,7 +740,362 @@ void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv) /* * RAID recovering for one disk AVX2 GFNI implementation */ -void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastq %0,%%ymm4" : : "m" (raid_gfaffine_raid[V][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm0,%ymm0"); + asm volatile ("vgf2p8affineqb $0,%ymm4,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for one disk GFNI implementation + */ +void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastq %0,%%zmm4" : : "m" (raid_gfaffine_raid[V][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vgf2p8affineqb $0,%zmm4,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for two disks AVX2 GFNI implementation + */ +void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + asm volatile ("vpbroadcastq %0,%%ymm10" : : "m" (raid_gfaffine_raid[V[0]][0])); + asm volatile ("vpbroadcastq %0,%%ymm11" : : "m" (raid_gfaffine_raid[V[1]][0])); + asm volatile ("vpbroadcastq %0,%%ymm12" : : "m" (raid_gfaffine_raid[V[2]][0])); + asm volatile ("vpbroadcastq %0,%%ymm13" : : "m" (raid_gfaffine_raid[V[3]][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32])); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32])); + + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + + asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm0,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm10,%ymm1,%ymm5"); + + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm2,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm4,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm11,%ymm3,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); + + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32])); + + asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm0,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm12,%ymm1,%ymm7"); + + asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm2,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm6,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm13,%ymm3,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); + + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); + asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32])); + } + + raid_avx_end(); +} + +/* + * RAID recovering for two disks GFNI implementation + */ +void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[V[0]][0])); + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[V[1]][0])); + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[V[2]][0])); + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[V[3]][0])); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); + asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); + + asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm0,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm1,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm0,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + + asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm1,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); + } + + raid_avx_end(); +} + +/* + * RAID recovering AVX2 GFNI implementation + */ +void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); + for (k = 0; k < N; ++k) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32])); + + asm volatile ("vpbroadcastq %0,%%ymm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0])); + + asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm4,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm0,%ymm0"); + + asm volatile ("vgf2p8affineqb $0,%ymm2,%ymm5,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + } + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32])); + } + } + + raid_avx_end(); +} + +/* + * RAID recovering GFNI implementation + */ +void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX * 64 + 64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); + for (k = 0; k < N; ++k) { + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64])); + asm volatile ("vpbroadcastq %0,%%zmm2" : : "m" (raid_gfaffine_raid[V[j * N + k]][0])); + asm volatile ("vgf2p8affineqb $0,%zmm2,%zmm4,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + } + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} + +/* + * AES recovering for one disk AVX2 GFNI implementation + */ +void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p, *pa; @@ -328,9 +1142,9 @@ void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv } /* - * RAID recovering for one disk GFNI implementation + * AES recovering for one disk GFNI implementation */ -void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; uint8_t *p, *pa; @@ -368,9 +1182,9 @@ void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } /* - * RAID recovering for two disks AVX2 GFNI implementation + * AES recovering for two disks AVX2 GFNI implementation */ -void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; const int N = 2; @@ -445,9 +1259,9 @@ void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv } /* - * RAID recovering for two disks GFNI implementation + * AES recovering for two disks GFNI implementation */ -void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; const int N = 2; @@ -509,9 +1323,9 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } /* - * RAID recovering AVX2 GFNI implementation + * AES recovering AVX2 GFNI implementation */ -void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; int N = nr; @@ -579,9 +1393,9 @@ void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv } /* - * RAID recovering GFNI implementation + * AES recovering GFNI implementation */ -void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv) { uint8_t **v = (uint8_t **)vv; int N = nr; @@ -636,18 +1450,31 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** void raid_register_avx2gfni(void) { if (raid_cpu_has_avx2gfni()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_raid, RAID_POLY_RAID); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni_aes, RAID_POLY_AES); } } @@ -655,18 +1482,31 @@ void raid_register_avx512gfni(void) { if (raid_cpu_has_avx512gfni()) { if (!raid_cpu_has_slow_avx512()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_aes, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); - raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_raid, RAID_POLY_RAID); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni512", raid_rec1_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_rec2_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_recX_avx512gfni_aes, RAID_POLY_AES); } } } diff --git a/raid/internal.h b/raid/internal.h index af43c4dc..255e67c4 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -104,6 +104,8 @@ void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv); void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen2_avx512bw(int nd, size_t size, void **vv); +void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv); void raid_genz_int32_raid(int nd, size_t size, void **vv); @@ -119,6 +121,8 @@ void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen3_avx512bw(int nd, size_t size, void **vv); +void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen4_int8(int nd, size_t size, void **vv); @@ -129,6 +133,8 @@ void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen4_avx512bw(int nd, size_t size, void **vv); +void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen5_int8(int nd, size_t size, void **vv); @@ -138,6 +144,8 @@ void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen5_avx512bw(int nd, size_t size, void **vv); +void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen6_int8(int nd, size_t size, void **vv); @@ -147,6 +155,8 @@ void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen6_avx512bw(int nd, size_t size, void **vv); +void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv); void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -161,12 +171,18 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); -void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx2gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx2gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx512gfni_raid(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx512gfni_aes(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_gen1_neon(int nd, size_t size, void **vv); void raid_gen2_neon_raid(int nd, size_t size, void **vv); void raid_gen2_neon_aes(int nd, size_t size, void **vv); @@ -325,6 +341,7 @@ extern const uint8_t raid_gfexp_raid[256] __aligned(256); extern const uint8_t raid_gfinv_raid[256] __aligned(256); extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256); extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256); +extern const uint8_t raid_gfaffine_raid[256][8] __aligned(256); #if defined(CONFIG_X86) || defined(CONFIG_NEON) extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256); extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256); diff --git a/raid/mktables.c b/raid/mktables.c index 0a201660..ef1fc63d 100644 --- a/raid/mktables.c +++ b/raid/mktables.c @@ -38,6 +38,34 @@ static uint8_t raid_gfmul(uint8_t poly, uint8_t a, uint8_t b) return v; } +/** + * Setup the GFNI affine matrix for multiplication by a constant. + * + * VGF2P8MULB is fixed to the AES polynomial 0x11b, while RAID parity is + * defined over GF(2^8) modulo 0x11d. VGF2P8AFFINEQB can implement the + * multiplication because multiplication by a constant is GF(2)-linear. + * + * VGF2P8AFFINEQB stores the row producing output bit j in byte 7-j. + * Within each row, bit k selects input bit k. + */ +static void set_affine(uint8_t poly, uint8_t c, uint8_t *matrix) +{ + int j, k; + + for (j = 0; j < 8; ++j) { + uint8_t row = 0; + + for (k = 0; k < 8; ++k) { + uint8_t v = raid_gfmul(poly, c, (uint8_t)(1U << k)); + + if ((v & (1U << j)) != 0) + row |= (uint8_t)(1U << k); + } + + matrix[7 - j] = row; + } +} + /** * Inversion (1/a) in GF(2^8). */ @@ -231,6 +259,31 @@ void tables(uint8_t poly, uint8_t generator, const char *tag) } printf("};\n\n"); + if (poly == RAID_POLY_RAID) { + uint8_t affine[8]; + + printf("/**\n"); + printf(" * GFNI affine matrices for multiplication in GF(2^8)/0x11d.\n"); + printf(" *\n"); + printf(" * VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane.\n"); + printf(" * Byte 7-j contains the row producing output bit j, and bit k\n"); + printf(" * of that byte selects input bit k. The instruction uses imm8=0.\n"); + printf(" */\n"); + printf("const uint8_t __aligned(256) raid_gfaffine_raid[256][8] =\n"); + printf("{\n"); + for (i = 0; i < 256; ++i) { + set_affine(poly, (uint8_t)i, affine); + printf("\t{ "); + for (j = 0; j < 8; ++j) { + printf("0x%02x", (unsigned)affine[j]); + if (j != 7) + printf(", "); + } + printf(" },\n"); + } + printf("};\n\n"); + } + /* generator^a */ printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag); printf("{\n"); diff --git a/raid/raid.h b/raid/raid.h index 8906d5bb..7e8a966a 100644 --- a/raid/raid.h +++ b/raid/raid.h @@ -21,26 +21,32 @@ * The polynomial used by the original RAID-6 specification and the * Linux kernel RAID implementation, used with primitive generator g=2. * Choosing this polynomial ensures on-disk compatibility with existing - * RAID arrays and interchange with other RAID implementations. Use this - * when reading or writing data that may be accessed by other software or - * hardware RAID controllers. + * RAID arrays and interchange with other RAID implementations. + * + * On CPUs supporting Intel GFNI (Galois Field New Instructions), + * generation and recovery are accelerated with vgf2p8affineqb. + * Multiplication by each GF(2^8) coefficient is represented as a + * precomputed GF(2) 8x8 affine matrix for the 0x11d field. * * 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial * * The polynomial used by the AES encryption standard (0x11b including - * the x^8 term), used with primitive generator g=3. Choosing this - * polynomial enables the use of the Intel GFNI (Galois Field New - * Instructions) extension, specifically the vgf2p8mulb instruction, - * which performs GF(2^8) multiplication natively in hardware using - * exactly this polynomial. This eliminates the nibble-split pshufb - * multiply sequence (vpsrlw + vpandq + 2x vpshufb + vpxorq) and the - * conditional multiply-by-2 sequence (vpcmpgtb + vpmovm2b + vpaddb + - * vpandq + vpxorq) used in the AVX-512BW path, replacing each with a - * single vgf2p8mulb instruction. On CPUs supporting GFNI (Ice Lake and - * later), this yields a significant throughput improvement. The - * trade-off is that parity data is not compatible with arrays using - * the standard 0x1d polynomial (g=2), so this option requires that all - * data be generated and recovered using the same operating mode. + * the x^8 term), used with primitive generator g=3. + * + * On CPUs supporting Intel GFNI, generation and recovery can use the + * vgf2p8mulb instruction, which performs GF(2^8) multiplication directly + * in hardware using this polynomial. This native multiplication is + * typically faster than the affine GFNI implementation required for the + * standard RAID polynomial. + * + * The trade-off is that parity data is not compatible with arrays using + * the standard 0x11d polynomial (g=2), so all data must be generated and + * recovered using the same operating mode. + * + * Both Cauchy modes can therefore use GFNI acceleration. The standard RAID + * mode preserves compatibility by using affine GFNI transformations, while + * the AES mode trades compatibility for the typically faster native GFNI + * multiplication instruction. */ /** @@ -51,7 +57,8 @@ /** * Default RAID mode supporting up to 6 parities using standard polynomial 0x1d and generator g=2. * - * Provides high performance on modern CPUs with SSSE3, AVX2, AVX-512, or NEON support. + * Provides high performance on modern CPUs with SSSE3, AVX2, AVX-512, GFNI, or NEON support. + * On GFNI CPUs, the standard RAID field is accelerated using affine transformations. * Ensures maximum compatibility with standard Linux RAID arrays. * * This is the default mode set after calling raid_init(). @@ -72,8 +79,11 @@ * RAID mode supporting up to 6 parities using AES polynomial 0x1b (0x11b) and primitive generator g=3. * * It has slightly lower performance than RAID_MODE_CAUCHY_RAID on CPUs without GFNI due - * to the additional cost of generator g=3, but achieves significantly higher throughput on - * CPUs supporting GFNI hardware acceleration. + * to the additional cost of generator g=3. On CPUs supporting GFNI, it uses native + * VGF2P8MULB multiplication and is typically faster than the affine GFNI implementation + * required by RAID_MODE_CAUCHY_RAID. + * + * Parity data is incompatible with RAID_MODE_CAUCHY_RAID. */ #define RAID_MODE_CAUCHY_AES 2 diff --git a/raid/tables.c b/raid/tables.c index 9d89ea82..932599e0 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -8713,6 +8713,273 @@ const uint8_t __aligned(256) raid_gfmul_raid[256][256] = }, }; +/** + * GFNI affine matrices for multiplication in GF(2^8)/0x11d. + * + * VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane. + * Byte 7-j contains the row producing output bit j, and bit k + * of that byte selects input bit k. The instruction uses imm8=0. + */ +const uint8_t __aligned(256) raid_gfaffine_raid[256][8] = +{ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }, + { 0x40, 0x20, 0x10, 0x88, 0x84, 0x82, 0x01, 0x80 }, + { 0xc0, 0x60, 0x30, 0x98, 0x8c, 0x86, 0x03, 0x81 }, + { 0x20, 0x10, 0x88, 0xc4, 0xc2, 0x41, 0x80, 0x40 }, + { 0xa0, 0x50, 0xa8, 0xd4, 0xca, 0x45, 0x82, 0x41 }, + { 0x60, 0x30, 0x98, 0x4c, 0x46, 0xc3, 0x81, 0xc0 }, + { 0xe0, 0x70, 0xb8, 0x5c, 0x4e, 0xc7, 0x83, 0xc1 }, + { 0x10, 0x88, 0xc4, 0xe2, 0x61, 0xa0, 0x40, 0x20 }, + { 0x90, 0xc8, 0xe4, 0xf2, 0x69, 0xa4, 0x42, 0x21 }, + { 0x50, 0xa8, 0xd4, 0x6a, 0xe5, 0x22, 0x41, 0xa0 }, + { 0xd0, 0xe8, 0xf4, 0x7a, 0xed, 0x26, 0x43, 0xa1 }, + { 0x30, 0x98, 0x4c, 0x26, 0xa3, 0xe1, 0xc0, 0x60 }, + { 0xb0, 0xd8, 0x6c, 0x36, 0xab, 0xe5, 0xc2, 0x61 }, + { 0x70, 0xb8, 0x5c, 0xae, 0x27, 0x63, 0xc1, 0xe0 }, + { 0xf0, 0xf8, 0x7c, 0xbe, 0x2f, 0x67, 0xc3, 0xe1 }, + { 0x88, 0xc4, 0xe2, 0x71, 0xb0, 0x50, 0x20, 0x10 }, + { 0x08, 0x84, 0xc2, 0x61, 0xb8, 0x54, 0x22, 0x11 }, + { 0xc8, 0xe4, 0xf2, 0xf9, 0x34, 0xd2, 0x21, 0x90 }, + { 0x48, 0xa4, 0xd2, 0xe9, 0x3c, 0xd6, 0x23, 0x91 }, + { 0xa8, 0xd4, 0x6a, 0xb5, 0x72, 0x11, 0xa0, 0x50 }, + { 0x28, 0x94, 0x4a, 0xa5, 0x7a, 0x15, 0xa2, 0x51 }, + { 0xe8, 0xf4, 0x7a, 0x3d, 0xf6, 0x93, 0xa1, 0xd0 }, + { 0x68, 0xb4, 0x5a, 0x2d, 0xfe, 0x97, 0xa3, 0xd1 }, + { 0x98, 0x4c, 0x26, 0x93, 0xd1, 0xf0, 0x60, 0x30 }, + { 0x18, 0x0c, 0x06, 0x83, 0xd9, 0xf4, 0x62, 0x31 }, + { 0xd8, 0x6c, 0x36, 0x1b, 0x55, 0x72, 0x61, 0xb0 }, + { 0x58, 0x2c, 0x16, 0x0b, 0x5d, 0x76, 0x63, 0xb1 }, + { 0xb8, 0x5c, 0xae, 0x57, 0x13, 0xb1, 0xe0, 0x70 }, + { 0x38, 0x1c, 0x8e, 0x47, 0x1b, 0xb5, 0xe2, 0x71 }, + { 0xf8, 0x7c, 0xbe, 0xdf, 0x97, 0x33, 0xe1, 0xf0 }, + { 0x78, 0x3c, 0x9e, 0xcf, 0x9f, 0x37, 0xe3, 0xf1 }, + { 0xc4, 0xe2, 0x71, 0x38, 0xd8, 0xa8, 0x10, 0x88 }, + { 0x44, 0xa2, 0x51, 0x28, 0xd0, 0xac, 0x12, 0x89 }, + { 0x84, 0xc2, 0x61, 0xb0, 0x5c, 0x2a, 0x11, 0x08 }, + { 0x04, 0x82, 0x41, 0xa0, 0x54, 0x2e, 0x13, 0x09 }, + { 0xe4, 0xf2, 0xf9, 0xfc, 0x1a, 0xe9, 0x90, 0xc8 }, + { 0x64, 0xb2, 0xd9, 0xec, 0x12, 0xed, 0x92, 0xc9 }, + { 0xa4, 0xd2, 0xe9, 0x74, 0x9e, 0x6b, 0x91, 0x48 }, + { 0x24, 0x92, 0xc9, 0x64, 0x96, 0x6f, 0x93, 0x49 }, + { 0xd4, 0x6a, 0xb5, 0xda, 0xb9, 0x08, 0x50, 0xa8 }, + { 0x54, 0x2a, 0x95, 0xca, 0xb1, 0x0c, 0x52, 0xa9 }, + { 0x94, 0x4a, 0xa5, 0x52, 0x3d, 0x8a, 0x51, 0x28 }, + { 0x14, 0x0a, 0x85, 0x42, 0x35, 0x8e, 0x53, 0x29 }, + { 0xf4, 0x7a, 0x3d, 0x1e, 0x7b, 0x49, 0xd0, 0xe8 }, + { 0x74, 0x3a, 0x1d, 0x0e, 0x73, 0x4d, 0xd2, 0xe9 }, + { 0xb4, 0x5a, 0x2d, 0x96, 0xff, 0xcb, 0xd1, 0x68 }, + { 0x34, 0x1a, 0x0d, 0x86, 0xf7, 0xcf, 0xd3, 0x69 }, + { 0x4c, 0x26, 0x93, 0x49, 0x68, 0xf8, 0x30, 0x98 }, + { 0xcc, 0x66, 0xb3, 0x59, 0x60, 0xfc, 0x32, 0x99 }, + { 0x0c, 0x06, 0x83, 0xc1, 0xec, 0x7a, 0x31, 0x18 }, + { 0x8c, 0x46, 0xa3, 0xd1, 0xe4, 0x7e, 0x33, 0x19 }, + { 0x6c, 0x36, 0x1b, 0x8d, 0xaa, 0xb9, 0xb0, 0xd8 }, + { 0xec, 0x76, 0x3b, 0x9d, 0xa2, 0xbd, 0xb2, 0xd9 }, + { 0x2c, 0x16, 0x0b, 0x05, 0x2e, 0x3b, 0xb1, 0x58 }, + { 0xac, 0x56, 0x2b, 0x15, 0x26, 0x3f, 0xb3, 0x59 }, + { 0x5c, 0xae, 0x57, 0xab, 0x09, 0x58, 0x70, 0xb8 }, + { 0xdc, 0xee, 0x77, 0xbb, 0x01, 0x5c, 0x72, 0xb9 }, + { 0x1c, 0x8e, 0x47, 0x23, 0x8d, 0xda, 0x71, 0x38 }, + { 0x9c, 0xce, 0x67, 0x33, 0x85, 0xde, 0x73, 0x39 }, + { 0x7c, 0xbe, 0xdf, 0x6f, 0xcb, 0x19, 0xf0, 0xf8 }, + { 0xfc, 0xfe, 0xff, 0x7f, 0xc3, 0x1d, 0xf2, 0xf9 }, + { 0x3c, 0x9e, 0xcf, 0xe7, 0x4f, 0x9b, 0xf1, 0x78 }, + { 0xbc, 0xde, 0xef, 0xf7, 0x47, 0x9f, 0xf3, 0x79 }, + { 0xe2, 0x71, 0x38, 0x1c, 0x6c, 0xd4, 0x88, 0xc4 }, + { 0x62, 0x31, 0x18, 0x0c, 0x64, 0xd0, 0x8a, 0xc5 }, + { 0xa2, 0x51, 0x28, 0x94, 0xe8, 0x56, 0x89, 0x44 }, + { 0x22, 0x11, 0x08, 0x84, 0xe0, 0x52, 0x8b, 0x45 }, + { 0xc2, 0x61, 0xb0, 0xd8, 0xae, 0x95, 0x08, 0x84 }, + { 0x42, 0x21, 0x90, 0xc8, 0xa6, 0x91, 0x0a, 0x85 }, + { 0x82, 0x41, 0xa0, 0x50, 0x2a, 0x17, 0x09, 0x04 }, + { 0x02, 0x01, 0x80, 0x40, 0x22, 0x13, 0x0b, 0x05 }, + { 0xf2, 0xf9, 0xfc, 0xfe, 0x0d, 0x74, 0xc8, 0xe4 }, + { 0x72, 0xb9, 0xdc, 0xee, 0x05, 0x70, 0xca, 0xe5 }, + { 0xb2, 0xd9, 0xec, 0x76, 0x89, 0xf6, 0xc9, 0x64 }, + { 0x32, 0x99, 0xcc, 0x66, 0x81, 0xf2, 0xcb, 0x65 }, + { 0xd2, 0xe9, 0x74, 0x3a, 0xcf, 0x35, 0x48, 0xa4 }, + { 0x52, 0xa9, 0x54, 0x2a, 0xc7, 0x31, 0x4a, 0xa5 }, + { 0x92, 0xc9, 0x64, 0xb2, 0x4b, 0xb7, 0x49, 0x24 }, + { 0x12, 0x89, 0x44, 0xa2, 0x43, 0xb3, 0x4b, 0x25 }, + { 0x6a, 0xb5, 0xda, 0x6d, 0xdc, 0x84, 0xa8, 0xd4 }, + { 0xea, 0xf5, 0xfa, 0x7d, 0xd4, 0x80, 0xaa, 0xd5 }, + { 0x2a, 0x95, 0xca, 0xe5, 0x58, 0x06, 0xa9, 0x54 }, + { 0xaa, 0xd5, 0xea, 0xf5, 0x50, 0x02, 0xab, 0x55 }, + { 0x4a, 0xa5, 0x52, 0xa9, 0x1e, 0xc5, 0x28, 0x94 }, + { 0xca, 0xe5, 0x72, 0xb9, 0x16, 0xc1, 0x2a, 0x95 }, + { 0x0a, 0x85, 0x42, 0x21, 0x9a, 0x47, 0x29, 0x14 }, + { 0x8a, 0xc5, 0x62, 0x31, 0x92, 0x43, 0x2b, 0x15 }, + { 0x7a, 0x3d, 0x1e, 0x8f, 0xbd, 0x24, 0xe8, 0xf4 }, + { 0xfa, 0x7d, 0x3e, 0x9f, 0xb5, 0x20, 0xea, 0xf5 }, + { 0x3a, 0x1d, 0x0e, 0x07, 0x39, 0xa6, 0xe9, 0x74 }, + { 0xba, 0x5d, 0x2e, 0x17, 0x31, 0xa2, 0xeb, 0x75 }, + { 0x5a, 0x2d, 0x96, 0x4b, 0x7f, 0x65, 0x68, 0xb4 }, + { 0xda, 0x6d, 0xb6, 0x5b, 0x77, 0x61, 0x6a, 0xb5 }, + { 0x1a, 0x0d, 0x86, 0xc3, 0xfb, 0xe7, 0x69, 0x34 }, + { 0x9a, 0x4d, 0xa6, 0xd3, 0xf3, 0xe3, 0x6b, 0x35 }, + { 0x26, 0x93, 0x49, 0x24, 0xb4, 0x7c, 0x98, 0x4c }, + { 0xa6, 0xd3, 0x69, 0x34, 0xbc, 0x78, 0x9a, 0x4d }, + { 0x66, 0xb3, 0x59, 0xac, 0x30, 0xfe, 0x99, 0xcc }, + { 0xe6, 0xf3, 0x79, 0xbc, 0x38, 0xfa, 0x9b, 0xcd }, + { 0x06, 0x83, 0xc1, 0xe0, 0x76, 0x3d, 0x18, 0x0c }, + { 0x86, 0xc3, 0xe1, 0xf0, 0x7e, 0x39, 0x1a, 0x0d }, + { 0x46, 0xa3, 0xd1, 0x68, 0xf2, 0xbf, 0x19, 0x8c }, + { 0xc6, 0xe3, 0xf1, 0x78, 0xfa, 0xbb, 0x1b, 0x8d }, + { 0x36, 0x1b, 0x8d, 0xc6, 0xd5, 0xdc, 0xd8, 0x6c }, + { 0xb6, 0x5b, 0xad, 0xd6, 0xdd, 0xd8, 0xda, 0x6d }, + { 0x76, 0x3b, 0x9d, 0x4e, 0x51, 0x5e, 0xd9, 0xec }, + { 0xf6, 0x7b, 0xbd, 0x5e, 0x59, 0x5a, 0xdb, 0xed }, + { 0x16, 0x0b, 0x05, 0x02, 0x17, 0x9d, 0x58, 0x2c }, + { 0x96, 0x4b, 0x25, 0x12, 0x1f, 0x99, 0x5a, 0x2d }, + { 0x56, 0x2b, 0x15, 0x8a, 0x93, 0x1f, 0x59, 0xac }, + { 0xd6, 0x6b, 0x35, 0x9a, 0x9b, 0x1b, 0x5b, 0xad }, + { 0xae, 0x57, 0xab, 0x55, 0x04, 0x2c, 0xb8, 0x5c }, + { 0x2e, 0x17, 0x8b, 0x45, 0x0c, 0x28, 0xba, 0x5d }, + { 0xee, 0x77, 0xbb, 0xdd, 0x80, 0xae, 0xb9, 0xdc }, + { 0x6e, 0x37, 0x9b, 0xcd, 0x88, 0xaa, 0xbb, 0xdd }, + { 0x8e, 0x47, 0x23, 0x91, 0xc6, 0x6d, 0x38, 0x1c }, + { 0x0e, 0x07, 0x03, 0x81, 0xce, 0x69, 0x3a, 0x1d }, + { 0xce, 0x67, 0x33, 0x19, 0x42, 0xef, 0x39, 0x9c }, + { 0x4e, 0x27, 0x13, 0x09, 0x4a, 0xeb, 0x3b, 0x9d }, + { 0xbe, 0xdf, 0x6f, 0xb7, 0x65, 0x8c, 0xf8, 0x7c }, + { 0x3e, 0x9f, 0x4f, 0xa7, 0x6d, 0x88, 0xfa, 0x7d }, + { 0xfe, 0xff, 0x7f, 0x3f, 0xe1, 0x0e, 0xf9, 0xfc }, + { 0x7e, 0xbf, 0x5f, 0x2f, 0xe9, 0x0a, 0xfb, 0xfd }, + { 0x9e, 0xcf, 0xe7, 0x73, 0xa7, 0xcd, 0x78, 0x3c }, + { 0x1e, 0x8f, 0xc7, 0x63, 0xaf, 0xc9, 0x7a, 0x3d }, + { 0xde, 0xef, 0xf7, 0xfb, 0x23, 0x4f, 0x79, 0xbc }, + { 0x5e, 0xaf, 0xd7, 0xeb, 0x2b, 0x4b, 0x7b, 0xbd }, + { 0x71, 0x38, 0x1c, 0x8e, 0x36, 0x6a, 0xc4, 0xe2 }, + { 0xf1, 0x78, 0x3c, 0x9e, 0x3e, 0x6e, 0xc6, 0xe3 }, + { 0x31, 0x18, 0x0c, 0x06, 0xb2, 0xe8, 0xc5, 0x62 }, + { 0xb1, 0x58, 0x2c, 0x16, 0xba, 0xec, 0xc7, 0x63 }, + { 0x51, 0x28, 0x94, 0x4a, 0xf4, 0x2b, 0x44, 0xa2 }, + { 0xd1, 0x68, 0xb4, 0x5a, 0xfc, 0x2f, 0x46, 0xa3 }, + { 0x11, 0x08, 0x84, 0xc2, 0x70, 0xa9, 0x45, 0x22 }, + { 0x91, 0x48, 0xa4, 0xd2, 0x78, 0xad, 0x47, 0x23 }, + { 0x61, 0xb0, 0xd8, 0x6c, 0x57, 0xca, 0x84, 0xc2 }, + { 0xe1, 0xf0, 0xf8, 0x7c, 0x5f, 0xce, 0x86, 0xc3 }, + { 0x21, 0x90, 0xc8, 0xe4, 0xd3, 0x48, 0x85, 0x42 }, + { 0xa1, 0xd0, 0xe8, 0xf4, 0xdb, 0x4c, 0x87, 0x43 }, + { 0x41, 0xa0, 0x50, 0xa8, 0x95, 0x8b, 0x04, 0x82 }, + { 0xc1, 0xe0, 0x70, 0xb8, 0x9d, 0x8f, 0x06, 0x83 }, + { 0x01, 0x80, 0x40, 0x20, 0x11, 0x09, 0x05, 0x02 }, + { 0x81, 0xc0, 0x60, 0x30, 0x19, 0x0d, 0x07, 0x03 }, + { 0xf9, 0xfc, 0xfe, 0xff, 0x86, 0x3a, 0xe4, 0xf2 }, + { 0x79, 0xbc, 0xde, 0xef, 0x8e, 0x3e, 0xe6, 0xf3 }, + { 0xb9, 0xdc, 0xee, 0x77, 0x02, 0xb8, 0xe5, 0x72 }, + { 0x39, 0x9c, 0xce, 0x67, 0x0a, 0xbc, 0xe7, 0x73 }, + { 0xd9, 0xec, 0x76, 0x3b, 0x44, 0x7b, 0x64, 0xb2 }, + { 0x59, 0xac, 0x56, 0x2b, 0x4c, 0x7f, 0x66, 0xb3 }, + { 0x99, 0xcc, 0x66, 0xb3, 0xc0, 0xf9, 0x65, 0x32 }, + { 0x19, 0x8c, 0x46, 0xa3, 0xc8, 0xfd, 0x67, 0x33 }, + { 0xe9, 0x74, 0x3a, 0x1d, 0xe7, 0x9a, 0xa4, 0xd2 }, + { 0x69, 0x34, 0x1a, 0x0d, 0xef, 0x9e, 0xa6, 0xd3 }, + { 0xa9, 0x54, 0x2a, 0x95, 0x63, 0x18, 0xa5, 0x52 }, + { 0x29, 0x14, 0x0a, 0x85, 0x6b, 0x1c, 0xa7, 0x53 }, + { 0xc9, 0x64, 0xb2, 0xd9, 0x25, 0xdb, 0x24, 0x92 }, + { 0x49, 0x24, 0x92, 0xc9, 0x2d, 0xdf, 0x26, 0x93 }, + { 0x89, 0x44, 0xa2, 0x51, 0xa1, 0x59, 0x25, 0x12 }, + { 0x09, 0x04, 0x82, 0x41, 0xa9, 0x5d, 0x27, 0x13 }, + { 0xb5, 0xda, 0x6d, 0xb6, 0xee, 0xc2, 0xd4, 0x6a }, + { 0x35, 0x9a, 0x4d, 0xa6, 0xe6, 0xc6, 0xd6, 0x6b }, + { 0xf5, 0xfa, 0x7d, 0x3e, 0x6a, 0x40, 0xd5, 0xea }, + { 0x75, 0xba, 0x5d, 0x2e, 0x62, 0x44, 0xd7, 0xeb }, + { 0x95, 0xca, 0xe5, 0x72, 0x2c, 0x83, 0x54, 0x2a }, + { 0x15, 0x8a, 0xc5, 0x62, 0x24, 0x87, 0x56, 0x2b }, + { 0xd5, 0xea, 0xf5, 0xfa, 0xa8, 0x01, 0x55, 0xaa }, + { 0x55, 0xaa, 0xd5, 0xea, 0xa0, 0x05, 0x57, 0xab }, + { 0xa5, 0x52, 0xa9, 0x54, 0x8f, 0x62, 0x94, 0x4a }, + { 0x25, 0x12, 0x89, 0x44, 0x87, 0x66, 0x96, 0x4b }, + { 0xe5, 0x72, 0xb9, 0xdc, 0x0b, 0xe0, 0x95, 0xca }, + { 0x65, 0x32, 0x99, 0xcc, 0x03, 0xe4, 0x97, 0xcb }, + { 0x85, 0x42, 0x21, 0x90, 0x4d, 0x23, 0x14, 0x0a }, + { 0x05, 0x02, 0x01, 0x80, 0x45, 0x27, 0x16, 0x0b }, + { 0xc5, 0x62, 0x31, 0x18, 0xc9, 0xa1, 0x15, 0x8a }, + { 0x45, 0x22, 0x11, 0x08, 0xc1, 0xa5, 0x17, 0x8b }, + { 0x3d, 0x1e, 0x8f, 0xc7, 0x5e, 0x92, 0xf4, 0x7a }, + { 0xbd, 0x5e, 0xaf, 0xd7, 0x56, 0x96, 0xf6, 0x7b }, + { 0x7d, 0x3e, 0x9f, 0x4f, 0xda, 0x10, 0xf5, 0xfa }, + { 0xfd, 0x7e, 0xbf, 0x5f, 0xd2, 0x14, 0xf7, 0xfb }, + { 0x1d, 0x0e, 0x07, 0x03, 0x9c, 0xd3, 0x74, 0x3a }, + { 0x9d, 0x4e, 0x27, 0x13, 0x94, 0xd7, 0x76, 0x3b }, + { 0x5d, 0x2e, 0x17, 0x8b, 0x18, 0x51, 0x75, 0xba }, + { 0xdd, 0x6e, 0x37, 0x9b, 0x10, 0x55, 0x77, 0xbb }, + { 0x2d, 0x96, 0x4b, 0x25, 0x3f, 0x32, 0xb4, 0x5a }, + { 0xad, 0xd6, 0x6b, 0x35, 0x37, 0x36, 0xb6, 0x5b }, + { 0x6d, 0xb6, 0x5b, 0xad, 0xbb, 0xb0, 0xb5, 0xda }, + { 0xed, 0xf6, 0x7b, 0xbd, 0xb3, 0xb4, 0xb7, 0xdb }, + { 0x0d, 0x86, 0xc3, 0xe1, 0xfd, 0x73, 0x34, 0x1a }, + { 0x8d, 0xc6, 0xe3, 0xf1, 0xf5, 0x77, 0x36, 0x1b }, + { 0x4d, 0xa6, 0xd3, 0x69, 0x79, 0xf1, 0x35, 0x9a }, + { 0xcd, 0xe6, 0xf3, 0x79, 0x71, 0xf5, 0x37, 0x9b }, + { 0x93, 0x49, 0x24, 0x92, 0x5a, 0xbe, 0x4c, 0x26 }, + { 0x13, 0x09, 0x04, 0x82, 0x52, 0xba, 0x4e, 0x27 }, + { 0xd3, 0x69, 0x34, 0x1a, 0xde, 0x3c, 0x4d, 0xa6 }, + { 0x53, 0x29, 0x14, 0x0a, 0xd6, 0x38, 0x4f, 0xa7 }, + { 0xb3, 0x59, 0xac, 0x56, 0x98, 0xff, 0xcc, 0x66 }, + { 0x33, 0x19, 0x8c, 0x46, 0x90, 0xfb, 0xce, 0x67 }, + { 0xf3, 0x79, 0xbc, 0xde, 0x1c, 0x7d, 0xcd, 0xe6 }, + { 0x73, 0x39, 0x9c, 0xce, 0x14, 0x79, 0xcf, 0xe7 }, + { 0x83, 0xc1, 0xe0, 0x70, 0x3b, 0x1e, 0x0c, 0x06 }, + { 0x03, 0x81, 0xc0, 0x60, 0x33, 0x1a, 0x0e, 0x07 }, + { 0xc3, 0xe1, 0xf0, 0xf8, 0xbf, 0x9c, 0x0d, 0x86 }, + { 0x43, 0xa1, 0xd0, 0xe8, 0xb7, 0x98, 0x0f, 0x87 }, + { 0xa3, 0xd1, 0x68, 0xb4, 0xf9, 0x5f, 0x8c, 0x46 }, + { 0x23, 0x91, 0x48, 0xa4, 0xf1, 0x5b, 0x8e, 0x47 }, + { 0xe3, 0xf1, 0x78, 0x3c, 0x7d, 0xdd, 0x8d, 0xc6 }, + { 0x63, 0xb1, 0x58, 0x2c, 0x75, 0xd9, 0x8f, 0xc7 }, + { 0x1b, 0x8d, 0xc6, 0xe3, 0xea, 0xee, 0x6c, 0x36 }, + { 0x9b, 0xcd, 0xe6, 0xf3, 0xe2, 0xea, 0x6e, 0x37 }, + { 0x5b, 0xad, 0xd6, 0x6b, 0x6e, 0x6c, 0x6d, 0xb6 }, + { 0xdb, 0xed, 0xf6, 0x7b, 0x66, 0x68, 0x6f, 0xb7 }, + { 0x3b, 0x9d, 0x4e, 0x27, 0x28, 0xaf, 0xec, 0x76 }, + { 0xbb, 0xdd, 0x6e, 0x37, 0x20, 0xab, 0xee, 0x77 }, + { 0x7b, 0xbd, 0x5e, 0xaf, 0xac, 0x2d, 0xed, 0xf6 }, + { 0xfb, 0xfd, 0x7e, 0xbf, 0xa4, 0x29, 0xef, 0xf7 }, + { 0x0b, 0x05, 0x02, 0x01, 0x8b, 0x4e, 0x2c, 0x16 }, + { 0x8b, 0x45, 0x22, 0x11, 0x83, 0x4a, 0x2e, 0x17 }, + { 0x4b, 0x25, 0x12, 0x89, 0x0f, 0xcc, 0x2d, 0x96 }, + { 0xcb, 0x65, 0x32, 0x99, 0x07, 0xc8, 0x2f, 0x97 }, + { 0x2b, 0x15, 0x8a, 0xc5, 0x49, 0x0f, 0xac, 0x56 }, + { 0xab, 0x55, 0xaa, 0xd5, 0x41, 0x0b, 0xae, 0x57 }, + { 0x6b, 0x35, 0x9a, 0x4d, 0xcd, 0x8d, 0xad, 0xd6 }, + { 0xeb, 0x75, 0xba, 0x5d, 0xc5, 0x89, 0xaf, 0xd7 }, + { 0x57, 0xab, 0x55, 0xaa, 0x82, 0x16, 0x5c, 0xae }, + { 0xd7, 0xeb, 0x75, 0xba, 0x8a, 0x12, 0x5e, 0xaf }, + { 0x17, 0x8b, 0x45, 0x22, 0x06, 0x94, 0x5d, 0x2e }, + { 0x97, 0xcb, 0x65, 0x32, 0x0e, 0x90, 0x5f, 0x2f }, + { 0x77, 0xbb, 0xdd, 0x6e, 0x40, 0x57, 0xdc, 0xee }, + { 0xf7, 0xfb, 0xfd, 0x7e, 0x48, 0x53, 0xde, 0xef }, + { 0x37, 0x9b, 0xcd, 0xe6, 0xc4, 0xd5, 0xdd, 0x6e }, + { 0xb7, 0xdb, 0xed, 0xf6, 0xcc, 0xd1, 0xdf, 0x6f }, + { 0x47, 0x23, 0x91, 0x48, 0xe3, 0xb6, 0x1c, 0x8e }, + { 0xc7, 0x63, 0xb1, 0x58, 0xeb, 0xb2, 0x1e, 0x8f }, + { 0x07, 0x03, 0x81, 0xc0, 0x67, 0x34, 0x1d, 0x0e }, + { 0x87, 0x43, 0xa1, 0xd0, 0x6f, 0x30, 0x1f, 0x0f }, + { 0x67, 0x33, 0x19, 0x8c, 0x21, 0xf7, 0x9c, 0xce }, + { 0xe7, 0x73, 0x39, 0x9c, 0x29, 0xf3, 0x9e, 0xcf }, + { 0x27, 0x13, 0x09, 0x04, 0xa5, 0x75, 0x9d, 0x4e }, + { 0xa7, 0x53, 0x29, 0x14, 0xad, 0x71, 0x9f, 0x4f }, + { 0xdf, 0x6f, 0xb7, 0xdb, 0x32, 0x46, 0x7c, 0xbe }, + { 0x5f, 0x2f, 0x97, 0xcb, 0x3a, 0x42, 0x7e, 0xbf }, + { 0x9f, 0x4f, 0xa7, 0x53, 0xb6, 0xc4, 0x7d, 0x3e }, + { 0x1f, 0x0f, 0x87, 0x43, 0xbe, 0xc0, 0x7f, 0x3f }, + { 0xff, 0x7f, 0x3f, 0x1f, 0xf0, 0x07, 0xfc, 0xfe }, + { 0x7f, 0x3f, 0x1f, 0x0f, 0xf8, 0x03, 0xfe, 0xff }, + { 0xbf, 0x5f, 0x2f, 0x97, 0x74, 0x85, 0xfd, 0x7e }, + { 0x3f, 0x1f, 0x0f, 0x87, 0x7c, 0x81, 0xff, 0x7f }, + { 0xcf, 0xe7, 0x73, 0x39, 0x53, 0xe6, 0x3c, 0x9e }, + { 0x4f, 0xa7, 0x53, 0x29, 0x5b, 0xe2, 0x3e, 0x9f }, + { 0x8f, 0xc7, 0x63, 0xb1, 0xd7, 0x64, 0x3d, 0x1e }, + { 0x0f, 0x87, 0x43, 0xa1, 0xdf, 0x60, 0x3f, 0x1f }, + { 0xef, 0xf7, 0xfb, 0xfd, 0x91, 0xa7, 0xbc, 0xde }, + { 0x6f, 0xb7, 0xdb, 0xed, 0x99, 0xa3, 0xbe, 0xdf }, + { 0xaf, 0xd7, 0xeb, 0x75, 0x15, 0x25, 0xbd, 0x5e }, + { 0x2f, 0x97, 0xcb, 0x65, 0x1d, 0x21, 0xbf, 0x5f }, +}; + const uint8_t __aligned(256) raid_gfexp_raid[256] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, diff --git a/raid/test.c b/raid/test.c index 14bccd3a..a2155dee 100644 --- a/raid/test.c +++ b/raid/test.c @@ -40,6 +40,39 @@ static void meminc(void *dst, size_t size) ++p[i]; } +/** + * Simulate the bit-level semantics of VGF2P8AFFINEQB with imm8=0. + */ +static uint8_t raid_gfaffine_mul(uint8_t c, uint8_t x) +{ + uint8_t v = 0; + int j; + + for (j = 0; j < 8; ++j) { + uint8_t row = raid_gfaffine_raid[c][7 - j] & x; + + row ^= row >> 4; + row ^= row >> 2; + row ^= row >> 1; + v |= (row & 1U) << j; + } + + return v; +} + +int raid_test_gfaffine(void) +{ + int c, x; + + /* all coefficients, including 0, 1, 2, and 0xff, are mandatory */ + for (c = 0; c < 256; ++c) + for (x = 0; x < 256; ++x) + if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x]) + return -1; + + return 0; +} + int raid_test_combo(void) { int r; @@ -366,11 +399,19 @@ int raid_test_rec(int mode, int nd, size_t size) if (raid_cpu_has_avx2()) test_setup(i) = raid_rec1_avx2; #ifdef CONFIG_X86_64 - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) - test_setup(i) = raid_rec1_avx2gfni; - if (raid_cpu_has_avx512gfni()) - test_setup(i) = raid_rec1_avx512gfni; + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec1_avx2gfni_aes; + else + test_setup(i) = raid_rec1_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec1_avx512gfni_aes; + else + test_setup(i) = raid_rec1_avx512gfni_raid; + } } #endif #endif @@ -387,11 +428,19 @@ int raid_test_rec(int mode, int nd, size_t size) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) test_setup(i) = raid_rec2_avx512bw; - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) - test_setup(i) = raid_rec2_avx2gfni; - if (raid_cpu_has_avx512gfni()) - test_setup(i) = raid_rec2_avx512gfni; + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec2_avx2gfni_aes; + else + test_setup(i) = raid_rec2_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_rec2_avx512gfni_aes; + else + test_setup(i) = raid_rec2_avx512gfni_raid; + } } #endif #endif @@ -408,11 +457,19 @@ int raid_test_rec(int mode, int nd, size_t size) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx512bw()) test_setup(i) = raid_recX_avx512bw; - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) - test_setup(i) = raid_recX_avx2gfni; - if (raid_cpu_has_avx512gfni()) - test_setup(i) = raid_recX_avx512gfni; + if (mode == RAID_MODE_CAUCHY_RAID || mode == RAID_MODE_CAUCHY_AES) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_recX_avx2gfni_aes; + else + test_setup(i) = raid_recX_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) + test_setup(i) = raid_recX_avx512gfni_aes; + else + test_setup(i) = raid_recX_avx512gfni_raid; + } } #endif #endif @@ -582,13 +639,17 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(1) = raid_gen1_avx512bw; test_setup(2) = raid_gen2_avx512bw; } - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) test_setup(2) = raid_gen2_avx2gfni_aes; - } - if (raid_cpu_has_avx512gfni()) { + else + test_setup(2) = raid_gen2_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) test_setup(2) = raid_gen2_avx512gfni_aes; - } + else + test_setup(2) = raid_gen2_avx512gfni_raid; } #endif #endif @@ -658,18 +719,30 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(5) = raid_gen5_avx512bw; test_setup(6) = raid_gen6_avx512bw; } - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { test_setup(3) = raid_gen3_avx2gfni_aes; test_setup(4) = raid_gen4_avx2gfni_aes; test_setup(5) = raid_gen5_avx2gfni_aes; test_setup(6) = raid_gen6_avx2gfni_aes; + } else { + test_setup(3) = raid_gen3_avx2gfni_raid; + test_setup(4) = raid_gen4_avx2gfni_raid; + test_setup(5) = raid_gen5_avx2gfni_raid; + test_setup(6) = raid_gen6_avx2gfni_raid; } - if (raid_cpu_has_avx512gfni()) { + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { test_setup(3) = raid_gen3_avx512gfni_aes; test_setup(4) = raid_gen4_avx512gfni_aes; test_setup(5) = raid_gen5_avx512gfni_aes; test_setup(6) = raid_gen6_avx512gfni_aes; + } else { + test_setup(3) = raid_gen3_avx512gfni_raid; + test_setup(4) = raid_gen4_avx512gfni_raid; + test_setup(5) = raid_gen5_avx512gfni_raid; + test_setup(6) = raid_gen6_avx512gfni_raid; } } #endif diff --git a/raid/test.h b/raid/test.h index af10522c..04a2f66e 100644 --- a/raid/test.h +++ b/raid/test.h @@ -31,6 +31,13 @@ int raid_test_sort(void); */ int raid_test_combo(void); +/** + * Tests all RAID affine GFNI matrices against scalar GF multiplication. + * + * Returns 0 on success. + */ +int raid_test_gfaffine(void); + /** * Tests the Galois field and Cauchy matrix properties. * diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index 1c6c9112..df53042c 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -73,6 +73,13 @@ int main(void) /* LCOV_EXCL_STOP */ } + printf("Test GFNI affine matrices...\n"); + if (raid_test_gfaffine() != 0) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } + printf("Test Cauchy RAID parity generation with 1-%u data disks...\n", RAID_DATA_MAX); for (int i = 1; i <= RAID_DATA_MAX; ++i) { if (raid_test_par(RAID_MODE_CAUCHY_RAID, i, TEST_SIZE) != 0) { diff --git a/raid/test/selftest.c b/raid/test/selftest.c index f7b1c474..b68d36df 100644 --- a/raid/test/selftest.c +++ b/raid/test/selftest.c @@ -16,6 +16,13 @@ int main(void) raid_init(); + printf("Test GFNI affine matrices\n"); + if (raid_test_gfaffine() != 0) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } + printf("Test Cauchy RAID\n"); if (raid_test_poly(RAID_MODE_CAUCHY_RAID) != 0) { /* LCOV_EXCL_START */ diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 5743d3eb..22b7b1a3 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -208,6 +208,10 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); } + if (raid_cpu_has_avx2gfni()) + printf("%8s", ""); + if (raid_cpu_has_avx512gfni()) + printf("%8s", ""); #endif #endif printf("\n"); @@ -351,7 +355,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -363,7 +371,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -470,7 +482,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -482,7 +498,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -589,7 +609,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -601,7 +625,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -706,7 +734,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -718,7 +750,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -823,7 +859,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -835,7 +875,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -1058,7 +1102,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) SPEED_START { raid_gen_force(1, raid_gen1_avx2); /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx2gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1068,7 +1112,7 @@ void speed_rec(int nd, void **v, int size, int delta, int period) SPEED_START { raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ /* +1 to avoid GEN1 optimized case */ - raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v); + raid_rec1_avx512gfni_raid(1, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1145,9 +1189,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx2gfni_aes); + raid_gen_force(2, raid_gen2_avx2gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx2gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1155,9 +1199,9 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(2, raid_gen2_avx512gfni_aes); + raid_gen_force(2, raid_gen2_avx512gfni_raid); /* +1 to avoid GEN2 optimized case */ - raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v); + raid_rec2_avx512gfni_raid(2, id, ip + 1, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1224,8 +1268,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx2gfni_aes); - raid_recX_avx2gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx2gfni_raid); + raid_recX_avx2gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1233,8 +1277,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(3, raid_gen3_avx512gfni_aes); - raid_recX_avx512gfni(3, id, ip, nd, size, v); + raid_gen_force(3, raid_gen3_avx512gfni_raid); + raid_recX_avx512gfni_raid(3, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1305,8 +1349,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx2gfni_aes); - raid_recX_avx2gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx2gfni_raid); + raid_recX_avx2gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1314,8 +1358,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(4, raid_gen4_avx512gfni_aes); - raid_recX_avx512gfni(4, id, ip, nd, size, v); + raid_gen_force(4, raid_gen4_avx512gfni_raid); + raid_recX_avx512gfni_raid(4, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1386,8 +1430,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx2gfni_aes); - raid_recX_avx2gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx2gfni_raid); + raid_recX_avx2gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1395,8 +1439,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(5, raid_gen5_avx512gfni_aes); - raid_recX_avx512gfni(5, id, ip, nd, size, v); + raid_gen_force(5, raid_gen5_avx512gfni_raid); + raid_recX_avx512gfni_raid(5, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1467,8 +1511,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2gfni()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx2gfni_aes); - raid_recX_avx2gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx2gfni_raid); + raid_recX_avx2gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt); @@ -1476,8 +1520,8 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx512gfni()) { SPEED_START { - raid_gen_force(6, raid_gen6_avx512gfni_aes); - raid_recX_avx512gfni(6, id, ip, nd, size, v); + raid_gen_force(6, raid_gen6_avx512gfni_raid); + raid_recX_avx512gfni_raid(6, id, ip, nd, size, v); } SPEED_STOP printf("%8" PRIu64, ds / dt);