diff --git a/cmdline/selftest.c b/cmdline/selftest.c index d8436c4a..30b2db4d 100644 --- a/cmdline/selftest.c +++ b/cmdline/selftest.c @@ -1151,6 +1151,12 @@ void selftest(void) exit(EXIT_FAILURE); /* LCOV_EXCL_STOP */ } + if (raid_test_gfaffine() != 0) { + /* LCOV_EXCL_START */ + log_fatal(EINTERNAL, "Failed GFNI AFFINE test\n"); + exit(EXIT_FAILURE); + /* LCOV_EXCL_STOP */ + } if (raid_test_par(RAID_MODE_VANDERMONDE_RAID, 32, 256) != 0) { /* LCOV_EXCL_START */ log_fatal(EINTERNAL, "Failed GEN Vandermonde RAID test\n"); diff --git a/cmdline/speed.c b/cmdline/speed.c index 89024118..947b4b53 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -306,6 +306,10 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); fflush(stdout); } + if (raid_cpu_has_avx2gfni()) + printf("%8s", ""); + if (raid_cpu_has_avx512gfni()) + printf("%8s", ""); #endif #endif printf("\n"); @@ -449,7 +453,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -461,7 +469,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -570,7 +582,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -582,7 +598,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -691,7 +711,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -703,7 +727,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -809,7 +837,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -821,7 +853,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -927,7 +963,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -939,7 +979,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } diff --git a/raid/gfni.c b/raid/gfni.c index bdea56c8..3a5616c4 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -6,6 +6,196 @@ #include "cpu.h" #ifdef CONFIG_X86_64 +/* + * GENX AVX2 GFNI implementation + */ +static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32])); + + asm volatile ("vmovdqa %ymm0,%ymm2"); + asm volatile ("vmovdqa %ymm1,%ymm3"); + if (np >= 3) { + asm volatile ("vmovdqa %ymm0,%ymm4"); + asm volatile ("vmovdqa %ymm1,%ymm5"); + } + if (np >= 4) { + asm volatile ("vmovdqa %ymm0,%ymm6"); + asm volatile ("vmovdqa %ymm1,%ymm7"); + } + if (np >= 5) { + asm volatile ("vmovdqa %ymm0,%ymm8"); + asm volatile ("vmovdqa %ymm1,%ymm9"); + } + if (np >= 6) { + asm volatile ("vmovdqa %ymm0,%ymm10"); + asm volatile ("vmovdqa %ymm1,%ymm11"); + } + + /* all other disks */ + for (d = 1; d < nd; ++d) { + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[1][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + if (np >= 3) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + } + if (np >= 4) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + } + if (np >= 5) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + } + if (np >= 6) { + asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0])); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } + } + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32])); + if (np >= 3) { + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32])); + } + if (np >= 4) { + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32])); + } + if (np >= 5) { + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32])); + } + if (np >= 6) { + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32])); + } + } + + raid_avx_end(); +} + +/* + * GENX AVX512 GFNI implementation + */ +static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void **vv, int np) +{ + uint8_t **v = (uint8_t **)vv; + size_t i; + int d; + + if (nd == 1) { + for (d = 0; d < np; ++d) + memcpy(v[1 + d], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i])); + + asm volatile ("vmovdqa64 %zmm0,%zmm1"); + if (np >= 3) + asm volatile ("vmovdqa64 %zmm0,%zmm2"); + if (np >= 4) + asm volatile ("vmovdqa64 %zmm0,%zmm3"); + if (np >= 5) + asm volatile ("vmovdqa64 %zmm0,%zmm4"); + if (np >= 6) + asm volatile ("vmovdqa64 %zmm0,%zmm5"); + + /* all other disks */ + for (d = 1; d < nd; ++d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[1][d]][0])); + if (np >= 3) + asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0])); + if (np >= 4) + asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0])); + if (np >= 5) + asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0])); + if (np >= 6) + asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0])); + + asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm12"); + if (np >= 3) + asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm6,%zmm13"); + if (np >= 4) + asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm6,%zmm14"); + if (np >= 5) + asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm6,%zmm15"); + if (np >= 6) + asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm6,%zmm16"); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); + if (np >= 3) + asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); + if (np >= 4) + asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); + if (np >= 5) + asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); + if (np >= 6) + asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); + } + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i])); + if (np >= 3) + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i])); + if (np >= 4) + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i])); + if (np >= 5) + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i])); + if (np >= 6) + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i])); + } + + raid_avx_end(); +} /* * GENX AVX2 GFNI implementation @@ -198,6 +388,86 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void * raid_avx_end(); } +/* + * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation + */ +void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_raid(nd, size, vv, 2); +} + +/* + * GEN2 (RAID6 with powers of 2) GFNI implementation + */ +void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_raid(nd, size, vv, 2); +} + +/* + * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_raid(nd, size, vv, 3); +} + +/* + * GEN3 (triple parity with Cauchy matrix) GFNI implementation + */ +void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_raid(nd, size, vv, 3); +} + +/* + * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_raid(nd, size, vv, 4); +} + +/* + * GEN4 (quad parity with Cauchy matrix) GFNI implementation + */ +void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_raid(nd, size, vv, 4); +} + +/* + * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_raid(nd, size, vv, 5); +} + +/* + * GEN5 (penta parity with Cauchy matrix) GFNI implementation + */ +void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_raid(nd, size, vv, 5); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx2gfni_raid(nd, size, vv, 6); +} + +/* + * GEN6 (hexa parity with Cauchy matrix) GFNI implementation + */ +void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv) +{ + raid_genX_avx512gfni_raid(nd, size, vv, 6); +} + /* * GEN2 Cauchy AVX2 GFNI implementation */ @@ -636,6 +906,12 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** void raid_register_avx2gfni(void) { if (raid_cpu_has_avx2gfni()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_aes, RAID_POLY_AES); @@ -655,6 +931,12 @@ void raid_register_avx512gfni(void) { if (raid_cpu_has_avx512gfni()) { if (!raid_cpu_has_slow_avx512()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_raid, RAID_POLY_RAID); + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_aes, RAID_POLY_AES); raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_aes, RAID_POLY_AES); diff --git a/raid/internal.h b/raid/internal.h index af43c4dc..68bf6a79 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -104,6 +104,8 @@ void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv); void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen2_avx512bw(int nd, size_t size, void **vv); +void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv); void raid_genz_int32_raid(int nd, size_t size, void **vv); @@ -119,6 +121,8 @@ void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen3_avx512bw(int nd, size_t size, void **vv); +void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen4_int8(int nd, size_t size, void **vv); @@ -129,6 +133,8 @@ void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen4_avx512bw(int nd, size_t size, void **vv); +void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen5_int8(int nd, size_t size, void **vv); @@ -138,6 +144,8 @@ void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen5_avx512bw(int nd, size_t size, void **vv); +void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv); void raid_gen6_int8(int nd, size_t size, void **vv); @@ -147,6 +155,8 @@ void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv); void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv); void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv); void raid_gen6_avx512bw(int nd, size_t size, void **vv); +void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv); +void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv); void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv); void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv); void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -325,6 +335,7 @@ extern const uint8_t raid_gfexp_raid[256] __aligned(256); extern const uint8_t raid_gfinv_raid[256] __aligned(256); extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256); extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256); +extern const uint8_t raid_gfaffine_raid[256][8] __aligned(256); #if defined(CONFIG_X86) || defined(CONFIG_NEON) extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256); extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256); diff --git a/raid/mktables.c b/raid/mktables.c index 0a201660..ef1fc63d 100644 --- a/raid/mktables.c +++ b/raid/mktables.c @@ -38,6 +38,34 @@ static uint8_t raid_gfmul(uint8_t poly, uint8_t a, uint8_t b) return v; } +/** + * Setup the GFNI affine matrix for multiplication by a constant. + * + * VGF2P8MULB is fixed to the AES polynomial 0x11b, while RAID parity is + * defined over GF(2^8) modulo 0x11d. VGF2P8AFFINEQB can implement the + * multiplication because multiplication by a constant is GF(2)-linear. + * + * VGF2P8AFFINEQB stores the row producing output bit j in byte 7-j. + * Within each row, bit k selects input bit k. + */ +static void set_affine(uint8_t poly, uint8_t c, uint8_t *matrix) +{ + int j, k; + + for (j = 0; j < 8; ++j) { + uint8_t row = 0; + + for (k = 0; k < 8; ++k) { + uint8_t v = raid_gfmul(poly, c, (uint8_t)(1U << k)); + + if ((v & (1U << j)) != 0) + row |= (uint8_t)(1U << k); + } + + matrix[7 - j] = row; + } +} + /** * Inversion (1/a) in GF(2^8). */ @@ -231,6 +259,31 @@ void tables(uint8_t poly, uint8_t generator, const char *tag) } printf("};\n\n"); + if (poly == RAID_POLY_RAID) { + uint8_t affine[8]; + + printf("/**\n"); + printf(" * GFNI affine matrices for multiplication in GF(2^8)/0x11d.\n"); + printf(" *\n"); + printf(" * VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane.\n"); + printf(" * Byte 7-j contains the row producing output bit j, and bit k\n"); + printf(" * of that byte selects input bit k. The instruction uses imm8=0.\n"); + printf(" */\n"); + printf("const uint8_t __aligned(256) raid_gfaffine_raid[256][8] =\n"); + printf("{\n"); + for (i = 0; i < 256; ++i) { + set_affine(poly, (uint8_t)i, affine); + printf("\t{ "); + for (j = 0; j < 8; ++j) { + printf("0x%02x", (unsigned)affine[j]); + if (j != 7) + printf(", "); + } + printf(" },\n"); + } + printf("};\n\n"); + } + /* generator^a */ printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag); printf("{\n"); diff --git a/raid/tables.c b/raid/tables.c index 9d89ea82..932599e0 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -8713,6 +8713,273 @@ const uint8_t __aligned(256) raid_gfmul_raid[256][256] = }, }; +/** + * GFNI affine matrices for multiplication in GF(2^8)/0x11d. + * + * VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane. + * Byte 7-j contains the row producing output bit j, and bit k + * of that byte selects input bit k. The instruction uses imm8=0. + */ +const uint8_t __aligned(256) raid_gfaffine_raid[256][8] = +{ + { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 }, + { 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 }, + { 0x40, 0x20, 0x10, 0x88, 0x84, 0x82, 0x01, 0x80 }, + { 0xc0, 0x60, 0x30, 0x98, 0x8c, 0x86, 0x03, 0x81 }, + { 0x20, 0x10, 0x88, 0xc4, 0xc2, 0x41, 0x80, 0x40 }, + { 0xa0, 0x50, 0xa8, 0xd4, 0xca, 0x45, 0x82, 0x41 }, + { 0x60, 0x30, 0x98, 0x4c, 0x46, 0xc3, 0x81, 0xc0 }, + { 0xe0, 0x70, 0xb8, 0x5c, 0x4e, 0xc7, 0x83, 0xc1 }, + { 0x10, 0x88, 0xc4, 0xe2, 0x61, 0xa0, 0x40, 0x20 }, + { 0x90, 0xc8, 0xe4, 0xf2, 0x69, 0xa4, 0x42, 0x21 }, + { 0x50, 0xa8, 0xd4, 0x6a, 0xe5, 0x22, 0x41, 0xa0 }, + { 0xd0, 0xe8, 0xf4, 0x7a, 0xed, 0x26, 0x43, 0xa1 }, + { 0x30, 0x98, 0x4c, 0x26, 0xa3, 0xe1, 0xc0, 0x60 }, + { 0xb0, 0xd8, 0x6c, 0x36, 0xab, 0xe5, 0xc2, 0x61 }, + { 0x70, 0xb8, 0x5c, 0xae, 0x27, 0x63, 0xc1, 0xe0 }, + { 0xf0, 0xf8, 0x7c, 0xbe, 0x2f, 0x67, 0xc3, 0xe1 }, + { 0x88, 0xc4, 0xe2, 0x71, 0xb0, 0x50, 0x20, 0x10 }, + { 0x08, 0x84, 0xc2, 0x61, 0xb8, 0x54, 0x22, 0x11 }, + { 0xc8, 0xe4, 0xf2, 0xf9, 0x34, 0xd2, 0x21, 0x90 }, + { 0x48, 0xa4, 0xd2, 0xe9, 0x3c, 0xd6, 0x23, 0x91 }, + { 0xa8, 0xd4, 0x6a, 0xb5, 0x72, 0x11, 0xa0, 0x50 }, + { 0x28, 0x94, 0x4a, 0xa5, 0x7a, 0x15, 0xa2, 0x51 }, + { 0xe8, 0xf4, 0x7a, 0x3d, 0xf6, 0x93, 0xa1, 0xd0 }, + { 0x68, 0xb4, 0x5a, 0x2d, 0xfe, 0x97, 0xa3, 0xd1 }, + { 0x98, 0x4c, 0x26, 0x93, 0xd1, 0xf0, 0x60, 0x30 }, + { 0x18, 0x0c, 0x06, 0x83, 0xd9, 0xf4, 0x62, 0x31 }, + { 0xd8, 0x6c, 0x36, 0x1b, 0x55, 0x72, 0x61, 0xb0 }, + { 0x58, 0x2c, 0x16, 0x0b, 0x5d, 0x76, 0x63, 0xb1 }, + { 0xb8, 0x5c, 0xae, 0x57, 0x13, 0xb1, 0xe0, 0x70 }, + { 0x38, 0x1c, 0x8e, 0x47, 0x1b, 0xb5, 0xe2, 0x71 }, + { 0xf8, 0x7c, 0xbe, 0xdf, 0x97, 0x33, 0xe1, 0xf0 }, + { 0x78, 0x3c, 0x9e, 0xcf, 0x9f, 0x37, 0xe3, 0xf1 }, + { 0xc4, 0xe2, 0x71, 0x38, 0xd8, 0xa8, 0x10, 0x88 }, + { 0x44, 0xa2, 0x51, 0x28, 0xd0, 0xac, 0x12, 0x89 }, + { 0x84, 0xc2, 0x61, 0xb0, 0x5c, 0x2a, 0x11, 0x08 }, + { 0x04, 0x82, 0x41, 0xa0, 0x54, 0x2e, 0x13, 0x09 }, + { 0xe4, 0xf2, 0xf9, 0xfc, 0x1a, 0xe9, 0x90, 0xc8 }, + { 0x64, 0xb2, 0xd9, 0xec, 0x12, 0xed, 0x92, 0xc9 }, + { 0xa4, 0xd2, 0xe9, 0x74, 0x9e, 0x6b, 0x91, 0x48 }, + { 0x24, 0x92, 0xc9, 0x64, 0x96, 0x6f, 0x93, 0x49 }, + { 0xd4, 0x6a, 0xb5, 0xda, 0xb9, 0x08, 0x50, 0xa8 }, + { 0x54, 0x2a, 0x95, 0xca, 0xb1, 0x0c, 0x52, 0xa9 }, + { 0x94, 0x4a, 0xa5, 0x52, 0x3d, 0x8a, 0x51, 0x28 }, + { 0x14, 0x0a, 0x85, 0x42, 0x35, 0x8e, 0x53, 0x29 }, + { 0xf4, 0x7a, 0x3d, 0x1e, 0x7b, 0x49, 0xd0, 0xe8 }, + { 0x74, 0x3a, 0x1d, 0x0e, 0x73, 0x4d, 0xd2, 0xe9 }, + { 0xb4, 0x5a, 0x2d, 0x96, 0xff, 0xcb, 0xd1, 0x68 }, + { 0x34, 0x1a, 0x0d, 0x86, 0xf7, 0xcf, 0xd3, 0x69 }, + { 0x4c, 0x26, 0x93, 0x49, 0x68, 0xf8, 0x30, 0x98 }, + { 0xcc, 0x66, 0xb3, 0x59, 0x60, 0xfc, 0x32, 0x99 }, + { 0x0c, 0x06, 0x83, 0xc1, 0xec, 0x7a, 0x31, 0x18 }, + { 0x8c, 0x46, 0xa3, 0xd1, 0xe4, 0x7e, 0x33, 0x19 }, + { 0x6c, 0x36, 0x1b, 0x8d, 0xaa, 0xb9, 0xb0, 0xd8 }, + { 0xec, 0x76, 0x3b, 0x9d, 0xa2, 0xbd, 0xb2, 0xd9 }, + { 0x2c, 0x16, 0x0b, 0x05, 0x2e, 0x3b, 0xb1, 0x58 }, + { 0xac, 0x56, 0x2b, 0x15, 0x26, 0x3f, 0xb3, 0x59 }, + { 0x5c, 0xae, 0x57, 0xab, 0x09, 0x58, 0x70, 0xb8 }, + { 0xdc, 0xee, 0x77, 0xbb, 0x01, 0x5c, 0x72, 0xb9 }, + { 0x1c, 0x8e, 0x47, 0x23, 0x8d, 0xda, 0x71, 0x38 }, + { 0x9c, 0xce, 0x67, 0x33, 0x85, 0xde, 0x73, 0x39 }, + { 0x7c, 0xbe, 0xdf, 0x6f, 0xcb, 0x19, 0xf0, 0xf8 }, + { 0xfc, 0xfe, 0xff, 0x7f, 0xc3, 0x1d, 0xf2, 0xf9 }, + { 0x3c, 0x9e, 0xcf, 0xe7, 0x4f, 0x9b, 0xf1, 0x78 }, + { 0xbc, 0xde, 0xef, 0xf7, 0x47, 0x9f, 0xf3, 0x79 }, + { 0xe2, 0x71, 0x38, 0x1c, 0x6c, 0xd4, 0x88, 0xc4 }, + { 0x62, 0x31, 0x18, 0x0c, 0x64, 0xd0, 0x8a, 0xc5 }, + { 0xa2, 0x51, 0x28, 0x94, 0xe8, 0x56, 0x89, 0x44 }, + { 0x22, 0x11, 0x08, 0x84, 0xe0, 0x52, 0x8b, 0x45 }, + { 0xc2, 0x61, 0xb0, 0xd8, 0xae, 0x95, 0x08, 0x84 }, + { 0x42, 0x21, 0x90, 0xc8, 0xa6, 0x91, 0x0a, 0x85 }, + { 0x82, 0x41, 0xa0, 0x50, 0x2a, 0x17, 0x09, 0x04 }, + { 0x02, 0x01, 0x80, 0x40, 0x22, 0x13, 0x0b, 0x05 }, + { 0xf2, 0xf9, 0xfc, 0xfe, 0x0d, 0x74, 0xc8, 0xe4 }, + { 0x72, 0xb9, 0xdc, 0xee, 0x05, 0x70, 0xca, 0xe5 }, + { 0xb2, 0xd9, 0xec, 0x76, 0x89, 0xf6, 0xc9, 0x64 }, + { 0x32, 0x99, 0xcc, 0x66, 0x81, 0xf2, 0xcb, 0x65 }, + { 0xd2, 0xe9, 0x74, 0x3a, 0xcf, 0x35, 0x48, 0xa4 }, + { 0x52, 0xa9, 0x54, 0x2a, 0xc7, 0x31, 0x4a, 0xa5 }, + { 0x92, 0xc9, 0x64, 0xb2, 0x4b, 0xb7, 0x49, 0x24 }, + { 0x12, 0x89, 0x44, 0xa2, 0x43, 0xb3, 0x4b, 0x25 }, + { 0x6a, 0xb5, 0xda, 0x6d, 0xdc, 0x84, 0xa8, 0xd4 }, + { 0xea, 0xf5, 0xfa, 0x7d, 0xd4, 0x80, 0xaa, 0xd5 }, + { 0x2a, 0x95, 0xca, 0xe5, 0x58, 0x06, 0xa9, 0x54 }, + { 0xaa, 0xd5, 0xea, 0xf5, 0x50, 0x02, 0xab, 0x55 }, + { 0x4a, 0xa5, 0x52, 0xa9, 0x1e, 0xc5, 0x28, 0x94 }, + { 0xca, 0xe5, 0x72, 0xb9, 0x16, 0xc1, 0x2a, 0x95 }, + { 0x0a, 0x85, 0x42, 0x21, 0x9a, 0x47, 0x29, 0x14 }, + { 0x8a, 0xc5, 0x62, 0x31, 0x92, 0x43, 0x2b, 0x15 }, + { 0x7a, 0x3d, 0x1e, 0x8f, 0xbd, 0x24, 0xe8, 0xf4 }, + { 0xfa, 0x7d, 0x3e, 0x9f, 0xb5, 0x20, 0xea, 0xf5 }, + { 0x3a, 0x1d, 0x0e, 0x07, 0x39, 0xa6, 0xe9, 0x74 }, + { 0xba, 0x5d, 0x2e, 0x17, 0x31, 0xa2, 0xeb, 0x75 }, + { 0x5a, 0x2d, 0x96, 0x4b, 0x7f, 0x65, 0x68, 0xb4 }, + { 0xda, 0x6d, 0xb6, 0x5b, 0x77, 0x61, 0x6a, 0xb5 }, + { 0x1a, 0x0d, 0x86, 0xc3, 0xfb, 0xe7, 0x69, 0x34 }, + { 0x9a, 0x4d, 0xa6, 0xd3, 0xf3, 0xe3, 0x6b, 0x35 }, + { 0x26, 0x93, 0x49, 0x24, 0xb4, 0x7c, 0x98, 0x4c }, + { 0xa6, 0xd3, 0x69, 0x34, 0xbc, 0x78, 0x9a, 0x4d }, + { 0x66, 0xb3, 0x59, 0xac, 0x30, 0xfe, 0x99, 0xcc }, + { 0xe6, 0xf3, 0x79, 0xbc, 0x38, 0xfa, 0x9b, 0xcd }, + { 0x06, 0x83, 0xc1, 0xe0, 0x76, 0x3d, 0x18, 0x0c }, + { 0x86, 0xc3, 0xe1, 0xf0, 0x7e, 0x39, 0x1a, 0x0d }, + { 0x46, 0xa3, 0xd1, 0x68, 0xf2, 0xbf, 0x19, 0x8c }, + { 0xc6, 0xe3, 0xf1, 0x78, 0xfa, 0xbb, 0x1b, 0x8d }, + { 0x36, 0x1b, 0x8d, 0xc6, 0xd5, 0xdc, 0xd8, 0x6c }, + { 0xb6, 0x5b, 0xad, 0xd6, 0xdd, 0xd8, 0xda, 0x6d }, + { 0x76, 0x3b, 0x9d, 0x4e, 0x51, 0x5e, 0xd9, 0xec }, + { 0xf6, 0x7b, 0xbd, 0x5e, 0x59, 0x5a, 0xdb, 0xed }, + { 0x16, 0x0b, 0x05, 0x02, 0x17, 0x9d, 0x58, 0x2c }, + { 0x96, 0x4b, 0x25, 0x12, 0x1f, 0x99, 0x5a, 0x2d }, + { 0x56, 0x2b, 0x15, 0x8a, 0x93, 0x1f, 0x59, 0xac }, + { 0xd6, 0x6b, 0x35, 0x9a, 0x9b, 0x1b, 0x5b, 0xad }, + { 0xae, 0x57, 0xab, 0x55, 0x04, 0x2c, 0xb8, 0x5c }, + { 0x2e, 0x17, 0x8b, 0x45, 0x0c, 0x28, 0xba, 0x5d }, + { 0xee, 0x77, 0xbb, 0xdd, 0x80, 0xae, 0xb9, 0xdc }, + { 0x6e, 0x37, 0x9b, 0xcd, 0x88, 0xaa, 0xbb, 0xdd }, + { 0x8e, 0x47, 0x23, 0x91, 0xc6, 0x6d, 0x38, 0x1c }, + { 0x0e, 0x07, 0x03, 0x81, 0xce, 0x69, 0x3a, 0x1d }, + { 0xce, 0x67, 0x33, 0x19, 0x42, 0xef, 0x39, 0x9c }, + { 0x4e, 0x27, 0x13, 0x09, 0x4a, 0xeb, 0x3b, 0x9d }, + { 0xbe, 0xdf, 0x6f, 0xb7, 0x65, 0x8c, 0xf8, 0x7c }, + { 0x3e, 0x9f, 0x4f, 0xa7, 0x6d, 0x88, 0xfa, 0x7d }, + { 0xfe, 0xff, 0x7f, 0x3f, 0xe1, 0x0e, 0xf9, 0xfc }, + { 0x7e, 0xbf, 0x5f, 0x2f, 0xe9, 0x0a, 0xfb, 0xfd }, + { 0x9e, 0xcf, 0xe7, 0x73, 0xa7, 0xcd, 0x78, 0x3c }, + { 0x1e, 0x8f, 0xc7, 0x63, 0xaf, 0xc9, 0x7a, 0x3d }, + { 0xde, 0xef, 0xf7, 0xfb, 0x23, 0x4f, 0x79, 0xbc }, + { 0x5e, 0xaf, 0xd7, 0xeb, 0x2b, 0x4b, 0x7b, 0xbd }, + { 0x71, 0x38, 0x1c, 0x8e, 0x36, 0x6a, 0xc4, 0xe2 }, + { 0xf1, 0x78, 0x3c, 0x9e, 0x3e, 0x6e, 0xc6, 0xe3 }, + { 0x31, 0x18, 0x0c, 0x06, 0xb2, 0xe8, 0xc5, 0x62 }, + { 0xb1, 0x58, 0x2c, 0x16, 0xba, 0xec, 0xc7, 0x63 }, + { 0x51, 0x28, 0x94, 0x4a, 0xf4, 0x2b, 0x44, 0xa2 }, + { 0xd1, 0x68, 0xb4, 0x5a, 0xfc, 0x2f, 0x46, 0xa3 }, + { 0x11, 0x08, 0x84, 0xc2, 0x70, 0xa9, 0x45, 0x22 }, + { 0x91, 0x48, 0xa4, 0xd2, 0x78, 0xad, 0x47, 0x23 }, + { 0x61, 0xb0, 0xd8, 0x6c, 0x57, 0xca, 0x84, 0xc2 }, + { 0xe1, 0xf0, 0xf8, 0x7c, 0x5f, 0xce, 0x86, 0xc3 }, + { 0x21, 0x90, 0xc8, 0xe4, 0xd3, 0x48, 0x85, 0x42 }, + { 0xa1, 0xd0, 0xe8, 0xf4, 0xdb, 0x4c, 0x87, 0x43 }, + { 0x41, 0xa0, 0x50, 0xa8, 0x95, 0x8b, 0x04, 0x82 }, + { 0xc1, 0xe0, 0x70, 0xb8, 0x9d, 0x8f, 0x06, 0x83 }, + { 0x01, 0x80, 0x40, 0x20, 0x11, 0x09, 0x05, 0x02 }, + { 0x81, 0xc0, 0x60, 0x30, 0x19, 0x0d, 0x07, 0x03 }, + { 0xf9, 0xfc, 0xfe, 0xff, 0x86, 0x3a, 0xe4, 0xf2 }, + { 0x79, 0xbc, 0xde, 0xef, 0x8e, 0x3e, 0xe6, 0xf3 }, + { 0xb9, 0xdc, 0xee, 0x77, 0x02, 0xb8, 0xe5, 0x72 }, + { 0x39, 0x9c, 0xce, 0x67, 0x0a, 0xbc, 0xe7, 0x73 }, + { 0xd9, 0xec, 0x76, 0x3b, 0x44, 0x7b, 0x64, 0xb2 }, + { 0x59, 0xac, 0x56, 0x2b, 0x4c, 0x7f, 0x66, 0xb3 }, + { 0x99, 0xcc, 0x66, 0xb3, 0xc0, 0xf9, 0x65, 0x32 }, + { 0x19, 0x8c, 0x46, 0xa3, 0xc8, 0xfd, 0x67, 0x33 }, + { 0xe9, 0x74, 0x3a, 0x1d, 0xe7, 0x9a, 0xa4, 0xd2 }, + { 0x69, 0x34, 0x1a, 0x0d, 0xef, 0x9e, 0xa6, 0xd3 }, + { 0xa9, 0x54, 0x2a, 0x95, 0x63, 0x18, 0xa5, 0x52 }, + { 0x29, 0x14, 0x0a, 0x85, 0x6b, 0x1c, 0xa7, 0x53 }, + { 0xc9, 0x64, 0xb2, 0xd9, 0x25, 0xdb, 0x24, 0x92 }, + { 0x49, 0x24, 0x92, 0xc9, 0x2d, 0xdf, 0x26, 0x93 }, + { 0x89, 0x44, 0xa2, 0x51, 0xa1, 0x59, 0x25, 0x12 }, + { 0x09, 0x04, 0x82, 0x41, 0xa9, 0x5d, 0x27, 0x13 }, + { 0xb5, 0xda, 0x6d, 0xb6, 0xee, 0xc2, 0xd4, 0x6a }, + { 0x35, 0x9a, 0x4d, 0xa6, 0xe6, 0xc6, 0xd6, 0x6b }, + { 0xf5, 0xfa, 0x7d, 0x3e, 0x6a, 0x40, 0xd5, 0xea }, + { 0x75, 0xba, 0x5d, 0x2e, 0x62, 0x44, 0xd7, 0xeb }, + { 0x95, 0xca, 0xe5, 0x72, 0x2c, 0x83, 0x54, 0x2a }, + { 0x15, 0x8a, 0xc5, 0x62, 0x24, 0x87, 0x56, 0x2b }, + { 0xd5, 0xea, 0xf5, 0xfa, 0xa8, 0x01, 0x55, 0xaa }, + { 0x55, 0xaa, 0xd5, 0xea, 0xa0, 0x05, 0x57, 0xab }, + { 0xa5, 0x52, 0xa9, 0x54, 0x8f, 0x62, 0x94, 0x4a }, + { 0x25, 0x12, 0x89, 0x44, 0x87, 0x66, 0x96, 0x4b }, + { 0xe5, 0x72, 0xb9, 0xdc, 0x0b, 0xe0, 0x95, 0xca }, + { 0x65, 0x32, 0x99, 0xcc, 0x03, 0xe4, 0x97, 0xcb }, + { 0x85, 0x42, 0x21, 0x90, 0x4d, 0x23, 0x14, 0x0a }, + { 0x05, 0x02, 0x01, 0x80, 0x45, 0x27, 0x16, 0x0b }, + { 0xc5, 0x62, 0x31, 0x18, 0xc9, 0xa1, 0x15, 0x8a }, + { 0x45, 0x22, 0x11, 0x08, 0xc1, 0xa5, 0x17, 0x8b }, + { 0x3d, 0x1e, 0x8f, 0xc7, 0x5e, 0x92, 0xf4, 0x7a }, + { 0xbd, 0x5e, 0xaf, 0xd7, 0x56, 0x96, 0xf6, 0x7b }, + { 0x7d, 0x3e, 0x9f, 0x4f, 0xda, 0x10, 0xf5, 0xfa }, + { 0xfd, 0x7e, 0xbf, 0x5f, 0xd2, 0x14, 0xf7, 0xfb }, + { 0x1d, 0x0e, 0x07, 0x03, 0x9c, 0xd3, 0x74, 0x3a }, + { 0x9d, 0x4e, 0x27, 0x13, 0x94, 0xd7, 0x76, 0x3b }, + { 0x5d, 0x2e, 0x17, 0x8b, 0x18, 0x51, 0x75, 0xba }, + { 0xdd, 0x6e, 0x37, 0x9b, 0x10, 0x55, 0x77, 0xbb }, + { 0x2d, 0x96, 0x4b, 0x25, 0x3f, 0x32, 0xb4, 0x5a }, + { 0xad, 0xd6, 0x6b, 0x35, 0x37, 0x36, 0xb6, 0x5b }, + { 0x6d, 0xb6, 0x5b, 0xad, 0xbb, 0xb0, 0xb5, 0xda }, + { 0xed, 0xf6, 0x7b, 0xbd, 0xb3, 0xb4, 0xb7, 0xdb }, + { 0x0d, 0x86, 0xc3, 0xe1, 0xfd, 0x73, 0x34, 0x1a }, + { 0x8d, 0xc6, 0xe3, 0xf1, 0xf5, 0x77, 0x36, 0x1b }, + { 0x4d, 0xa6, 0xd3, 0x69, 0x79, 0xf1, 0x35, 0x9a }, + { 0xcd, 0xe6, 0xf3, 0x79, 0x71, 0xf5, 0x37, 0x9b }, + { 0x93, 0x49, 0x24, 0x92, 0x5a, 0xbe, 0x4c, 0x26 }, + { 0x13, 0x09, 0x04, 0x82, 0x52, 0xba, 0x4e, 0x27 }, + { 0xd3, 0x69, 0x34, 0x1a, 0xde, 0x3c, 0x4d, 0xa6 }, + { 0x53, 0x29, 0x14, 0x0a, 0xd6, 0x38, 0x4f, 0xa7 }, + { 0xb3, 0x59, 0xac, 0x56, 0x98, 0xff, 0xcc, 0x66 }, + { 0x33, 0x19, 0x8c, 0x46, 0x90, 0xfb, 0xce, 0x67 }, + { 0xf3, 0x79, 0xbc, 0xde, 0x1c, 0x7d, 0xcd, 0xe6 }, + { 0x73, 0x39, 0x9c, 0xce, 0x14, 0x79, 0xcf, 0xe7 }, + { 0x83, 0xc1, 0xe0, 0x70, 0x3b, 0x1e, 0x0c, 0x06 }, + { 0x03, 0x81, 0xc0, 0x60, 0x33, 0x1a, 0x0e, 0x07 }, + { 0xc3, 0xe1, 0xf0, 0xf8, 0xbf, 0x9c, 0x0d, 0x86 }, + { 0x43, 0xa1, 0xd0, 0xe8, 0xb7, 0x98, 0x0f, 0x87 }, + { 0xa3, 0xd1, 0x68, 0xb4, 0xf9, 0x5f, 0x8c, 0x46 }, + { 0x23, 0x91, 0x48, 0xa4, 0xf1, 0x5b, 0x8e, 0x47 }, + { 0xe3, 0xf1, 0x78, 0x3c, 0x7d, 0xdd, 0x8d, 0xc6 }, + { 0x63, 0xb1, 0x58, 0x2c, 0x75, 0xd9, 0x8f, 0xc7 }, + { 0x1b, 0x8d, 0xc6, 0xe3, 0xea, 0xee, 0x6c, 0x36 }, + { 0x9b, 0xcd, 0xe6, 0xf3, 0xe2, 0xea, 0x6e, 0x37 }, + { 0x5b, 0xad, 0xd6, 0x6b, 0x6e, 0x6c, 0x6d, 0xb6 }, + { 0xdb, 0xed, 0xf6, 0x7b, 0x66, 0x68, 0x6f, 0xb7 }, + { 0x3b, 0x9d, 0x4e, 0x27, 0x28, 0xaf, 0xec, 0x76 }, + { 0xbb, 0xdd, 0x6e, 0x37, 0x20, 0xab, 0xee, 0x77 }, + { 0x7b, 0xbd, 0x5e, 0xaf, 0xac, 0x2d, 0xed, 0xf6 }, + { 0xfb, 0xfd, 0x7e, 0xbf, 0xa4, 0x29, 0xef, 0xf7 }, + { 0x0b, 0x05, 0x02, 0x01, 0x8b, 0x4e, 0x2c, 0x16 }, + { 0x8b, 0x45, 0x22, 0x11, 0x83, 0x4a, 0x2e, 0x17 }, + { 0x4b, 0x25, 0x12, 0x89, 0x0f, 0xcc, 0x2d, 0x96 }, + { 0xcb, 0x65, 0x32, 0x99, 0x07, 0xc8, 0x2f, 0x97 }, + { 0x2b, 0x15, 0x8a, 0xc5, 0x49, 0x0f, 0xac, 0x56 }, + { 0xab, 0x55, 0xaa, 0xd5, 0x41, 0x0b, 0xae, 0x57 }, + { 0x6b, 0x35, 0x9a, 0x4d, 0xcd, 0x8d, 0xad, 0xd6 }, + { 0xeb, 0x75, 0xba, 0x5d, 0xc5, 0x89, 0xaf, 0xd7 }, + { 0x57, 0xab, 0x55, 0xaa, 0x82, 0x16, 0x5c, 0xae }, + { 0xd7, 0xeb, 0x75, 0xba, 0x8a, 0x12, 0x5e, 0xaf }, + { 0x17, 0x8b, 0x45, 0x22, 0x06, 0x94, 0x5d, 0x2e }, + { 0x97, 0xcb, 0x65, 0x32, 0x0e, 0x90, 0x5f, 0x2f }, + { 0x77, 0xbb, 0xdd, 0x6e, 0x40, 0x57, 0xdc, 0xee }, + { 0xf7, 0xfb, 0xfd, 0x7e, 0x48, 0x53, 0xde, 0xef }, + { 0x37, 0x9b, 0xcd, 0xe6, 0xc4, 0xd5, 0xdd, 0x6e }, + { 0xb7, 0xdb, 0xed, 0xf6, 0xcc, 0xd1, 0xdf, 0x6f }, + { 0x47, 0x23, 0x91, 0x48, 0xe3, 0xb6, 0x1c, 0x8e }, + { 0xc7, 0x63, 0xb1, 0x58, 0xeb, 0xb2, 0x1e, 0x8f }, + { 0x07, 0x03, 0x81, 0xc0, 0x67, 0x34, 0x1d, 0x0e }, + { 0x87, 0x43, 0xa1, 0xd0, 0x6f, 0x30, 0x1f, 0x0f }, + { 0x67, 0x33, 0x19, 0x8c, 0x21, 0xf7, 0x9c, 0xce }, + { 0xe7, 0x73, 0x39, 0x9c, 0x29, 0xf3, 0x9e, 0xcf }, + { 0x27, 0x13, 0x09, 0x04, 0xa5, 0x75, 0x9d, 0x4e }, + { 0xa7, 0x53, 0x29, 0x14, 0xad, 0x71, 0x9f, 0x4f }, + { 0xdf, 0x6f, 0xb7, 0xdb, 0x32, 0x46, 0x7c, 0xbe }, + { 0x5f, 0x2f, 0x97, 0xcb, 0x3a, 0x42, 0x7e, 0xbf }, + { 0x9f, 0x4f, 0xa7, 0x53, 0xb6, 0xc4, 0x7d, 0x3e }, + { 0x1f, 0x0f, 0x87, 0x43, 0xbe, 0xc0, 0x7f, 0x3f }, + { 0xff, 0x7f, 0x3f, 0x1f, 0xf0, 0x07, 0xfc, 0xfe }, + { 0x7f, 0x3f, 0x1f, 0x0f, 0xf8, 0x03, 0xfe, 0xff }, + { 0xbf, 0x5f, 0x2f, 0x97, 0x74, 0x85, 0xfd, 0x7e }, + { 0x3f, 0x1f, 0x0f, 0x87, 0x7c, 0x81, 0xff, 0x7f }, + { 0xcf, 0xe7, 0x73, 0x39, 0x53, 0xe6, 0x3c, 0x9e }, + { 0x4f, 0xa7, 0x53, 0x29, 0x5b, 0xe2, 0x3e, 0x9f }, + { 0x8f, 0xc7, 0x63, 0xb1, 0xd7, 0x64, 0x3d, 0x1e }, + { 0x0f, 0x87, 0x43, 0xa1, 0xdf, 0x60, 0x3f, 0x1f }, + { 0xef, 0xf7, 0xfb, 0xfd, 0x91, 0xa7, 0xbc, 0xde }, + { 0x6f, 0xb7, 0xdb, 0xed, 0x99, 0xa3, 0xbe, 0xdf }, + { 0xaf, 0xd7, 0xeb, 0x75, 0x15, 0x25, 0xbd, 0x5e }, + { 0x2f, 0x97, 0xcb, 0x65, 0x1d, 0x21, 0xbf, 0x5f }, +}; + const uint8_t __aligned(256) raid_gfexp_raid[256] = { 0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80, diff --git a/raid/test.c b/raid/test.c index 14bccd3a..682747b2 100644 --- a/raid/test.c +++ b/raid/test.c @@ -40,6 +40,39 @@ static void meminc(void *dst, size_t size) ++p[i]; } +/** + * Simulate the bit-level semantics of VGF2P8AFFINEQB with imm8=0. + */ +static uint8_t raid_gfaffine_mul(uint8_t c, uint8_t x) +{ + uint8_t v = 0; + int j; + + for (j = 0; j < 8; ++j) { + uint8_t row = raid_gfaffine_raid[c][7 - j] & x; + + row ^= row >> 4; + row ^= row >> 2; + row ^= row >> 1; + v |= (row & 1U) << j; + } + + return v; +} + +int raid_test_gfaffine(void) +{ + int c, x; + + /* all coefficients, including 0, 1, 2, and 0xff, are mandatory */ + for (c = 0; c < 256; ++c) + for (x = 0; x < 256; ++x) + if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x]) + return -1; + + return 0; +} + int raid_test_combo(void) { int r; @@ -582,13 +615,17 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(1) = raid_gen1_avx512bw; test_setup(2) = raid_gen2_avx512bw; } - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) test_setup(2) = raid_gen2_avx2gfni_aes; - } - if (raid_cpu_has_avx512gfni()) { + else + test_setup(2) = raid_gen2_avx2gfni_raid; + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) test_setup(2) = raid_gen2_avx512gfni_aes; - } + else + test_setup(2) = raid_gen2_avx512gfni_raid; } #endif #endif @@ -658,18 +695,30 @@ int raid_test_par(int mode, int nd, size_t size) test_setup(5) = raid_gen5_avx512bw; test_setup(6) = raid_gen6_avx512bw; } - if (mode == RAID_MODE_CAUCHY_AES) { - if (raid_cpu_has_avx2gfni()) { + if (raid_cpu_has_avx2gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { test_setup(3) = raid_gen3_avx2gfni_aes; test_setup(4) = raid_gen4_avx2gfni_aes; test_setup(5) = raid_gen5_avx2gfni_aes; test_setup(6) = raid_gen6_avx2gfni_aes; + } else { + test_setup(3) = raid_gen3_avx2gfni_raid; + test_setup(4) = raid_gen4_avx2gfni_raid; + test_setup(5) = raid_gen5_avx2gfni_raid; + test_setup(6) = raid_gen6_avx2gfni_raid; } - if (raid_cpu_has_avx512gfni()) { + } + if (raid_cpu_has_avx512gfni()) { + if (mode == RAID_MODE_CAUCHY_AES) { test_setup(3) = raid_gen3_avx512gfni_aes; test_setup(4) = raid_gen4_avx512gfni_aes; test_setup(5) = raid_gen5_avx512gfni_aes; test_setup(6) = raid_gen6_avx512gfni_aes; + } else { + test_setup(3) = raid_gen3_avx512gfni_raid; + test_setup(4) = raid_gen4_avx512gfni_raid; + test_setup(5) = raid_gen5_avx512gfni_raid; + test_setup(6) = raid_gen6_avx512gfni_raid; } } #endif diff --git a/raid/test.h b/raid/test.h index af10522c..04a2f66e 100644 --- a/raid/test.h +++ b/raid/test.h @@ -31,6 +31,13 @@ int raid_test_sort(void); */ int raid_test_combo(void); +/** + * Tests all RAID affine GFNI matrices against scalar GF multiplication. + * + * Returns 0 on success. + */ +int raid_test_gfaffine(void); + /** * Tests the Galois field and Cauchy matrix properties. * diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index 1c6c9112..df53042c 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -73,6 +73,13 @@ int main(void) /* LCOV_EXCL_STOP */ } + printf("Test GFNI affine matrices...\n"); + if (raid_test_gfaffine() != 0) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } + printf("Test Cauchy RAID parity generation with 1-%u data disks...\n", RAID_DATA_MAX); for (int i = 1; i <= RAID_DATA_MAX; ++i) { if (raid_test_par(RAID_MODE_CAUCHY_RAID, i, TEST_SIZE) != 0) { diff --git a/raid/test/selftest.c b/raid/test/selftest.c index f7b1c474..b68d36df 100644 --- a/raid/test/selftest.c +++ b/raid/test/selftest.c @@ -16,6 +16,13 @@ int main(void) raid_init(); + printf("Test GFNI affine matrices\n"); + if (raid_test_gfaffine() != 0) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } + printf("Test Cauchy RAID\n"); if (raid_test_poly(RAID_MODE_CAUCHY_RAID) != 0) { /* LCOV_EXCL_START */ diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 6393d3f9..0a489529 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -208,6 +208,10 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); fflush(stdout); } + if (raid_cpu_has_avx2gfni()) + printf("%8s", ""); + if (raid_cpu_has_avx512gfni()) + printf("%8s", ""); #endif #endif printf("\n"); @@ -351,7 +355,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -363,7 +371,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen2_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -470,7 +482,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -482,7 +498,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen3_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -589,7 +609,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -601,7 +625,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen4_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -706,7 +734,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -718,7 +750,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen5_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -823,7 +859,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx2gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); } @@ -835,7 +875,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms printf("%8" PRIu64, ds / dt); } else { - printf("%8s", ""); + SPEED_START { + raid_gen6_avx512gfni_raid(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); } fflush(stdout); }