rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add GFNI affine parity generation for RAID polynomial

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-19 17:59:02 +02:00
parent 711ccba629
commit c34cfe24da
11 changed files with 805 additions and 28 deletions
Download patch file Download diff file Expand all files Collapse all files

6
cmdline/selftest.c
View file

@ -1151,6 +1151,12 @@ void selftest(void)
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_gfaffine() != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed GFNI AFFINE test\n");
exit(EXIT_FAILURE);
/* LCOV_EXCL_STOP */
}
if (raid_test_par(RAID_MODE_VANDERMONDE_RAID, 32, 256) != 0) {
/* LCOV_EXCL_START */
log_fatal(EINTERNAL, "Failed GEN Vandermonde RAID test\n");

64
cmdline/speed.c
View file

@ -306,6 +306,10 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni())
printf("%8s", "");
if (raid_cpu_has_avx512gfni())
printf("%8s", "");
#endif
#endif
printf("\n");
@ -449,7 +453,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen2_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -461,7 +469,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen2_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -570,7 +582,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen3_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -582,7 +598,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen3_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -691,7 +711,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen4_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -703,7 +727,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen4_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -809,7 +837,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen5_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -821,7 +853,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen5_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -927,7 +963,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen6_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -939,7 +979,11 @@ void speed_gen(int nd, void** v, int size, int delta, int period, const char* ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen6_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}

282
raid/gfni.c
View file

@ -6,6 +6,196 @@
#include "cpu.h"
#ifdef CONFIG_X86_64
/*
* GENX AVX2 GFNI implementation
*/
static __always_inline void raid_genX_avx2gfni_raid(int nd, size_t size, void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d;
if (nd == 1) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
raid_avx_begin();
for (i = 0; i < size; i += 64) {
/* first disk with all coefficients at 1 */
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[0][i]));
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[0][i + 32]));
asm volatile ("vmovdqa %ymm0,%ymm2");
asm volatile ("vmovdqa %ymm1,%ymm3");
if (np >= 3) {
asm volatile ("vmovdqa %ymm0,%ymm4");
asm volatile ("vmovdqa %ymm1,%ymm5");
}
if (np >= 4) {
asm volatile ("vmovdqa %ymm0,%ymm6");
asm volatile ("vmovdqa %ymm1,%ymm7");
}
if (np >= 5) {
asm volatile ("vmovdqa %ymm0,%ymm8");
asm volatile ("vmovdqa %ymm1,%ymm9");
}
if (np >= 6) {
asm volatile ("vmovdqa %ymm0,%ymm10");
asm volatile ("vmovdqa %ymm1,%ymm11");
}
/* all other disks */
for (d = 1; d < nd; ++d) {
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32]));
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[1][d]][0]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
if (np >= 3) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
}
if (np >= 4) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
}
if (np >= 5) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
}
if (np >= 6) {
asm volatile ("vpbroadcastq %0,%%ymm14" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0]));
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm12,%ymm15");
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
asm volatile ("vgf2p8affineqb $0,%ymm14,%ymm13,%ymm15");
asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
}
}
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (v[nd][i]));
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (v[nd][i + 32]));
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (v[nd + 1][i]));
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (v[nd + 1][i + 32]));
if (np >= 3) {
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (v[nd + 2][i]));
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (v[nd + 2][i + 32]));
}
if (np >= 4) {
asm volatile ("vmovntdq %%ymm6,%0" : "=m" (v[nd + 3][i]));
asm volatile ("vmovntdq %%ymm7,%0" : "=m" (v[nd + 3][i + 32]));
}
if (np >= 5) {
asm volatile ("vmovntdq %%ymm8,%0" : "=m" (v[nd + 4][i]));
asm volatile ("vmovntdq %%ymm9,%0" : "=m" (v[nd + 4][i + 32]));
}
if (np >= 6) {
asm volatile ("vmovntdq %%ymm10,%0" : "=m" (v[nd + 5][i]));
asm volatile ("vmovntdq %%ymm11,%0" : "=m" (v[nd + 5][i + 32]));
}
}
raid_avx_end();
}
/*
* GENX AVX512 GFNI implementation
*/
static __always_inline void raid_genX_avx512gfni_raid(int nd, size_t size, void **vv, int np)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
int d;
if (nd == 1) {
for (d = 0; d < np; ++d)
memcpy(v[1 + d], v[0], size);
return;
}
raid_avx_begin();
for (i = 0; i < size; i += 64) {
/* first disk with all coefficients at 1 */
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[0][i]));
asm volatile ("vmovdqa64 %zmm0,%zmm1");
if (np >= 3)
asm volatile ("vmovdqa64 %zmm0,%zmm2");
if (np >= 4)
asm volatile ("vmovdqa64 %zmm0,%zmm3");
if (np >= 5)
asm volatile ("vmovdqa64 %zmm0,%zmm4");
if (np >= 6)
asm volatile ("vmovdqa64 %zmm0,%zmm5");
/* all other disks */
for (d = 1; d < nd; ++d) {
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
asm volatile ("vpbroadcastq %0,%%zmm7" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[1][d]][0]));
if (np >= 3)
asm volatile ("vpbroadcastq %0,%%zmm8" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[2][d]][0]));
if (np >= 4)
asm volatile ("vpbroadcastq %0,%%zmm9" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[3][d]][0]));
if (np >= 5)
asm volatile ("vpbroadcastq %0,%%zmm10" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[4][d]][0]));
if (np >= 6)
asm volatile ("vpbroadcastq %0,%%zmm11" : : "m" (raid_gfaffine_raid[raid_gfcauchy_raid[5][d]][0]));
asm volatile ("vgf2p8affineqb $0,%zmm7,%zmm6,%zmm12");
if (np >= 3)
asm volatile ("vgf2p8affineqb $0,%zmm8,%zmm6,%zmm13");
if (np >= 4)
asm volatile ("vgf2p8affineqb $0,%zmm9,%zmm6,%zmm14");
if (np >= 5)
asm volatile ("vgf2p8affineqb $0,%zmm10,%zmm6,%zmm15");
if (np >= 6)
asm volatile ("vgf2p8affineqb $0,%zmm11,%zmm6,%zmm16");
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
asm volatile ("vpxorq %zmm12,%zmm1,%zmm1");
if (np >= 3)
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
if (np >= 4)
asm volatile ("vpxorq %zmm14,%zmm3,%zmm3");
if (np >= 5)
asm volatile ("vpxorq %zmm15,%zmm4,%zmm4");
if (np >= 6)
asm volatile ("vpxorq %zmm16,%zmm5,%zmm5");
}
asm volatile ("vmovntdq %%zmm0,%0" : "=m" (v[nd][i]));
asm volatile ("vmovntdq %%zmm1,%0" : "=m" (v[nd + 1][i]));
if (np >= 3)
asm volatile ("vmovntdq %%zmm2,%0" : "=m" (v[nd + 2][i]));
if (np >= 4)
asm volatile ("vmovntdq %%zmm3,%0" : "=m" (v[nd + 3][i]));
if (np >= 5)
asm volatile ("vmovntdq %%zmm4,%0" : "=m" (v[nd + 4][i]));
if (np >= 6)
asm volatile ("vmovntdq %%zmm5,%0" : "=m" (v[nd + 5][i]));
}
raid_avx_end();
}
/*
* GENX AVX2 GFNI implementation
@ -198,6 +388,86 @@ static __always_inline void raid_genX_avx512gfni_aes(int nd, size_t size, void *
raid_avx_end();
}
/*
* GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation
*/
void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni_raid(nd, size, vv, 2);
}
/*
* GEN2 (RAID6 with powers of 2) GFNI implementation
*/
void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni_raid(nd, size, vv, 2);
}
/*
* GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni_raid(nd, size, vv, 3);
}
/*
* GEN3 (triple parity with Cauchy matrix) GFNI implementation
*/
void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni_raid(nd, size, vv, 3);
}
/*
* GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni_raid(nd, size, vv, 4);
}
/*
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
*/
void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni_raid(nd, size, vv, 4);
}
/*
* GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni_raid(nd, size, vv, 5);
}
/*
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
*/
void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni_raid(nd, size, vv, 5);
}
/*
* GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation
*/
void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx2gfni_raid(nd, size, vv, 6);
}
/*
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
*/
void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv)
{
raid_genX_avx512gfni_raid(nd, size, vv, 6);
}
/*
* GEN2 Cauchy AVX2 GFNI implementation
*/
@ -636,6 +906,12 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
void raid_register_avx2gfni(void)
{
if (raid_cpu_has_avx2gfni()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni_aes, RAID_POLY_AES);
@ -655,6 +931,12 @@ void raid_register_avx512gfni(void)
{
if (raid_cpu_has_avx512gfni()) {
if (!raid_cpu_has_slow_avx512()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni512", raid_gen5_avx512gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni512", raid_gen6_avx512gfni_raid, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni512", raid_gen3_avx512gfni_aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni512", raid_gen4_avx512gfni_aes, RAID_POLY_AES);

11
raid/internal.h
View file

@ -104,6 +104,8 @@ void raid_gen2_sse2ext_aes(int nd, size_t size, void **vv);
void raid_gen2_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen2_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen2_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_avx2gfni_raid(int nd, size_t size, void **vv);
void raid_gen2_avx512gfni_raid(int nd, size_t size, void **vv);
void raid_gen2_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen2_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_genz_int32_raid(int nd, size_t size, void **vv);
@ -119,6 +121,8 @@ void raid_gen3_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen3_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen3_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen3_avx512bw(int nd, size_t size, void **vv);
void raid_gen3_avx2gfni_raid(int nd, size_t size, void **vv);
void raid_gen3_avx512gfni_raid(int nd, size_t size, void **vv);
void raid_gen3_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen3_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen4_int8(int nd, size_t size, void **vv);
@ -129,6 +133,8 @@ void raid_gen4_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen4_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen4_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen4_avx512bw(int nd, size_t size, void **vv);
void raid_gen4_avx2gfni_raid(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni_raid(int nd, size_t size, void **vv);
void raid_gen4_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_int8(int nd, size_t size, void **vv);
@ -138,6 +144,8 @@ void raid_gen5_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen5_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen5_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen5_avx512bw(int nd, size_t size, void **vv);
void raid_gen5_avx2gfni_raid(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni_raid(int nd, size_t size, void **vv);
void raid_gen5_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_int8(int nd, size_t size, void **vv);
@ -147,6 +155,8 @@ void raid_gen6_ssse3ext_aes(int nd, size_t size, void **vv);
void raid_gen6_avx2ext_raid(int nd, size_t size, void **vv);
void raid_gen6_avx2ext_aes(int nd, size_t size, void **vv);
void raid_gen6_avx512bw(int nd, size_t size, void **vv);
void raid_gen6_avx2gfni_raid(int nd, size_t size, void **vv);
void raid_gen6_avx512gfni_raid(int nd, size_t size, void **vv);
void raid_gen6_avx2gfni_aes(int nd, size_t size, void **vv);
void raid_gen6_avx512gfni_aes(int nd, size_t size, void **vv);
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
@ -325,6 +335,7 @@ extern const uint8_t raid_gfexp_raid[256] __aligned(256);
extern const uint8_t raid_gfinv_raid[256] __aligned(256);
extern const uint8_t raid_gfvandermonde_raid[3][256] __aligned(256);
extern const uint8_t raid_gfcauchy_raid[6][256] __aligned(256);
extern const uint8_t raid_gfaffine_raid[256][8] __aligned(256);
#if defined(CONFIG_X86) || defined(CONFIG_NEON)
extern const uint8_t raid_gfcauchypshufb_raid[251][5][2][16] __aligned(256);
extern const uint8_t raid_gfmulpshufb_raid[256][2][16] __aligned(256);

53
raid/mktables.c
View file

@ -38,6 +38,34 @@ static uint8_t raid_gfmul(uint8_t poly, uint8_t a, uint8_t b)
return v;
}
/**
* Setup the GFNI affine matrix for multiplication by a constant.
*
* VGF2P8MULB is fixed to the AES polynomial 0x11b, while RAID parity is
* defined over GF(2^8) modulo 0x11d. VGF2P8AFFINEQB can implement the
* multiplication because multiplication by a constant is GF(2)-linear.
*
* VGF2P8AFFINEQB stores the row producing output bit j in byte 7-j.
* Within each row, bit k selects input bit k.
*/
static void set_affine(uint8_t poly, uint8_t c, uint8_t *matrix)
{
int j, k;
for (j = 0; j < 8; ++j) {
uint8_t row = 0;
for (k = 0; k < 8; ++k) {
uint8_t v = raid_gfmul(poly, c, (uint8_t)(1U << k));
if ((v & (1U << j)) != 0)
row |= (uint8_t)(1U << k);
}
matrix[7 - j] = row;
}
}
/**
* Inversion (1/a) in GF(2^8).
*/
@ -231,6 +259,31 @@ void tables(uint8_t poly, uint8_t generator, const char *tag)
}
printf("};\n\n");
if (poly == RAID_POLY_RAID) {
uint8_t affine[8];
printf("/**\n");
printf(" * GFNI affine matrices for multiplication in GF(2^8)/0x11d.\n");
printf(" *\n");
printf(" * VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane.\n");
printf(" * Byte 7-j contains the row producing output bit j, and bit k\n");
printf(" * of that byte selects input bit k. The instruction uses imm8=0.\n");
printf(" */\n");
printf("const uint8_t __aligned(256) raid_gfaffine_raid[256][8] =\n");
printf("{\n");
for (i = 0; i < 256; ++i) {
set_affine(poly, (uint8_t)i, affine);
printf("\t{ ");
for (j = 0; j < 8; ++j) {
printf("0x%02x", (unsigned)affine[j]);
if (j != 7)
printf(", ");
}
printf(" },\n");
}
printf("};\n\n");
}
/* generator^a */
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
printf("{\n");

267
raid/tables.c
View file

@ -8713,6 +8713,273 @@ const uint8_t __aligned(256) raid_gfmul_raid[256][256] =
},
};
/**
* GFNI affine matrices for multiplication in GF(2^8)/0x11d.
*
* VGF2P8AFFINEQB consumes one 8x8 matrix per 64-bit lane.
* Byte 7-j contains the row producing output bit j, and bit k
* of that byte selects input bit k. The instruction uses imm8=0.
*/
const uint8_t __aligned(256) raid_gfaffine_raid[256][8] =
{
{ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00 },
{ 0x80, 0x40, 0x20, 0x10, 0x08, 0x04, 0x02, 0x01 },
{ 0x40, 0x20, 0x10, 0x88, 0x84, 0x82, 0x01, 0x80 },
{ 0xc0, 0x60, 0x30, 0x98, 0x8c, 0x86, 0x03, 0x81 },
{ 0x20, 0x10, 0x88, 0xc4, 0xc2, 0x41, 0x80, 0x40 },
{ 0xa0, 0x50, 0xa8, 0xd4, 0xca, 0x45, 0x82, 0x41 },
{ 0x60, 0x30, 0x98, 0x4c, 0x46, 0xc3, 0x81, 0xc0 },
{ 0xe0, 0x70, 0xb8, 0x5c, 0x4e, 0xc7, 0x83, 0xc1 },
{ 0x10, 0x88, 0xc4, 0xe2, 0x61, 0xa0, 0x40, 0x20 },
{ 0x90, 0xc8, 0xe4, 0xf2, 0x69, 0xa4, 0x42, 0x21 },
{ 0x50, 0xa8, 0xd4, 0x6a, 0xe5, 0x22, 0x41, 0xa0 },
{ 0xd0, 0xe8, 0xf4, 0x7a, 0xed, 0x26, 0x43, 0xa1 },
{ 0x30, 0x98, 0x4c, 0x26, 0xa3, 0xe1, 0xc0, 0x60 },
{ 0xb0, 0xd8, 0x6c, 0x36, 0xab, 0xe5, 0xc2, 0x61 },
{ 0x70, 0xb8, 0x5c, 0xae, 0x27, 0x63, 0xc1, 0xe0 },
{ 0xf0, 0xf8, 0x7c, 0xbe, 0x2f, 0x67, 0xc3, 0xe1 },
{ 0x88, 0xc4, 0xe2, 0x71, 0xb0, 0x50, 0x20, 0x10 },
{ 0x08, 0x84, 0xc2, 0x61, 0xb8, 0x54, 0x22, 0x11 },
{ 0xc8, 0xe4, 0xf2, 0xf9, 0x34, 0xd2, 0x21, 0x90 },
{ 0x48, 0xa4, 0xd2, 0xe9, 0x3c, 0xd6, 0x23, 0x91 },
{ 0xa8, 0xd4, 0x6a, 0xb5, 0x72, 0x11, 0xa0, 0x50 },
{ 0x28, 0x94, 0x4a, 0xa5, 0x7a, 0x15, 0xa2, 0x51 },
{ 0xe8, 0xf4, 0x7a, 0x3d, 0xf6, 0x93, 0xa1, 0xd0 },
{ 0x68, 0xb4, 0x5a, 0x2d, 0xfe, 0x97, 0xa3, 0xd1 },
{ 0x98, 0x4c, 0x26, 0x93, 0xd1, 0xf0, 0x60, 0x30 },
{ 0x18, 0x0c, 0x06, 0x83, 0xd9, 0xf4, 0x62, 0x31 },
{ 0xd8, 0x6c, 0x36, 0x1b, 0x55, 0x72, 0x61, 0xb0 },
{ 0x58, 0x2c, 0x16, 0x0b, 0x5d, 0x76, 0x63, 0xb1 },
{ 0xb8, 0x5c, 0xae, 0x57, 0x13, 0xb1, 0xe0, 0x70 },
{ 0x38, 0x1c, 0x8e, 0x47, 0x1b, 0xb5, 0xe2, 0x71 },
{ 0xf8, 0x7c, 0xbe, 0xdf, 0x97, 0x33, 0xe1, 0xf0 },
{ 0x78, 0x3c, 0x9e, 0xcf, 0x9f, 0x37, 0xe3, 0xf1 },
{ 0xc4, 0xe2, 0x71, 0x38, 0xd8, 0xa8, 0x10, 0x88 },
{ 0x44, 0xa2, 0x51, 0x28, 0xd0, 0xac, 0x12, 0x89 },
{ 0x84, 0xc2, 0x61, 0xb0, 0x5c, 0x2a, 0x11, 0x08 },
{ 0x04, 0x82, 0x41, 0xa0, 0x54, 0x2e, 0x13, 0x09 },
{ 0xe4, 0xf2, 0xf9, 0xfc, 0x1a, 0xe9, 0x90, 0xc8 },
{ 0x64, 0xb2, 0xd9, 0xec, 0x12, 0xed, 0x92, 0xc9 },
{ 0xa4, 0xd2, 0xe9, 0x74, 0x9e, 0x6b, 0x91, 0x48 },
{ 0x24, 0x92, 0xc9, 0x64, 0x96, 0x6f, 0x93, 0x49 },
{ 0xd4, 0x6a, 0xb5, 0xda, 0xb9, 0x08, 0x50, 0xa8 },
{ 0x54, 0x2a, 0x95, 0xca, 0xb1, 0x0c, 0x52, 0xa9 },
{ 0x94, 0x4a, 0xa5, 0x52, 0x3d, 0x8a, 0x51, 0x28 },
{ 0x14, 0x0a, 0x85, 0x42, 0x35, 0x8e, 0x53, 0x29 },
{ 0xf4, 0x7a, 0x3d, 0x1e, 0x7b, 0x49, 0xd0, 0xe8 },
{ 0x74, 0x3a, 0x1d, 0x0e, 0x73, 0x4d, 0xd2, 0xe9 },
{ 0xb4, 0x5a, 0x2d, 0x96, 0xff, 0xcb, 0xd1, 0x68 },
{ 0x34, 0x1a, 0x0d, 0x86, 0xf7, 0xcf, 0xd3, 0x69 },
{ 0x4c, 0x26, 0x93, 0x49, 0x68, 0xf8, 0x30, 0x98 },
{ 0xcc, 0x66, 0xb3, 0x59, 0x60, 0xfc, 0x32, 0x99 },
{ 0x0c, 0x06, 0x83, 0xc1, 0xec, 0x7a, 0x31, 0x18 },
{ 0x8c, 0x46, 0xa3, 0xd1, 0xe4, 0x7e, 0x33, 0x19 },
{ 0x6c, 0x36, 0x1b, 0x8d, 0xaa, 0xb9, 0xb0, 0xd8 },
{ 0xec, 0x76, 0x3b, 0x9d, 0xa2, 0xbd, 0xb2, 0xd9 },
{ 0x2c, 0x16, 0x0b, 0x05, 0x2e, 0x3b, 0xb1, 0x58 },
{ 0xac, 0x56, 0x2b, 0x15, 0x26, 0x3f, 0xb3, 0x59 },
{ 0x5c, 0xae, 0x57, 0xab, 0x09, 0x58, 0x70, 0xb8 },
{ 0xdc, 0xee, 0x77, 0xbb, 0x01, 0x5c, 0x72, 0xb9 },
{ 0x1c, 0x8e, 0x47, 0x23, 0x8d, 0xda, 0x71, 0x38 },
{ 0x9c, 0xce, 0x67, 0x33, 0x85, 0xde, 0x73, 0x39 },
{ 0x7c, 0xbe, 0xdf, 0x6f, 0xcb, 0x19, 0xf0, 0xf8 },
{ 0xfc, 0xfe, 0xff, 0x7f, 0xc3, 0x1d, 0xf2, 0xf9 },
{ 0x3c, 0x9e, 0xcf, 0xe7, 0x4f, 0x9b, 0xf1, 0x78 },
{ 0xbc, 0xde, 0xef, 0xf7, 0x47, 0x9f, 0xf3, 0x79 },
{ 0xe2, 0x71, 0x38, 0x1c, 0x6c, 0xd4, 0x88, 0xc4 },
{ 0x62, 0x31, 0x18, 0x0c, 0x64, 0xd0, 0x8a, 0xc5 },
{ 0xa2, 0x51, 0x28, 0x94, 0xe8, 0x56, 0x89, 0x44 },
{ 0x22, 0x11, 0x08, 0x84, 0xe0, 0x52, 0x8b, 0x45 },
{ 0xc2, 0x61, 0xb0, 0xd8, 0xae, 0x95, 0x08, 0x84 },
{ 0x42, 0x21, 0x90, 0xc8, 0xa6, 0x91, 0x0a, 0x85 },
{ 0x82, 0x41, 0xa0, 0x50, 0x2a, 0x17, 0x09, 0x04 },
{ 0x02, 0x01, 0x80, 0x40, 0x22, 0x13, 0x0b, 0x05 },
{ 0xf2, 0xf9, 0xfc, 0xfe, 0x0d, 0x74, 0xc8, 0xe4 },
{ 0x72, 0xb9, 0xdc, 0xee, 0x05, 0x70, 0xca, 0xe5 },
{ 0xb2, 0xd9, 0xec, 0x76, 0x89, 0xf6, 0xc9, 0x64 },
{ 0x32, 0x99, 0xcc, 0x66, 0x81, 0xf2, 0xcb, 0x65 },
{ 0xd2, 0xe9, 0x74, 0x3a, 0xcf, 0x35, 0x48, 0xa4 },
{ 0x52, 0xa9, 0x54, 0x2a, 0xc7, 0x31, 0x4a, 0xa5 },
{ 0x92, 0xc9, 0x64, 0xb2, 0x4b, 0xb7, 0x49, 0x24 },
{ 0x12, 0x89, 0x44, 0xa2, 0x43, 0xb3, 0x4b, 0x25 },
{ 0x6a, 0xb5, 0xda, 0x6d, 0xdc, 0x84, 0xa8, 0xd4 },
{ 0xea, 0xf5, 0xfa, 0x7d, 0xd4, 0x80, 0xaa, 0xd5 },
{ 0x2a, 0x95, 0xca, 0xe5, 0x58, 0x06, 0xa9, 0x54 },
{ 0xaa, 0xd5, 0xea, 0xf5, 0x50, 0x02, 0xab, 0x55 },
{ 0x4a, 0xa5, 0x52, 0xa9, 0x1e, 0xc5, 0x28, 0x94 },
{ 0xca, 0xe5, 0x72, 0xb9, 0x16, 0xc1, 0x2a, 0x95 },
{ 0x0a, 0x85, 0x42, 0x21, 0x9a, 0x47, 0x29, 0x14 },
{ 0x8a, 0xc5, 0x62, 0x31, 0x92, 0x43, 0x2b, 0x15 },
{ 0x7a, 0x3d, 0x1e, 0x8f, 0xbd, 0x24, 0xe8, 0xf4 },
{ 0xfa, 0x7d, 0x3e, 0x9f, 0xb5, 0x20, 0xea, 0xf5 },
{ 0x3a, 0x1d, 0x0e, 0x07, 0x39, 0xa6, 0xe9, 0x74 },
{ 0xba, 0x5d, 0x2e, 0x17, 0x31, 0xa2, 0xeb, 0x75 },
{ 0x5a, 0x2d, 0x96, 0x4b, 0x7f, 0x65, 0x68, 0xb4 },
{ 0xda, 0x6d, 0xb6, 0x5b, 0x77, 0x61, 0x6a, 0xb5 },
{ 0x1a, 0x0d, 0x86, 0xc3, 0xfb, 0xe7, 0x69, 0x34 },
{ 0x9a, 0x4d, 0xa6, 0xd3, 0xf3, 0xe3, 0x6b, 0x35 },
{ 0x26, 0x93, 0x49, 0x24, 0xb4, 0x7c, 0x98, 0x4c },
{ 0xa6, 0xd3, 0x69, 0x34, 0xbc, 0x78, 0x9a, 0x4d },
{ 0x66, 0xb3, 0x59, 0xac, 0x30, 0xfe, 0x99, 0xcc },
{ 0xe6, 0xf3, 0x79, 0xbc, 0x38, 0xfa, 0x9b, 0xcd },
{ 0x06, 0x83, 0xc1, 0xe0, 0x76, 0x3d, 0x18, 0x0c },
{ 0x86, 0xc3, 0xe1, 0xf0, 0x7e, 0x39, 0x1a, 0x0d },
{ 0x46, 0xa3, 0xd1, 0x68, 0xf2, 0xbf, 0x19, 0x8c },
{ 0xc6, 0xe3, 0xf1, 0x78, 0xfa, 0xbb, 0x1b, 0x8d },
{ 0x36, 0x1b, 0x8d, 0xc6, 0xd5, 0xdc, 0xd8, 0x6c },
{ 0xb6, 0x5b, 0xad, 0xd6, 0xdd, 0xd8, 0xda, 0x6d },
{ 0x76, 0x3b, 0x9d, 0x4e, 0x51, 0x5e, 0xd9, 0xec },
{ 0xf6, 0x7b, 0xbd, 0x5e, 0x59, 0x5a, 0xdb, 0xed },
{ 0x16, 0x0b, 0x05, 0x02, 0x17, 0x9d, 0x58, 0x2c },
{ 0x96, 0x4b, 0x25, 0x12, 0x1f, 0x99, 0x5a, 0x2d },
{ 0x56, 0x2b, 0x15, 0x8a, 0x93, 0x1f, 0x59, 0xac },
{ 0xd6, 0x6b, 0x35, 0x9a, 0x9b, 0x1b, 0x5b, 0xad },
{ 0xae, 0x57, 0xab, 0x55, 0x04, 0x2c, 0xb8, 0x5c },
{ 0x2e, 0x17, 0x8b, 0x45, 0x0c, 0x28, 0xba, 0x5d },
{ 0xee, 0x77, 0xbb, 0xdd, 0x80, 0xae, 0xb9, 0xdc },
{ 0x6e, 0x37, 0x9b, 0xcd, 0x88, 0xaa, 0xbb, 0xdd },
{ 0x8e, 0x47, 0x23, 0x91, 0xc6, 0x6d, 0x38, 0x1c },
{ 0x0e, 0x07, 0x03, 0x81, 0xce, 0x69, 0x3a, 0x1d },
{ 0xce, 0x67, 0x33, 0x19, 0x42, 0xef, 0x39, 0x9c },
{ 0x4e, 0x27, 0x13, 0x09, 0x4a, 0xeb, 0x3b, 0x9d },
{ 0xbe, 0xdf, 0x6f, 0xb7, 0x65, 0x8c, 0xf8, 0x7c },
{ 0x3e, 0x9f, 0x4f, 0xa7, 0x6d, 0x88, 0xfa, 0x7d },
{ 0xfe, 0xff, 0x7f, 0x3f, 0xe1, 0x0e, 0xf9, 0xfc },
{ 0x7e, 0xbf, 0x5f, 0x2f, 0xe9, 0x0a, 0xfb, 0xfd },
{ 0x9e, 0xcf, 0xe7, 0x73, 0xa7, 0xcd, 0x78, 0x3c },
{ 0x1e, 0x8f, 0xc7, 0x63, 0xaf, 0xc9, 0x7a, 0x3d },
{ 0xde, 0xef, 0xf7, 0xfb, 0x23, 0x4f, 0x79, 0xbc },
{ 0x5e, 0xaf, 0xd7, 0xeb, 0x2b, 0x4b, 0x7b, 0xbd },
{ 0x71, 0x38, 0x1c, 0x8e, 0x36, 0x6a, 0xc4, 0xe2 },
{ 0xf1, 0x78, 0x3c, 0x9e, 0x3e, 0x6e, 0xc6, 0xe3 },
{ 0x31, 0x18, 0x0c, 0x06, 0xb2, 0xe8, 0xc5, 0x62 },
{ 0xb1, 0x58, 0x2c, 0x16, 0xba, 0xec, 0xc7, 0x63 },
{ 0x51, 0x28, 0x94, 0x4a, 0xf4, 0x2b, 0x44, 0xa2 },
{ 0xd1, 0x68, 0xb4, 0x5a, 0xfc, 0x2f, 0x46, 0xa3 },
{ 0x11, 0x08, 0x84, 0xc2, 0x70, 0xa9, 0x45, 0x22 },
{ 0x91, 0x48, 0xa4, 0xd2, 0x78, 0xad, 0x47, 0x23 },
{ 0x61, 0xb0, 0xd8, 0x6c, 0x57, 0xca, 0x84, 0xc2 },
{ 0xe1, 0xf0, 0xf8, 0x7c, 0x5f, 0xce, 0x86, 0xc3 },
{ 0x21, 0x90, 0xc8, 0xe4, 0xd3, 0x48, 0x85, 0x42 },
{ 0xa1, 0xd0, 0xe8, 0xf4, 0xdb, 0x4c, 0x87, 0x43 },
{ 0x41, 0xa0, 0x50, 0xa8, 0x95, 0x8b, 0x04, 0x82 },
{ 0xc1, 0xe0, 0x70, 0xb8, 0x9d, 0x8f, 0x06, 0x83 },
{ 0x01, 0x80, 0x40, 0x20, 0x11, 0x09, 0x05, 0x02 },
{ 0x81, 0xc0, 0x60, 0x30, 0x19, 0x0d, 0x07, 0x03 },
{ 0xf9, 0xfc, 0xfe, 0xff, 0x86, 0x3a, 0xe4, 0xf2 },
{ 0x79, 0xbc, 0xde, 0xef, 0x8e, 0x3e, 0xe6, 0xf3 },
{ 0xb9, 0xdc, 0xee, 0x77, 0x02, 0xb8, 0xe5, 0x72 },
{ 0x39, 0x9c, 0xce, 0x67, 0x0a, 0xbc, 0xe7, 0x73 },
{ 0xd9, 0xec, 0x76, 0x3b, 0x44, 0x7b, 0x64, 0xb2 },
{ 0x59, 0xac, 0x56, 0x2b, 0x4c, 0x7f, 0x66, 0xb3 },
{ 0x99, 0xcc, 0x66, 0xb3, 0xc0, 0xf9, 0x65, 0x32 },
{ 0x19, 0x8c, 0x46, 0xa3, 0xc8, 0xfd, 0x67, 0x33 },
{ 0xe9, 0x74, 0x3a, 0x1d, 0xe7, 0x9a, 0xa4, 0xd2 },
{ 0x69, 0x34, 0x1a, 0x0d, 0xef, 0x9e, 0xa6, 0xd3 },
{ 0xa9, 0x54, 0x2a, 0x95, 0x63, 0x18, 0xa5, 0x52 },
{ 0x29, 0x14, 0x0a, 0x85, 0x6b, 0x1c, 0xa7, 0x53 },
{ 0xc9, 0x64, 0xb2, 0xd9, 0x25, 0xdb, 0x24, 0x92 },
{ 0x49, 0x24, 0x92, 0xc9, 0x2d, 0xdf, 0x26, 0x93 },
{ 0x89, 0x44, 0xa2, 0x51, 0xa1, 0x59, 0x25, 0x12 },
{ 0x09, 0x04, 0x82, 0x41, 0xa9, 0x5d, 0x27, 0x13 },
{ 0xb5, 0xda, 0x6d, 0xb6, 0xee, 0xc2, 0xd4, 0x6a },
{ 0x35, 0x9a, 0x4d, 0xa6, 0xe6, 0xc6, 0xd6, 0x6b },
{ 0xf5, 0xfa, 0x7d, 0x3e, 0x6a, 0x40, 0xd5, 0xea },
{ 0x75, 0xba, 0x5d, 0x2e, 0x62, 0x44, 0xd7, 0xeb },
{ 0x95, 0xca, 0xe5, 0x72, 0x2c, 0x83, 0x54, 0x2a },
{ 0x15, 0x8a, 0xc5, 0x62, 0x24, 0x87, 0x56, 0x2b },
{ 0xd5, 0xea, 0xf5, 0xfa, 0xa8, 0x01, 0x55, 0xaa },
{ 0x55, 0xaa, 0xd5, 0xea, 0xa0, 0x05, 0x57, 0xab },
{ 0xa5, 0x52, 0xa9, 0x54, 0x8f, 0x62, 0x94, 0x4a },
{ 0x25, 0x12, 0x89, 0x44, 0x87, 0x66, 0x96, 0x4b },
{ 0xe5, 0x72, 0xb9, 0xdc, 0x0b, 0xe0, 0x95, 0xca },
{ 0x65, 0x32, 0x99, 0xcc, 0x03, 0xe4, 0x97, 0xcb },
{ 0x85, 0x42, 0x21, 0x90, 0x4d, 0x23, 0x14, 0x0a },
{ 0x05, 0x02, 0x01, 0x80, 0x45, 0x27, 0x16, 0x0b },
{ 0xc5, 0x62, 0x31, 0x18, 0xc9, 0xa1, 0x15, 0x8a },
{ 0x45, 0x22, 0x11, 0x08, 0xc1, 0xa5, 0x17, 0x8b },
{ 0x3d, 0x1e, 0x8f, 0xc7, 0x5e, 0x92, 0xf4, 0x7a },
{ 0xbd, 0x5e, 0xaf, 0xd7, 0x56, 0x96, 0xf6, 0x7b },
{ 0x7d, 0x3e, 0x9f, 0x4f, 0xda, 0x10, 0xf5, 0xfa },
{ 0xfd, 0x7e, 0xbf, 0x5f, 0xd2, 0x14, 0xf7, 0xfb },
{ 0x1d, 0x0e, 0x07, 0x03, 0x9c, 0xd3, 0x74, 0x3a },
{ 0x9d, 0x4e, 0x27, 0x13, 0x94, 0xd7, 0x76, 0x3b },
{ 0x5d, 0x2e, 0x17, 0x8b, 0x18, 0x51, 0x75, 0xba },
{ 0xdd, 0x6e, 0x37, 0x9b, 0x10, 0x55, 0x77, 0xbb },
{ 0x2d, 0x96, 0x4b, 0x25, 0x3f, 0x32, 0xb4, 0x5a },
{ 0xad, 0xd6, 0x6b, 0x35, 0x37, 0x36, 0xb6, 0x5b },
{ 0x6d, 0xb6, 0x5b, 0xad, 0xbb, 0xb0, 0xb5, 0xda },
{ 0xed, 0xf6, 0x7b, 0xbd, 0xb3, 0xb4, 0xb7, 0xdb },
{ 0x0d, 0x86, 0xc3, 0xe1, 0xfd, 0x73, 0x34, 0x1a },
{ 0x8d, 0xc6, 0xe3, 0xf1, 0xf5, 0x77, 0x36, 0x1b },
{ 0x4d, 0xa6, 0xd3, 0x69, 0x79, 0xf1, 0x35, 0x9a },
{ 0xcd, 0xe6, 0xf3, 0x79, 0x71, 0xf5, 0x37, 0x9b },
{ 0x93, 0x49, 0x24, 0x92, 0x5a, 0xbe, 0x4c, 0x26 },
{ 0x13, 0x09, 0x04, 0x82, 0x52, 0xba, 0x4e, 0x27 },
{ 0xd3, 0x69, 0x34, 0x1a, 0xde, 0x3c, 0x4d, 0xa6 },
{ 0x53, 0x29, 0x14, 0x0a, 0xd6, 0x38, 0x4f, 0xa7 },
{ 0xb3, 0x59, 0xac, 0x56, 0x98, 0xff, 0xcc, 0x66 },
{ 0x33, 0x19, 0x8c, 0x46, 0x90, 0xfb, 0xce, 0x67 },
{ 0xf3, 0x79, 0xbc, 0xde, 0x1c, 0x7d, 0xcd, 0xe6 },
{ 0x73, 0x39, 0x9c, 0xce, 0x14, 0x79, 0xcf, 0xe7 },
{ 0x83, 0xc1, 0xe0, 0x70, 0x3b, 0x1e, 0x0c, 0x06 },
{ 0x03, 0x81, 0xc0, 0x60, 0x33, 0x1a, 0x0e, 0x07 },
{ 0xc3, 0xe1, 0xf0, 0xf8, 0xbf, 0x9c, 0x0d, 0x86 },
{ 0x43, 0xa1, 0xd0, 0xe8, 0xb7, 0x98, 0x0f, 0x87 },
{ 0xa3, 0xd1, 0x68, 0xb4, 0xf9, 0x5f, 0x8c, 0x46 },
{ 0x23, 0x91, 0x48, 0xa4, 0xf1, 0x5b, 0x8e, 0x47 },
{ 0xe3, 0xf1, 0x78, 0x3c, 0x7d, 0xdd, 0x8d, 0xc6 },
{ 0x63, 0xb1, 0x58, 0x2c, 0x75, 0xd9, 0x8f, 0xc7 },
{ 0x1b, 0x8d, 0xc6, 0xe3, 0xea, 0xee, 0x6c, 0x36 },
{ 0x9b, 0xcd, 0xe6, 0xf3, 0xe2, 0xea, 0x6e, 0x37 },
{ 0x5b, 0xad, 0xd6, 0x6b, 0x6e, 0x6c, 0x6d, 0xb6 },
{ 0xdb, 0xed, 0xf6, 0x7b, 0x66, 0x68, 0x6f, 0xb7 },
{ 0x3b, 0x9d, 0x4e, 0x27, 0x28, 0xaf, 0xec, 0x76 },
{ 0xbb, 0xdd, 0x6e, 0x37, 0x20, 0xab, 0xee, 0x77 },
{ 0x7b, 0xbd, 0x5e, 0xaf, 0xac, 0x2d, 0xed, 0xf6 },
{ 0xfb, 0xfd, 0x7e, 0xbf, 0xa4, 0x29, 0xef, 0xf7 },
{ 0x0b, 0x05, 0x02, 0x01, 0x8b, 0x4e, 0x2c, 0x16 },
{ 0x8b, 0x45, 0x22, 0x11, 0x83, 0x4a, 0x2e, 0x17 },
{ 0x4b, 0x25, 0x12, 0x89, 0x0f, 0xcc, 0x2d, 0x96 },
{ 0xcb, 0x65, 0x32, 0x99, 0x07, 0xc8, 0x2f, 0x97 },
{ 0x2b, 0x15, 0x8a, 0xc5, 0x49, 0x0f, 0xac, 0x56 },
{ 0xab, 0x55, 0xaa, 0xd5, 0x41, 0x0b, 0xae, 0x57 },
{ 0x6b, 0x35, 0x9a, 0x4d, 0xcd, 0x8d, 0xad, 0xd6 },
{ 0xeb, 0x75, 0xba, 0x5d, 0xc5, 0x89, 0xaf, 0xd7 },
{ 0x57, 0xab, 0x55, 0xaa, 0x82, 0x16, 0x5c, 0xae },
{ 0xd7, 0xeb, 0x75, 0xba, 0x8a, 0x12, 0x5e, 0xaf },
{ 0x17, 0x8b, 0x45, 0x22, 0x06, 0x94, 0x5d, 0x2e },
{ 0x97, 0xcb, 0x65, 0x32, 0x0e, 0x90, 0x5f, 0x2f },
{ 0x77, 0xbb, 0xdd, 0x6e, 0x40, 0x57, 0xdc, 0xee },
{ 0xf7, 0xfb, 0xfd, 0x7e, 0x48, 0x53, 0xde, 0xef },
{ 0x37, 0x9b, 0xcd, 0xe6, 0xc4, 0xd5, 0xdd, 0x6e },
{ 0xb7, 0xdb, 0xed, 0xf6, 0xcc, 0xd1, 0xdf, 0x6f },
{ 0x47, 0x23, 0x91, 0x48, 0xe3, 0xb6, 0x1c, 0x8e },
{ 0xc7, 0x63, 0xb1, 0x58, 0xeb, 0xb2, 0x1e, 0x8f },
{ 0x07, 0x03, 0x81, 0xc0, 0x67, 0x34, 0x1d, 0x0e },
{ 0x87, 0x43, 0xa1, 0xd0, 0x6f, 0x30, 0x1f, 0x0f },
{ 0x67, 0x33, 0x19, 0x8c, 0x21, 0xf7, 0x9c, 0xce },
{ 0xe7, 0x73, 0x39, 0x9c, 0x29, 0xf3, 0x9e, 0xcf },
{ 0x27, 0x13, 0x09, 0x04, 0xa5, 0x75, 0x9d, 0x4e },
{ 0xa7, 0x53, 0x29, 0x14, 0xad, 0x71, 0x9f, 0x4f },
{ 0xdf, 0x6f, 0xb7, 0xdb, 0x32, 0x46, 0x7c, 0xbe },
{ 0x5f, 0x2f, 0x97, 0xcb, 0x3a, 0x42, 0x7e, 0xbf },
{ 0x9f, 0x4f, 0xa7, 0x53, 0xb6, 0xc4, 0x7d, 0x3e },
{ 0x1f, 0x0f, 0x87, 0x43, 0xbe, 0xc0, 0x7f, 0x3f },
{ 0xff, 0x7f, 0x3f, 0x1f, 0xf0, 0x07, 0xfc, 0xfe },
{ 0x7f, 0x3f, 0x1f, 0x0f, 0xf8, 0x03, 0xfe, 0xff },
{ 0xbf, 0x5f, 0x2f, 0x97, 0x74, 0x85, 0xfd, 0x7e },
{ 0x3f, 0x1f, 0x0f, 0x87, 0x7c, 0x81, 0xff, 0x7f },
{ 0xcf, 0xe7, 0x73, 0x39, 0x53, 0xe6, 0x3c, 0x9e },
{ 0x4f, 0xa7, 0x53, 0x29, 0x5b, 0xe2, 0x3e, 0x9f },
{ 0x8f, 0xc7, 0x63, 0xb1, 0xd7, 0x64, 0x3d, 0x1e },
{ 0x0f, 0x87, 0x43, 0xa1, 0xdf, 0x60, 0x3f, 0x1f },
{ 0xef, 0xf7, 0xfb, 0xfd, 0x91, 0xa7, 0xbc, 0xde },
{ 0x6f, 0xb7, 0xdb, 0xed, 0x99, 0xa3, 0xbe, 0xdf },
{ 0xaf, 0xd7, 0xeb, 0x75, 0x15, 0x25, 0xbd, 0x5e },
{ 0x2f, 0x97, 0xcb, 0x65, 0x1d, 0x21, 0xbf, 0x5f },
};
const uint8_t __aligned(256) raid_gfexp_raid[256] =
{
0x01, 0x02, 0x04, 0x08, 0x10, 0x20, 0x40, 0x80,

65
raid/test.c
View file

@ -40,6 +40,39 @@ static void meminc(void *dst, size_t size)
++p[i];
}
/**
* Simulate the bit-level semantics of VGF2P8AFFINEQB with imm8=0.
*/
static uint8_t raid_gfaffine_mul(uint8_t c, uint8_t x)
{
uint8_t v = 0;
int j;
for (j = 0; j < 8; ++j) {
uint8_t row = raid_gfaffine_raid[c][7 - j] & x;
row ^= row >> 4;
row ^= row >> 2;
row ^= row >> 1;
v |= (row & 1U) << j;
}
return v;
}
int raid_test_gfaffine(void)
{
int c, x;
/* all coefficients, including 0, 1, 2, and 0xff, are mandatory */
for (c = 0; c < 256; ++c)
for (x = 0; x < 256; ++x)
if (raid_gfaffine_mul((uint8_t)c, (uint8_t)x) != raid_gfmul_raid[c][x])
return -1;
return 0;
}
int raid_test_combo(void)
{
int r;
@ -582,13 +615,17 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(1) = raid_gen1_avx512bw;
test_setup(2) = raid_gen2_avx512bw;
}
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_avx2gfni_aes;
}
if (raid_cpu_has_avx512gfni()) {
else
test_setup(2) = raid_gen2_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_avx512gfni_aes;
}
else
test_setup(2) = raid_gen2_avx512gfni_raid;
}
#endif
#endif
@ -658,18 +695,30 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(5) = raid_gen5_avx512bw;
test_setup(6) = raid_gen6_avx512bw;
}
if (mode == RAID_MODE_CAUCHY_AES) {
if (raid_cpu_has_avx2gfni()) {
if (raid_cpu_has_avx2gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_avx2gfni_aes;
test_setup(4) = raid_gen4_avx2gfni_aes;
test_setup(5) = raid_gen5_avx2gfni_aes;
test_setup(6) = raid_gen6_avx2gfni_aes;
} else {
test_setup(3) = raid_gen3_avx2gfni_raid;
test_setup(4) = raid_gen4_avx2gfni_raid;
test_setup(5) = raid_gen5_avx2gfni_raid;
test_setup(6) = raid_gen6_avx2gfni_raid;
}
if (raid_cpu_has_avx512gfni()) {
}
if (raid_cpu_has_avx512gfni()) {
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_avx512gfni_aes;
test_setup(4) = raid_gen4_avx512gfni_aes;
test_setup(5) = raid_gen5_avx512gfni_aes;
test_setup(6) = raid_gen6_avx512gfni_aes;
} else {
test_setup(3) = raid_gen3_avx512gfni_raid;
test_setup(4) = raid_gen4_avx512gfni_raid;
test_setup(5) = raid_gen5_avx512gfni_raid;
test_setup(6) = raid_gen6_avx512gfni_raid;
}
}
#endif

7
raid/test.h
View file

@ -31,6 +31,13 @@ int raid_test_sort(void);
*/
int raid_test_combo(void);
/**
* Tests all RAID affine GFNI matrices against scalar GF multiplication.
*
* Returns 0 on success.
*/
int raid_test_gfaffine(void);
/**
* Tests the Galois field and Cauchy matrix properties.
*

7
raid/test/fulltest.c
View file

@ -73,6 +73,13 @@ int main(void)
/* LCOV_EXCL_STOP */
}
printf("Test GFNI affine matrices...\n");
if (raid_test_gfaffine() != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("Test Cauchy RAID parity generation with 1-%u data disks...\n", RAID_DATA_MAX);
for (int i = 1; i <= RAID_DATA_MAX; ++i) {
if (raid_test_par(RAID_MODE_CAUCHY_RAID, i, TEST_SIZE) != 0) {

7
raid/test/selftest.c
View file

@ -16,6 +16,13 @@ int main(void)
raid_init();
printf("Test GFNI affine matrices\n");
if (raid_test_gfaffine() != 0) {
/* LCOV_EXCL_START */
goto bail;
/* LCOV_EXCL_STOP */
}
printf("Test Cauchy RAID\n");
if (raid_test_poly(RAID_MODE_CAUCHY_RAID) != 0) {
/* LCOV_EXCL_START */

64
raid/test/speedtest.c
View file

@ -208,6 +208,10 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
fflush(stdout);
}
if (raid_cpu_has_avx2gfni())
printf("%8s", "");
if (raid_cpu_has_avx512gfni())
printf("%8s", "");
#endif
#endif
printf("\n");
@ -351,7 +355,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen2_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -363,7 +371,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen2_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -470,7 +482,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen3_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -482,7 +498,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen3_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -589,7 +609,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen4_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -601,7 +625,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen4_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -706,7 +734,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen5_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -718,7 +750,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen5_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -823,7 +859,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen6_avx2gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}
@ -835,7 +875,11 @@ void speed_gen(int nd, void **v, int size, int delta, int period, const char *ms
printf("%8" PRIu64, ds / dt);
} else {
printf("%8s", "");
SPEED_START {
raid_gen6_avx512gfni_raid(nd, size, v);
} SPEED_STOP
printf("%8" PRIu64, ds / dt);
}
fflush(stdout);
}