mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-25 18:40:19 -04:00
Add support for AVX2 GFNI
This commit is contained in:
Andrea Mazzoleni
parent
8ae654d8a6
commit
f9a1370b42
9 changed files with 1074 additions and 28 deletions
142
cmdline/speed.c
142
cmdline/speed.c
|
|
@ -286,9 +286,16 @@ void speed(int period, int nd, int size)
|
|||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx2e");
|
||||
#ifdef CONFIG_AVX512BW
|
||||
printf("%8s", "avx512");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
printf("%8s", "gfni512");
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -427,6 +434,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen2_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -572,6 +589,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen3_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -657,6 +684,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen4_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -742,6 +779,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen5_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -827,6 +874,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen6_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -851,9 +908,16 @@ void speed(int period, int nd, int size)
|
|||
printf("%8s", "ssse3");
|
||||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
printf("%8s", "avx512");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
printf("%8s", "gfni512");
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -911,6 +975,19 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -985,6 +1062,19 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx2gfni);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1060,12 +1150,23 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(3, raid_gen3_avx2gfni);
|
||||
raid_recX_avx2gfni(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(3, raid_gen3_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1136,12 +1237,23 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(4, raid_gen4_avx2gfni);
|
||||
raid_recX_avx2gfni(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(4, raid_gen4_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1211,12 +1323,23 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(5, raid_gen5_avx2gfni);
|
||||
raid_recX_avx2gfni(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(5, raid_gen5_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
@ -1286,12 +1409,23 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(6, raid_gen6_avx2gfni);
|
||||
raid_recX_avx2gfni(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(6, raid_gen6_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
|
|
|
|||
16
configure.ac
16
configure.ac
|
|
@ -284,6 +284,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
|||
[AC_DEFINE([HAVE_AVX2], [1], [Define to 1 if avx2 is supported by the assembler.]) asmavx2=yes])
|
||||
AC_MSG_RESULT([$asmavx2])
|
||||
|
||||
dnl Checks for AS supporting the AVX2FFNI instructions.
|
||||
AC_MSG_CHECKING([for avx2gfni])
|
||||
asmavx2gfni=no
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
#if defined(__x86_64__)
|
||||
void f(void* ptr)
|
||||
{
|
||||
asm volatile ("vgf2p8mulb %0, %%ymm2, %%ymm2" : : "m" (ptr));
|
||||
}
|
||||
#else
|
||||
#error not x86
|
||||
#endif
|
||||
]])],
|
||||
[AC_DEFINE([HAVE_AVX2GFNI], [1], [Define to 1 if avx2gfni is supported by the assembler.]) asmavx2gfni=yes])
|
||||
AC_MSG_RESULT([$asmavx2gfni])
|
||||
|
||||
dnl Checks for AS supporting the AVX512BW instructions.
|
||||
AC_MSG_CHECKING([for avx512bw])
|
||||
asmavx512bw=no
|
||||
|
|
|
|||
700
raid/gfni.c
700
raid/gfni.c
|
|
@ -5,6 +5,50 @@
|
|||
#include "gf.h"
|
||||
#include "cpu.h"
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx2gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
uint8_t gf2 = 0x02;
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gf2));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0, %%ymm0" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0, %%ymm1" : : "m" (v[l][i + 32]));
|
||||
asm volatile ("vmovdqa %ymm0, %ymm2");
|
||||
asm volatile ("vmovdqa %ymm1, %ymm3");
|
||||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
asm volatile ("vgf2p8mulb %ymm15, %ymm2, %ymm2");
|
||||
asm volatile ("vgf2p8mulb %ymm15, %ymm3, %ymm3");
|
||||
asm volatile ("vmovdqa %0, %%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0, %%ymm5" : : "m" (v[d][i + 32]));
|
||||
asm volatile ("vpxor %ymm4, %ymm0, %ymm0");
|
||||
asm volatile ("vpxor %ymm5, %ymm1, %ymm1");
|
||||
asm volatile ("vpxor %ymm4, %ymm2, %ymm2");
|
||||
asm volatile ("vpxor %ymm5, %ymm3, %ymm3");
|
||||
}
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0, %0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%ymm1, %0" : "=m" (p[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2, %0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%ymm3, %0" : "=m" (q[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) GFNI implementation
|
||||
|
|
@ -41,6 +85,87 @@ void raid_gen2_avx512gfni(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx2gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 3; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[l][i + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm9" : : "m" (gfcauchy[2][l]));
|
||||
|
||||
asm volatile ("vmovdqa %ymm6,%ymm0");
|
||||
asm volatile ("vmovdqa %ymm7,%ymm1");
|
||||
asm volatile ("vgf2p8mulb %ymm6,%ymm8,%ymm2");
|
||||
asm volatile ("vgf2p8mulb %ymm7,%ymm8,%ymm3");
|
||||
asm volatile ("vgf2p8mulb %ymm6,%ymm9,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm7,%ymm9,%ymm5");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d][i + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm9" : : "m" (gfcauchy[2][d]));
|
||||
|
||||
asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm7,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm6,%ymm8,%ymm10");
|
||||
asm volatile ("vgf2p8mulb %ymm7,%ymm8,%ymm11");
|
||||
asm volatile ("vpxor %ymm10,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm11,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm6,%ymm9,%ymm10");
|
||||
asm volatile ("vgf2p8mulb %ymm7,%ymm9,%ymm11");
|
||||
asm volatile ("vpxor %ymm10,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm11,%ymm5,%ymm5");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm6,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm7,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm7,%ymm3,%ymm3");
|
||||
asm volatile ("vpxor %ymm6,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm7,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) GFNI implementation
|
||||
|
|
@ -104,6 +229,101 @@ void raid_gen3_avx512gfni(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx2gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
uint8_t *s = v[nd + 3];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 4; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[l][i + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfcauchy[2][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][l]));
|
||||
|
||||
asm volatile ("vmovdqa %ymm8,%ymm0");
|
||||
asm volatile ("vmovdqa %ymm9,%ymm1");
|
||||
asm volatile ("vgf2p8mulb %ymm8,%ymm10,%ymm2");
|
||||
asm volatile ("vgf2p8mulb %ymm9,%ymm10,%ymm3");
|
||||
asm volatile ("vgf2p8mulb %ymm8,%ymm11,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm9,%ymm11,%ymm5");
|
||||
asm volatile ("vgf2p8mulb %ymm8,%ymm12,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm9,%ymm12,%ymm7");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[d][i + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfcauchy[2][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][d]));
|
||||
|
||||
asm volatile ("vpxor %ymm8,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm9,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm8,%ymm10,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm9,%ymm10,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm14,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm8,%ymm11,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm9,%ymm11,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm14,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm8,%ymm12,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm9,%ymm12,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm14,%ymm7,%ymm7");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm8,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm9,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm8,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm9,%ymm3,%ymm3");
|
||||
asm volatile ("vpxor %ymm8,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm9,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm8,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm9,%ymm7,%ymm7");
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i]));
|
||||
asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
|
||||
|
|
@ -175,6 +395,117 @@ void raid_gen4_avx512gfni(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx2gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
uint8_t *s = v[nd + 3];
|
||||
uint8_t *t = v[nd + 4];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 5; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[l][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %ymm10,%ymm0");
|
||||
asm volatile ("vmovdqa %ymm11,%ymm1");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm2");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm3");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[2][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm5");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm7");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[4][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm8");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm9");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[d][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm14,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[2][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm14,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm14,%ymm7,%ymm7");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[4][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13");
|
||||
asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14");
|
||||
asm volatile ("vpxor %ymm13,%ymm8,%ymm8");
|
||||
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm10,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm11,%ymm3,%ymm3");
|
||||
asm volatile ("vpxor %ymm10,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm11,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm10,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm11,%ymm7,%ymm7");
|
||||
asm volatile ("vpxor %ymm10,%ymm8,%ymm8");
|
||||
asm volatile ("vpxor %ymm11,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i]));
|
||||
asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm8,%0" : "=m" (t[i]));
|
||||
asm volatile ("vmovntdq %%ymm9,%0" : "=m" (t[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
|
||||
|
|
@ -254,6 +585,132 @@ void raid_gen5_avx512gfni(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx2gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
uint8_t *s = v[nd + 3];
|
||||
uint8_t *t = v[nd + 4];
|
||||
uint8_t *u = v[nd + 5];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 6; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[l][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %ymm12,%ymm0");
|
||||
asm volatile ("vmovdqa %ymm13,%ymm1");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm2");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm3");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm5");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm7");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm8");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm9");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][l]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm10");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm11");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm2,%ymm2");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm4,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm6,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm7,%ymm7");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm8,%ymm8");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm9,%ymm9");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][d]));
|
||||
asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm10,%ymm10");
|
||||
asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15");
|
||||
asm volatile ("vpxor %ymm15,%ymm11,%ymm11");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm12,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm13,%ymm1,%ymm1");
|
||||
asm volatile ("vpxor %ymm12,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm13,%ymm3,%ymm3");
|
||||
asm volatile ("vpxor %ymm12,%ymm4,%ymm4");
|
||||
asm volatile ("vpxor %ymm13,%ymm5,%ymm5");
|
||||
asm volatile ("vpxor %ymm12,%ymm6,%ymm6");
|
||||
asm volatile ("vpxor %ymm13,%ymm7,%ymm7");
|
||||
asm volatile ("vpxor %ymm12,%ymm8,%ymm8");
|
||||
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
|
||||
asm volatile ("vpxor %ymm12,%ymm10,%ymm10");
|
||||
asm volatile ("vpxor %ymm13,%ymm11,%ymm11");
|
||||
|
||||
asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i]));
|
||||
asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm8,%0" : "=m" (t[i]));
|
||||
asm volatile ("vmovntdq %%ymm9,%0" : "=m" (t[i + 32]));
|
||||
asm volatile ("vmovntdq %%ymm10,%0" : "=m" (u[i]));
|
||||
asm volatile ("vmovntdq %%ymm11,%0" : "=m" (u[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
|
||||
|
|
@ -341,6 +798,57 @@ void raid_gen6_avx512gfni(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* RAID recovering for one disk AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
uint8_t G, V;
|
||||
size_t i;
|
||||
|
||||
(void)nr;
|
||||
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
G = A(ip[0], id[0]);
|
||||
V = inv(G);
|
||||
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm4" : : "m" (V));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vgf2p8mulb %ymm4,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* RAID recovering for one disk GFNI implementation
|
||||
|
|
@ -383,6 +891,85 @@ void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* RAID recovering for two disks AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[N], *pa[N];
|
||||
uint8_t G[N * N], V[N * N];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm6,%ymm2,%ymm2");
|
||||
asm volatile ("vpxor %ymm7,%ymm3,%ymm3");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[0]));
|
||||
asm volatile ("vgf2p8mulb %ymm0,%ymm8,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm1,%ymm8,%ymm5");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[1]));
|
||||
asm volatile ("vgf2p8mulb %ymm2,%ymm8,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm4,%ymm4");
|
||||
asm volatile ("vgf2p8mulb %ymm3,%ymm8,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm5,%ymm5");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[2]));
|
||||
asm volatile ("vgf2p8mulb %ymm0,%ymm8,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm1,%ymm8,%ymm7");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[3]));
|
||||
asm volatile ("vgf2p8mulb %ymm2,%ymm8,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm6,%ymm6");
|
||||
asm volatile ("vgf2p8mulb %ymm3,%ymm8,%ymm9");
|
||||
asm volatile ("vpxor %ymm9,%ymm7,%ymm7");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i]));
|
||||
asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* RAID recovering for two disks GFNI implementation
|
||||
|
|
@ -422,9 +1009,6 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
/*
|
||||
* pa[0] = V[0]*delta[0] ^ V[1]*delta[1]
|
||||
*/
|
||||
asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[0]));
|
||||
asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm6,%zmm6");
|
||||
|
|
@ -437,7 +1021,6 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
/* pa[1] = V[2]*delta[0] ^ V[3]*delta[1] */
|
||||
asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[2]));
|
||||
asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm6,%zmm6");
|
||||
|
|
@ -453,6 +1036,78 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
/*
|
||||
* RAID recovering AVX2 GFNI implementation
|
||||
*/
|
||||
void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t buffer[RAID_PARITY_MAX*64+64];
|
||||
uint8_t *pd = __align_ptr(buffer, 64);
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* delta */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32]));
|
||||
|
||||
asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32]));
|
||||
|
||||
asm volatile ("vpxor %ymm2,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32]));
|
||||
}
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vpxor %ymm0,%ymm0,%ymm0");
|
||||
asm volatile ("vpxor %ymm1,%ymm1,%ymm1");
|
||||
for (k = 0; k < N; ++k) {
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64]));
|
||||
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%ymm2" : : "m" (V[j * N + k]));
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm4,%ymm2,%ymm3");
|
||||
asm volatile ("vpxor %ymm3,%ymm0,%ymm0");
|
||||
|
||||
asm volatile ("vgf2p8mulb %ymm5,%ymm2,%ymm3");
|
||||
asm volatile ("vpxor %ymm3,%ymm1,%ymm1");
|
||||
}
|
||||
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i]));
|
||||
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* RAID recovering GFNI implementation
|
||||
|
|
@ -490,13 +1145,14 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i]));
|
||||
asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64]));
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64]));
|
||||
}
|
||||
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vpxorq %zmm0,%zmm0,%zmm0");
|
||||
for (k = 0; k < N; ++k) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[j * N + k]));
|
||||
asm volatile ("vgf2p8mulb %zmm4,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
|
|
@ -509,10 +1165,35 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
|
|||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
void raid_register_avx2gfni(void)
|
||||
{
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni);
|
||||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
void raid_register_avx512gfni(void)
|
||||
{
|
||||
#if defined(USE_RAID_AES)
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
if (!raid_cpu_has_slow_avx512()) {
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni);
|
||||
|
|
@ -530,5 +1211,6 @@ void raid_register_avx512gfni(void)
|
|||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
|
|
|||
|
|
@ -48,6 +48,9 @@
|
|||
#if HAVE_AVX512GFNI
|
||||
#define CONFIG_AVX512GFNI 1
|
||||
#endif
|
||||
#if HAVE_AVX2GFNI
|
||||
#define CONFIG_AVX2GFNI 1
|
||||
#endif
|
||||
|
||||
#else /* if HAVE_CONFIG_H is not defined */
|
||||
|
||||
|
|
@ -71,6 +74,7 @@
|
|||
#ifdef CONFIG_X86_64
|
||||
#define CONFIG_AVX512BW 1
|
||||
#define CONFIG_AVX512GFNI 1
|
||||
#define CONFIG_AVX2GFNI 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
|
@ -140,6 +144,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv);
|
|||
void raid_gen2_avx2(int nd, size_t size, void **vv);
|
||||
void raid_gen2_sse2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx2gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_genz_int32(int nd, size_t size, void **vv);
|
||||
void raid_genz_int64(int nd, size_t size, void **vv);
|
||||
|
|
@ -151,24 +156,28 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv);
|
|||
void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen4_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen4_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx2gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen5_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen5_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx2gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen6_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen6_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx2gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
|
|
@ -182,6 +191,9 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
|||
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
|
|
@ -288,6 +300,7 @@ void raid_register_x86(void);
|
|||
/**
|
||||
* Register all the functions based on GFNI intructions.
|
||||
*/
|
||||
void raid_register_avx2gfni(void);
|
||||
void raid_register_avx512gfni(void);
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -247,6 +247,9 @@ void raid_init(void)
|
|||
#if defined(CONFIG_X86) && defined(CONFIG_SSE2)
|
||||
raid_register_x86();
|
||||
#endif
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI)
|
||||
raid_register_avx2gfni();
|
||||
#endif
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
raid_register_avx512gfni();
|
||||
#endif
|
||||
|
|
|
|||
64
raid/test.c
64
raid/test.c
|
|
@ -134,7 +134,7 @@ int raid_test_sort(void)
|
|||
|
||||
int raid_test_rec(int mode, int nd, size_t size)
|
||||
{
|
||||
void (*f[RAID_PARITY_MAX][4])(
|
||||
void (*f[RAID_PARITY_MAX][128])(
|
||||
int nr, int *id, int *ip, int nd, size_t size, void **vbuf);
|
||||
void *v_alloc;
|
||||
void **v;
|
||||
|
|
@ -203,6 +203,16 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
if (raid_cpu_has_avx512bw())
|
||||
f[i][nf[i]++] = raid_rec1_avx512bw;
|
||||
#endif
|
||||
#endif
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
f[i][nf[i]++] = raid_rec1_avx2gfni;
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
f[i][nf[i]++] = raid_rec1_avx512gfni;
|
||||
#endif
|
||||
#endif
|
||||
} else if (i == 1) {
|
||||
f[i][nf[i]++] = raid_rec2_int8;
|
||||
|
|
@ -221,6 +231,16 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
if (raid_cpu_has_avx512bw())
|
||||
f[i][nf[i]++] = raid_rec2_avx512bw;
|
||||
#endif
|
||||
#endif
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
f[i][nf[i]++] = raid_rec2_avx2gfni;
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
f[i][nf[i]++] = raid_rec2_avx512gfni;
|
||||
#endif
|
||||
#endif
|
||||
} else {
|
||||
f[i][nf[i]++] = raid_recX_int8;
|
||||
|
|
@ -239,6 +259,16 @@ int raid_test_rec(int mode, int nd, size_t size)
|
|||
if (raid_cpu_has_avx512bw())
|
||||
f[i][nf[i]++] = raid_recX_avx512bw;
|
||||
#endif
|
||||
#endif
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
f[i][nf[i]++] = raid_recX_avx2gfni;
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
f[i][nf[i]++] = raid_recX_avx512gfni;
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
}
|
||||
|
|
@ -307,7 +337,7 @@ bail:
|
|||
|
||||
int raid_test_par(int mode, int nd, size_t size)
|
||||
{
|
||||
void (*f[64])(int nd, size_t size, void **vbuf);
|
||||
void (*f[128])(int nd, size_t size, void **vbuf);
|
||||
void *v_alloc;
|
||||
void **v;
|
||||
int nv;
|
||||
|
|
@ -381,6 +411,18 @@ int raid_test_par(int mode, int nd, size_t size)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
f[nf++] = raid_gen2_avx2gfni;
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
f[nf++] = raid_gen2_avx512gfni;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif /* CONFIG_X86 */
|
||||
|
||||
if (mode == RAID_MODE_CAUCHY) {
|
||||
|
|
@ -426,6 +468,24 @@ int raid_test_par(int mode, int nd, size_t size)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef USE_RAID_AES
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
f[nf++] = raid_gen3_avx2gfni;
|
||||
f[nf++] = raid_gen4_avx2gfni;
|
||||
f[nf++] = raid_gen5_avx2gfni;
|
||||
f[nf++] = raid_gen6_avx2gfni;
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
f[nf++] = raid_gen3_avx512gfni;
|
||||
f[nf++] = raid_gen4_avx512gfni;
|
||||
f[nf++] = raid_gen5_avx512gfni;
|
||||
f[nf++] = raid_gen6_avx512gfni;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
|
||||
#endif /* CONFIG_X86 */
|
||||
} else {
|
||||
|
|
|
|||
|
|
@ -101,7 +101,6 @@ sdecovtest:
|
|||
# Use this target to run a complete test on Ice Lake simulator
|
||||
sde:
|
||||
$(SDE) -icl -- ./selftest
|
||||
$(SDE) -clx -- ./speedtest
|
||||
$(SDE) -icl -- ./speedtest
|
||||
$(SDE) -icl -- ./fulltest
|
||||
|
||||
|
|
|
|||
|
|
@ -39,6 +39,12 @@ int main(void)
|
|||
printf("Including x86 AVX2\n");
|
||||
if (raid_cpu_has_avx512bw())
|
||||
printf("Including x86 AVX512BW\n");
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
#ifdef USE_RAID_AES
|
||||
printf("Including x86 AVX2GFNI\n");
|
||||
#else
|
||||
printf("Excluding x86 AVX2GFNI due polynomial mismatch\n");
|
||||
#endif
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
#ifdef USE_RAID_AES
|
||||
printf("Including x86 AVX512GFNI\n");
|
||||
|
|
@ -73,18 +79,13 @@ int main(void)
|
|||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("Test Cauchy parity generation with %u data disks...\n", RAID_DATA_MAX);
|
||||
if (raid_test_par(RAID_MODE_CAUCHY, RAID_DATA_MAX, TEST_SIZE) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
|
||||
printf("Test Cauchy parity generation with 1 data disk...\n");
|
||||
if (raid_test_par(RAID_MODE_CAUCHY, 1, TEST_SIZE) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
printf("Test Cauchy parity generation with 1-%u data disks...\n", RAID_DATA_MAX);
|
||||
for (int i = 1; i <= RAID_DATA_MAX; ++i) {
|
||||
if (raid_test_par(RAID_MODE_CAUCHY, i, TEST_SIZE) != 0) {
|
||||
/* LCOV_EXCL_START */
|
||||
goto bail;
|
||||
/* LCOV_EXCL_STOP */
|
||||
}
|
||||
}
|
||||
|
||||
printf("Test Cauchy recovering with all combinations of %u data and 6 parity blocks...\n", TEST_COUNT);
|
||||
|
|
|
|||
|
|
@ -137,9 +137,16 @@ void speed(void)
|
|||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx2e");
|
||||
#ifdef CONFIG_AVX512BW
|
||||
printf("%8s", "avx512");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
printf("%8s", "gfni512");
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -278,6 +285,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen2_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -423,6 +440,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen3_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -508,6 +535,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen4_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -593,6 +630,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen5_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -678,6 +725,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen6_avx2gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -702,9 +759,16 @@ void speed(void)
|
|||
printf("%8s", "ssse3");
|
||||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
printf("%8s", "avx512");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
printf("%8s", "gfni512");
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -759,6 +823,18 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx2);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -828,6 +904,18 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx2gfni);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -893,6 +981,17 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx2gfni);
|
||||
raid_recX_avx2gfni(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -961,6 +1060,17 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx2gfni);
|
||||
raid_recX_avx2gfni(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1029,6 +1139,17 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx2gfni);
|
||||
raid_recX_avx2gfni(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1097,6 +1218,17 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX2GFNI
|
||||
if (raid_cpu_has_avx2gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx2gfni);
|
||||
raid_recX_avx2gfni(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1130,6 +1262,12 @@ int main(void)
|
|||
printf("Including x86 AVX2\n");
|
||||
if (raid_cpu_has_avx512bw())
|
||||
printf("Including x86 AVX512BW\n");
|
||||
if (raid_cpu_has_avx2gfni())
|
||||
#ifdef USE_RAID_AES
|
||||
printf("Including x86 AVX2GFNI\n");
|
||||
#else
|
||||
printf("Including x86 AVX2GFNI (Performance reference only. Polynomial mismatch prevents use)\n");
|
||||
#endif
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
#ifdef USE_RAID_AES
|
||||
printf("Including x86 AVX512GFNI\n");
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue