diff --git a/cmdline/speed.c b/cmdline/speed.c index 2ec15222..49b72852 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -286,9 +286,16 @@ void speed(int period, int nd, int size) printf("%8s", "avx2"); #ifdef CONFIG_X86_64 printf("%8s", "avx2e"); +#ifdef CONFIG_AVX512BW printf("%8s", "avx512"); +#endif +#ifdef CONFIG_AVX2GFNI + printf("%8s", "gfni"); +#endif +#ifdef CONFIG_AVX512GFNI printf("%8s", "gfni512"); #endif +#endif #endif printf("\n"); @@ -427,6 +434,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen2_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -572,6 +589,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen3_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -657,6 +684,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen4_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -742,6 +779,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen5_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -827,6 +874,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen6_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -851,9 +908,16 @@ void speed(int period, int nd, int size) printf("%8s", "ssse3"); printf("%8s", "avx2"); #ifdef CONFIG_X86_64 +#ifdef CONFIG_AVX512BW printf("%8s", "avx512"); +#endif +#ifdef CONFIG_AVX2GFNI + printf("%8s", "gfni"); +#endif +#ifdef CONFIG_AVX512GFNI printf("%8s", "gfni512"); #endif +#endif #endif printf("\n"); @@ -911,6 +975,19 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(1, raid_gen1_avx2); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -985,6 +1062,19 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(2, raid_gen2_avx2gfni); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -1060,12 +1150,23 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(3, raid_gen3_avx2gfni); + raid_recX_avx2gfni(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ raid_gen_force(3, raid_gen3_avx512gfni); - /* +1 to avoid GEN1 optimized case */ raid_recX_avx512gfni(3, id, ip, nd, size, v); } SPEED_STOP @@ -1136,12 +1237,23 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(4, raid_gen4_avx2gfni); + raid_recX_avx2gfni(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ raid_gen_force(4, raid_gen4_avx512gfni); - /* +1 to avoid GEN1 optimized case */ raid_recX_avx512gfni(4, id, ip, nd, size, v); } SPEED_STOP @@ -1211,12 +1323,23 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(5, raid_gen5_avx2gfni); + raid_recX_avx2gfni(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ raid_gen_force(5, raid_gen5_avx512gfni); - /* +1 to avoid GEN1 optimized case */ raid_recX_avx512gfni(5, id, ip, nd, size, v); } SPEED_STOP @@ -1286,12 +1409,23 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(6, raid_gen6_avx2gfni); + raid_recX_avx2gfni(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { /* ensure to use same hardware in the delta step */ raid_gen_force(6, raid_gen6_avx512gfni); - /* +1 to avoid GEN1 optimized case */ raid_recX_avx512gfni(6, id, ip, nd, size, v); } SPEED_STOP diff --git a/configure.ac b/configure.ac index 946f9cee..6efd4aa3 100644 --- a/configure.ac +++ b/configure.ac @@ -284,6 +284,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [AC_DEFINE([HAVE_AVX2], [1], [Define to 1 if avx2 is supported by the assembler.]) asmavx2=yes]) AC_MSG_RESULT([$asmavx2]) +dnl Checks for AS supporting the AVX2FFNI instructions. +AC_MSG_CHECKING([for avx2gfni]) +asmavx2gfni=no +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#if defined(__x86_64__) + void f(void* ptr) + { + asm volatile ("vgf2p8mulb %0, %%ymm2, %%ymm2" : : "m" (ptr)); + } +#else +#error not x86 +#endif +]])], +[AC_DEFINE([HAVE_AVX2GFNI], [1], [Define to 1 if avx2gfni is supported by the assembler.]) asmavx2gfni=yes]) +AC_MSG_RESULT([$asmavx2gfni]) + dnl Checks for AS supporting the AVX512BW instructions. AC_MSG_CHECKING([for avx512bw]) asmavx512bw=no diff --git a/raid/gfni.c b/raid/gfni.c index c6e951ce..30d1f9e1 100644 --- a/raid/gfni.c +++ b/raid/gfni.c @@ -5,6 +5,50 @@ #include "gf.h" #include "cpu.h" +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * GEN2 (RAID6 with powers of 2) AVX2 GFNI implementation + */ +void raid_gen2_avx2gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + int d, l = nd - 1; + size_t i; + uint8_t gf2 = 0x02; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gf2)); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0, %%ymm0" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0, %%ymm1" : : "m" (v[l][i + 32])); + asm volatile ("vmovdqa %ymm0, %ymm2"); + asm volatile ("vmovdqa %ymm1, %ymm3"); + + for (d = l - 1; d >= 0; --d) { + asm volatile ("vgf2p8mulb %ymm15, %ymm2, %ymm2"); + asm volatile ("vgf2p8mulb %ymm15, %ymm3, %ymm3"); + asm volatile ("vmovdqa %0, %%ymm4" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0, %%ymm5" : : "m" (v[d][i + 32])); + asm volatile ("vpxor %ymm4, %ymm0, %ymm0"); + asm volatile ("vpxor %ymm5, %ymm1, %ymm1"); + asm volatile ("vpxor %ymm4, %ymm2, %ymm2"); + asm volatile ("vpxor %ymm5, %ymm3, %ymm3"); + } + + asm volatile ("vmovntdq %%ymm0, %0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1, %0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2, %0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3, %0" : "=m" (q[i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * GEN2 (RAID6 with powers of 2) GFNI implementation @@ -41,6 +85,87 @@ void raid_gen2_avx512gfni(int nd, size_t size, void **vv) } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * GEN3 (triple parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen3_avx2gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[l][i + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (gfcauchy[1][l])); + asm volatile ("vpbroadcastb %0,%%ymm9" : : "m" (gfcauchy[2][l])); + + asm volatile ("vmovdqa %ymm6,%ymm0"); + asm volatile ("vmovdqa %ymm7,%ymm1"); + asm volatile ("vgf2p8mulb %ymm6,%ymm8,%ymm2"); + asm volatile ("vgf2p8mulb %ymm7,%ymm8,%ymm3"); + asm volatile ("vgf2p8mulb %ymm6,%ymm9,%ymm4"); + asm volatile ("vgf2p8mulb %ymm7,%ymm9,%ymm5"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[d][i + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (gfcauchy[1][d])); + asm volatile ("vpbroadcastb %0,%%ymm9" : : "m" (gfcauchy[2][d])); + + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm7,%ymm1,%ymm1"); + + asm volatile ("vgf2p8mulb %ymm6,%ymm8,%ymm10"); + asm volatile ("vgf2p8mulb %ymm7,%ymm8,%ymm11"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + + asm volatile ("vgf2p8mulb %ymm6,%ymm9,%ymm10"); + asm volatile ("vgf2p8mulb %ymm7,%ymm9,%ymm11"); + asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm11,%ymm5,%ymm5"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm6,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm7,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm6,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm7,%ymm5,%ymm5"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * GEN3 (triple parity with Cauchy matrix) GFNI implementation @@ -104,6 +229,101 @@ void raid_gen3_avx512gfni(int nd, size_t size, void **vv) } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * GEN4 (quad parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen4_avx2gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + uint8_t *s = v[nd + 3]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[l][i + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (gfcauchy[1][l])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfcauchy[2][l])); + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][l])); + + asm volatile ("vmovdqa %ymm8,%ymm0"); + asm volatile ("vmovdqa %ymm9,%ymm1"); + asm volatile ("vgf2p8mulb %ymm8,%ymm10,%ymm2"); + asm volatile ("vgf2p8mulb %ymm9,%ymm10,%ymm3"); + asm volatile ("vgf2p8mulb %ymm8,%ymm11,%ymm4"); + asm volatile ("vgf2p8mulb %ymm9,%ymm11,%ymm5"); + asm volatile ("vgf2p8mulb %ymm8,%ymm12,%ymm6"); + asm volatile ("vgf2p8mulb %ymm9,%ymm12,%ymm7"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[d][i + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm10" : : "m" (gfcauchy[1][d])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfcauchy[2][d])); + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][d])); + + asm volatile ("vpxor %ymm8,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm9,%ymm1,%ymm1"); + + asm volatile ("vgf2p8mulb %ymm8,%ymm10,%ymm13"); + asm volatile ("vgf2p8mulb %ymm9,%ymm10,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm3,%ymm3"); + + asm volatile ("vgf2p8mulb %ymm8,%ymm11,%ymm13"); + asm volatile ("vgf2p8mulb %ymm9,%ymm11,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm14,%ymm5,%ymm5"); + + asm volatile ("vgf2p8mulb %ymm8,%ymm12,%ymm13"); + asm volatile ("vgf2p8mulb %ymm9,%ymm12,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm14,%ymm7,%ymm7"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm8" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm9" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm8,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm9,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm8,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm9,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm8,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm8,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * GEN4 (quad parity with Cauchy matrix) GFNI implementation @@ -175,6 +395,117 @@ void raid_gen4_avx512gfni(int nd, size_t size, void **vv) } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * GEN5 (penta parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen5_avx2gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + uint8_t *s = v[nd + 3]; + uint8_t *t = v[nd + 4]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 5; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm10,%ymm0"); + asm volatile ("vmovdqa %ymm11,%ymm1"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[1][l])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm2"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm3"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[2][l])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm4"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm5"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][l])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm6"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm7"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[4][l])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm8"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm9"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[1][d])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm14,%ymm3,%ymm3"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[2][d])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm14,%ymm5,%ymm5"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[3][d])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm14,%ymm7,%ymm7"); + + asm volatile ("vpbroadcastb %0,%%ymm12" : : "m" (gfcauchy[4][d])); + asm volatile ("vgf2p8mulb %ymm10,%ymm12,%ymm13"); + asm volatile ("vgf2p8mulb %ymm11,%ymm12,%ymm14"); + asm volatile ("vpxor %ymm13,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm14,%ymm9,%ymm9"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm11" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm10,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm11,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm10,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm11,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm10,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm11,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm10,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm11,%ymm7,%ymm7"); + asm volatile ("vpxor %ymm10,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm11,%ymm9,%ymm9"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (t[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (t[i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * GEN5 (penta parity with Cauchy matrix) GFNI implementation @@ -254,6 +585,132 @@ void raid_gen5_avx512gfni(int nd, size_t size, void **vv) } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * GEN6 (hexa parity with Cauchy matrix) AVX2 GFNI implementation + */ +void raid_gen6_avx2gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + uint8_t *s = v[nd + 3]; + uint8_t *t = v[nd + 4]; + uint8_t *u = v[nd + 5]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 6; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[l][i + 32])); + + asm volatile ("vmovdqa %ymm12,%ymm0"); + asm volatile ("vmovdqa %ymm13,%ymm1"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[1][l])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm2"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm3"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][l])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm4"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm5"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][l])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm6"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm7"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][l])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm8"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm9"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][l])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm10"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm11"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[d][i + 32])); + + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[1][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm2,%ymm2"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm3,%ymm3"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[2][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm4,%ymm4"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm5,%ymm5"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[3][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm6,%ymm6"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm7,%ymm7"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[4][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm8,%ymm8"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm9,%ymm9"); + + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfcauchy[5][d])); + asm volatile ("vgf2p8mulb %ymm12,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm10,%ymm10"); + asm volatile ("vgf2p8mulb %ymm13,%ymm14,%ymm15"); + asm volatile ("vpxor %ymm15,%ymm11,%ymm11"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i])); + asm volatile ("vmovdqa %0,%%ymm13" : : "m" (v[0][i + 32])); + + asm volatile ("vpxor %ymm12,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm13,%ymm1,%ymm1"); + asm volatile ("vpxor %ymm12,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm13,%ymm3,%ymm3"); + asm volatile ("vpxor %ymm12,%ymm4,%ymm4"); + asm volatile ("vpxor %ymm13,%ymm5,%ymm5"); + asm volatile ("vpxor %ymm12,%ymm6,%ymm6"); + asm volatile ("vpxor %ymm13,%ymm7,%ymm7"); + asm volatile ("vpxor %ymm12,%ymm8,%ymm8"); + asm volatile ("vpxor %ymm13,%ymm9,%ymm9"); + asm volatile ("vpxor %ymm12,%ymm10,%ymm10"); + asm volatile ("vpxor %ymm13,%ymm11,%ymm11"); + + asm volatile ("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%ymm1,%0" : "=m" (p[i + 32])); + asm volatile ("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%ymm3,%0" : "=m" (q[i + 32])); + asm volatile ("vmovntdq %%ymm4,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%ymm5,%0" : "=m" (r[i + 32])); + asm volatile ("vmovntdq %%ymm6,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%ymm7,%0" : "=m" (s[i + 32])); + asm volatile ("vmovntdq %%ymm8,%0" : "=m" (t[i])); + asm volatile ("vmovntdq %%ymm9,%0" : "=m" (t[i + 32])); + asm volatile ("vmovntdq %%ymm10,%0" : "=m" (u[i])); + asm volatile ("vmovntdq %%ymm11,%0" : "=m" (u[i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * GEN6 (hexa parity with Cauchy matrix) GFNI implementation @@ -341,6 +798,57 @@ void raid_gen6_avx512gfni(int nd, size_t size, void **vv) } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * RAID recovering for one disk AVX2 GFNI implementation + */ +void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%ymm4" : : "m" (V)); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vgf2p8mulb %ymm4,%ymm0,%ymm0"); + asm volatile ("vgf2p8mulb %ymm4,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * RAID recovering for one disk GFNI implementation @@ -383,6 +891,85 @@ void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * RAID recovering for two disks AVX2 GFNI implementation + */ +void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[0][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pa[0][i])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pa[0][i + 32])); + + asm volatile ("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm5,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (p[1][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (p[1][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm6" : : "m" (pa[1][i])); + asm volatile ("vmovdqa %0,%%ymm7" : : "m" (pa[1][i + 32])); + + asm volatile ("vpxor %ymm6,%ymm2,%ymm2"); + asm volatile ("vpxor %ymm7,%ymm3,%ymm3"); + + asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[0])); + asm volatile ("vgf2p8mulb %ymm0,%ymm8,%ymm4"); + asm volatile ("vgf2p8mulb %ymm1,%ymm8,%ymm5"); + + asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[1])); + asm volatile ("vgf2p8mulb %ymm2,%ymm8,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm4,%ymm4"); + asm volatile ("vgf2p8mulb %ymm3,%ymm8,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm5,%ymm5"); + + asm volatile ("vmovdqa %%ymm4,%0" : "=m" (pa[0][i])); + asm volatile ("vmovdqa %%ymm5,%0" : "=m" (pa[0][i + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[2])); + asm volatile ("vgf2p8mulb %ymm0,%ymm8,%ymm6"); + asm volatile ("vgf2p8mulb %ymm1,%ymm8,%ymm7"); + + asm volatile ("vpbroadcastb %0,%%ymm8" : : "m" (V[3])); + asm volatile ("vgf2p8mulb %ymm2,%ymm8,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm6,%ymm6"); + asm volatile ("vgf2p8mulb %ymm3,%ymm8,%ymm9"); + asm volatile ("vpxor %ymm9,%ymm7,%ymm7"); + + asm volatile ("vmovdqa %%ymm6,%0" : "=m" (pa[1][i])); + asm volatile ("vmovdqa %%ymm7,%0" : "=m" (pa[1][i + 32])); + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * RAID recovering for two disks GFNI implementation @@ -422,9 +1009,6 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - /* - * pa[0] = V[0]*delta[0] ^ V[1]*delta[1] - */ asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[0])); asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2"); asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); @@ -437,7 +1021,6 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - /* pa[1] = V[2]*delta[0] ^ V[3]*delta[1] */ asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[2])); asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2"); asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); @@ -453,6 +1036,78 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) +/* + * RAID recovering AVX2 GFNI implementation + */ +void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*64+64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[j][i + 32])); + + asm volatile ("vmovdqa %0,%%ymm2" : : "m" (pa[j][i])); + asm volatile ("vmovdqa %0,%%ymm3" : : "m" (pa[j][i + 32])); + + asm volatile ("vpxor %ymm2,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pd[j * 64])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pd[j * 64 + 32])); + } + + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxor %ymm0,%ymm0,%ymm0"); + asm volatile ("vpxor %ymm1,%ymm1,%ymm1"); + for (k = 0; k < N; ++k) { + asm volatile ("vmovdqa %0,%%ymm4" : : "m" (pd[k * 64])); + asm volatile ("vmovdqa %0,%%ymm5" : : "m" (pd[k * 64 + 32])); + + asm volatile ("vpbroadcastb %0,%%ymm2" : : "m" (V[j * N + k])); + + asm volatile ("vgf2p8mulb %ymm4,%ymm2,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm0,%ymm0"); + + asm volatile ("vgf2p8mulb %ymm5,%ymm2,%ymm3"); + asm volatile ("vpxor %ymm3,%ymm1,%ymm1"); + } + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[j][i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[j][i + 32])); + } + } + + raid_avx_end(); +} +#endif + #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) /* * RAID recovering GFNI implementation @@ -490,13 +1145,14 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); - asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64])); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j * 64])); } + /* reconstruct */ for (j = 0; j < N; ++j) { asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); for (k = 0; k < N; ++k) { - asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64])); + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k * 64])); asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[j * N + k])); asm volatile ("vgf2p8mulb %zmm4,%zmm2,%zmm2"); asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); @@ -509,10 +1165,35 @@ void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void ** } #endif -#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +void raid_register_avx2gfni(void) +{ +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX2GFNI +#ifdef CONFIG_X86_64 + if (raid_cpu_has_avx2gfni()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx2gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx2gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx2gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx2gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx2gfni); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx2gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx2gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx2gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx2gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx2gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx2gfni); + } +#endif +#endif +#endif +} + void raid_register_avx512gfni(void) { -#if defined(USE_RAID_AES) +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX512GFNI +#ifdef CONFIG_X86_64 if (raid_cpu_has_avx512gfni()) { if (!raid_cpu_has_slow_avx512()) { raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni512", raid_gen2_avx512gfni); @@ -530,5 +1211,6 @@ void raid_register_avx512gfni(void) } } #endif -} #endif +#endif +} diff --git a/raid/internal.h b/raid/internal.h index 06426af3..c500661f 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -48,6 +48,9 @@ #if HAVE_AVX512GFNI #define CONFIG_AVX512GFNI 1 #endif +#if HAVE_AVX2GFNI +#define CONFIG_AVX2GFNI 1 +#endif #else /* if HAVE_CONFIG_H is not defined */ @@ -71,6 +74,7 @@ #ifdef CONFIG_X86_64 #define CONFIG_AVX512BW 1 #define CONFIG_AVX512GFNI 1 +#define CONFIG_AVX2GFNI 1 #endif #endif @@ -140,6 +144,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv); void raid_gen2_avx2(int nd, size_t size, void **vv); void raid_gen2_sse2ext(int nd, size_t size, void **vv); void raid_gen2_avx512bw(int nd, size_t size, void **vv); +void raid_gen2_avx2gfni(int nd, size_t size, void **vv); void raid_gen2_avx512gfni(int nd, size_t size, void **vv); void raid_genz_int32(int nd, size_t size, void **vv); void raid_genz_int64(int nd, size_t size, void **vv); @@ -151,24 +156,28 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv); void raid_gen3_ssse3ext(int nd, size_t size, void **vv); void raid_gen3_avx2ext(int nd, size_t size, void **vv); void raid_gen3_avx512bw(int nd, size_t size, void **vv); +void raid_gen3_avx2gfni(int nd, size_t size, void **vv); void raid_gen3_avx512gfni(int nd, size_t size, void **vv); void raid_gen4_int8(int nd, size_t size, void **vv); void raid_gen4_ssse3(int nd, size_t size, void **vv); void raid_gen4_ssse3ext(int nd, size_t size, void **vv); void raid_gen4_avx2ext(int nd, size_t size, void **vv); void raid_gen4_avx512bw(int nd, size_t size, void **vv); +void raid_gen4_avx2gfni(int nd, size_t size, void **vv); void raid_gen4_avx512gfni(int nd, size_t size, void **vv); void raid_gen5_int8(int nd, size_t size, void **vv); void raid_gen5_ssse3(int nd, size_t size, void **vv); void raid_gen5_ssse3ext(int nd, size_t size, void **vv); void raid_gen5_avx2ext(int nd, size_t size, void **vv); void raid_gen5_avx512bw(int nd, size_t size, void **vv); +void raid_gen5_avx2gfni(int nd, size_t size, void **vv); void raid_gen5_avx512gfni(int nd, size_t size, void **vv); void raid_gen6_int8(int nd, size_t size, void **vv); void raid_gen6_ssse3(int nd, size_t size, void **vv); void raid_gen6_ssse3ext(int nd, size_t size, void **vv); void raid_gen6_avx2ext(int nd, size_t size, void **vv); void raid_gen6_avx512bw(int nd, size_t size, void **vv); +void raid_gen6_avx2gfni(int nd, size_t size, void **vv); void raid_gen6_avx512gfni(int nd, size_t size, void **vv); void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -182,6 +191,9 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx2gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -288,6 +300,7 @@ void raid_register_x86(void); /** * Register all the functions based on GFNI intructions. */ +void raid_register_avx2gfni(void); void raid_register_avx512gfni(void); /* diff --git a/raid/module.c b/raid/module.c index 0b0458a9..ee4fa06e 100644 --- a/raid/module.c +++ b/raid/module.c @@ -247,6 +247,9 @@ void raid_init(void) #if defined(CONFIG_X86) && defined(CONFIG_SSE2) raid_register_x86(); #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX2GFNI) + raid_register_avx2gfni(); +#endif #if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) raid_register_avx512gfni(); #endif diff --git a/raid/test.c b/raid/test.c index ab474cb2..0392e76c 100644 --- a/raid/test.c +++ b/raid/test.c @@ -134,7 +134,7 @@ int raid_test_sort(void) int raid_test_rec(int mode, int nd, size_t size) { - void (*f[RAID_PARITY_MAX][4])( + void (*f[RAID_PARITY_MAX][128])( int nr, int *id, int *ip, int nd, size_t size, void **vbuf); void *v_alloc; void **v; @@ -203,6 +203,16 @@ int raid_test_rec(int mode, int nd, size_t size) if (raid_cpu_has_avx512bw()) f[i][nf[i]++] = raid_rec1_avx512bw; #endif +#endif +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) + f[i][nf[i]++] = raid_rec1_avx2gfni; +#endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) + f[i][nf[i]++] = raid_rec1_avx512gfni; +#endif #endif } else if (i == 1) { f[i][nf[i]++] = raid_rec2_int8; @@ -221,6 +231,16 @@ int raid_test_rec(int mode, int nd, size_t size) if (raid_cpu_has_avx512bw()) f[i][nf[i]++] = raid_rec2_avx512bw; #endif +#endif +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) + f[i][nf[i]++] = raid_rec2_avx2gfni; +#endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) + f[i][nf[i]++] = raid_rec2_avx512gfni; +#endif #endif } else { f[i][nf[i]++] = raid_recX_int8; @@ -239,6 +259,16 @@ int raid_test_rec(int mode, int nd, size_t size) if (raid_cpu_has_avx512bw()) f[i][nf[i]++] = raid_recX_avx512bw; #endif +#endif +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) + f[i][nf[i]++] = raid_recX_avx2gfni; +#endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) + f[i][nf[i]++] = raid_recX_avx512gfni; +#endif #endif } } @@ -307,7 +337,7 @@ bail: int raid_test_par(int mode, int nd, size_t size) { - void (*f[64])(int nd, size_t size, void **vbuf); + void (*f[128])(int nd, size_t size, void **vbuf); void *v_alloc; void **v; int nv; @@ -381,6 +411,18 @@ int raid_test_par(int mode, int nd, size_t size) } #endif #endif +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + f[nf++] = raid_gen2_avx2gfni; + } +#endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + f[nf++] = raid_gen2_avx512gfni; + } +#endif +#endif #endif /* CONFIG_X86 */ if (mode == RAID_MODE_CAUCHY) { @@ -426,6 +468,24 @@ int raid_test_par(int mode, int nd, size_t size) } #endif #endif +#ifdef USE_RAID_AES +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + f[nf++] = raid_gen3_avx2gfni; + f[nf++] = raid_gen4_avx2gfni; + f[nf++] = raid_gen5_avx2gfni; + f[nf++] = raid_gen6_avx2gfni; + } +#endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + f[nf++] = raid_gen3_avx512gfni; + f[nf++] = raid_gen4_avx512gfni; + f[nf++] = raid_gen5_avx512gfni; + f[nf++] = raid_gen6_avx512gfni; + } +#endif +#endif #endif /* CONFIG_X86 */ } else { diff --git a/raid/test/Makefile b/raid/test/Makefile index 0e57e11f..9a4cf10d 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -101,7 +101,6 @@ sdecovtest: # Use this target to run a complete test on Ice Lake simulator sde: $(SDE) -icl -- ./selftest - $(SDE) -clx -- ./speedtest $(SDE) -icl -- ./speedtest $(SDE) -icl -- ./fulltest diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index e4c521a2..dcccb2e5 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -39,6 +39,12 @@ int main(void) printf("Including x86 AVX2\n"); if (raid_cpu_has_avx512bw()) printf("Including x86 AVX512BW\n"); + if (raid_cpu_has_avx2gfni()) +#ifdef USE_RAID_AES + printf("Including x86 AVX2GFNI\n"); +#else + printf("Excluding x86 AVX2GFNI due polynomial mismatch\n"); +#endif if (raid_cpu_has_avx512gfni()) #ifdef USE_RAID_AES printf("Including x86 AVX512GFNI\n"); @@ -73,18 +79,13 @@ int main(void) /* LCOV_EXCL_STOP */ } - printf("Test Cauchy parity generation with %u data disks...\n", RAID_DATA_MAX); - if (raid_test_par(RAID_MODE_CAUCHY, RAID_DATA_MAX, TEST_SIZE) != 0) { - /* LCOV_EXCL_START */ - goto bail; - /* LCOV_EXCL_STOP */ - } - - printf("Test Cauchy parity generation with 1 data disk...\n"); - if (raid_test_par(RAID_MODE_CAUCHY, 1, TEST_SIZE) != 0) { - /* LCOV_EXCL_START */ - goto bail; - /* LCOV_EXCL_STOP */ + printf("Test Cauchy parity generation with 1-%u data disks...\n", RAID_DATA_MAX); + for (int i = 1; i <= RAID_DATA_MAX; ++i) { + if (raid_test_par(RAID_MODE_CAUCHY, i, TEST_SIZE) != 0) { + /* LCOV_EXCL_START */ + goto bail; + /* LCOV_EXCL_STOP */ + } } printf("Test Cauchy recovering with all combinations of %u data and 6 parity blocks...\n", TEST_COUNT); diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index ebcde8a2..5fb77511 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -137,9 +137,16 @@ void speed(void) printf("%8s", "avx2"); #ifdef CONFIG_X86_64 printf("%8s", "avx2e"); +#ifdef CONFIG_AVX512BW printf("%8s", "avx512"); +#endif +#ifdef CONFIG_AVX2GFNI + printf("%8s", "gfni"); +#endif +#ifdef CONFIG_AVX512GFNI printf("%8s", "gfni512"); #endif +#endif #endif printf("\n"); @@ -278,6 +285,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen2_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -423,6 +440,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen3_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -508,6 +535,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen4_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -593,6 +630,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen5_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -678,6 +725,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen6_avx2gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -702,9 +759,16 @@ void speed(void) printf("%8s", "ssse3"); printf("%8s", "avx2"); #ifdef CONFIG_X86_64 +#ifdef CONFIG_AVX512BW printf("%8s", "avx512"); +#endif +#ifdef CONFIG_AVX2GFNI + printf("%8s", "gfni"); +#endif +#ifdef CONFIG_AVX512GFNI printf("%8s", "gfni512"); #endif +#endif #endif printf("\n"); @@ -759,6 +823,18 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen_force(1, raid_gen1_avx2); + /* +1 to avoid GEN1 optimized case */ + raid_rec1_avx2gfni(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -828,6 +904,18 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen_force(2, raid_gen2_avx2gfni); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_avx2gfni(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -893,6 +981,17 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen_force(3, raid_gen3_avx2gfni); + raid_recX_avx2gfni(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -961,6 +1060,17 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen_force(4, raid_gen4_avx2gfni); + raid_recX_avx2gfni(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -1029,6 +1139,17 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen_force(5, raid_gen5_avx2gfni); + raid_recX_avx2gfni(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -1097,6 +1218,17 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX2GFNI + if (raid_cpu_has_avx2gfni()) { + SPEED_START { + raid_gen_force(6, raid_gen6_avx2gfni); + raid_recX_avx2gfni(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #ifdef CONFIG_AVX512GFNI if (raid_cpu_has_avx512gfni()) { SPEED_START { @@ -1130,6 +1262,12 @@ int main(void) printf("Including x86 AVX2\n"); if (raid_cpu_has_avx512bw()) printf("Including x86 AVX512BW\n"); + if (raid_cpu_has_avx2gfni()) +#ifdef USE_RAID_AES + printf("Including x86 AVX2GFNI\n"); +#else + printf("Including x86 AVX2GFNI (Performance reference only. Polynomial mismatch prevents use)\n"); +#endif if (raid_cpu_has_avx512gfni()) #ifdef USE_RAID_AES printf("Including x86 AVX512GFNI\n");