diff --git a/Makefile.am b/Makefile.am index aa7c6332..d5587087 100644 --- a/Makefile.am +++ b/Makefile.am @@ -26,6 +26,7 @@ snapraid_SOURCES = \ raid/tables.c \ raid/int.c \ raid/x86.c \ + raid/gfni.c \ raid/intz.c \ raid/x86z.c \ raid/helper.c \ diff --git a/cmdline/speed.c b/cmdline/speed.c index 5bc866bc..e5f0e4aa 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -286,6 +286,7 @@ void speed(int period, int nd, int size) #ifdef CONFIG_X86_64 printf("%8s", "avx2e"); printf("%8s", "avx512"); + printf("%8s", "gfni"); #endif #endif printf("\n"); @@ -342,6 +343,7 @@ void speed(int period, int nd, int size) #endif #ifdef CONFIG_X86_64 printf("%8s", ""); +#endif #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -352,7 +354,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#endif #endif printf("\n"); @@ -414,6 +415,7 @@ void speed(int period, int nd, int size) #endif #ifdef CONFIG_X86_64 printf("%8s", ""); +#endif #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -424,6 +426,15 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen2_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); @@ -560,6 +571,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen3_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -635,6 +656,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen4_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -710,6 +741,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen5_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -785,6 +826,16 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen6_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -800,6 +851,7 @@ void speed(int period, int nd, int size) printf("%8s", "avx2"); #ifdef CONFIG_X86_64 printf("%8s", "avx512"); + printf("%8s", "gfni"); #endif #endif printf("\n"); @@ -845,7 +897,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -859,6 +910,18 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ + /* +1 to avoid GEN1 optimized case */ + raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); @@ -908,7 +971,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -922,6 +984,18 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(2, raid_gen2_avx512gfni); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); @@ -972,7 +1046,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -986,6 +1059,18 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(3, raid_gen3_avx512gfni); + /* +1 to avoid GEN1 optimized case */ + raid_recX_avx512gfni(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); @@ -1037,7 +1122,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -1051,6 +1135,18 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(4, raid_gen4_avx512gfni); + /* +1 to avoid GEN1 optimized case */ + raid_recX_avx512gfni(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); @@ -1101,7 +1197,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -1115,6 +1210,18 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(5, raid_gen5_avx512gfni); + /* +1 to avoid GEN1 optimized case */ + raid_recX_avx512gfni(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); @@ -1165,7 +1272,6 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -1179,6 +1285,18 @@ void speed(int period, int nd, int size) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + /* ensure to use same hardware in the delta step */ + raid_gen_force(6, raid_gen6_avx512gfni); + /* +1 to avoid GEN1 optimized case */ + raid_recX_avx512gfni(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif #endif printf("\n"); diff --git a/configure.ac b/configure.ac index 9de437b3..946f9cee 100644 --- a/configure.ac +++ b/configure.ac @@ -288,7 +288,7 @@ dnl Checks for AS supporting the AVX512BW instructions. AC_MSG_CHECKING([for avx512bw]) asmavx512bw=no AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ -#if defined(__i386__) || defined(__x86_64__) +#if defined(__x86_64__) void f(void* ptr) { asm volatile("vpshufb %0, %%zmm0, %%zmm1" : : "m" (ptr)); @@ -300,6 +300,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ [AC_DEFINE([HAVE_AVX512BW], [1], [Define to 1 if avx512bw is supported by the assembler.]) asmavx512bw=yes]) AC_MSG_RESULT([$asmavx512bw]) +dnl Checks for AS supporting the AVX512GFNI instructions. +AC_MSG_CHECKING([for avx512gfni]) +asmavx512gfni=no +AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[ +#if defined(__x86_64__) + void f(void* ptr) + { + asm volatile ("vgf2p8mulb %0, %%zmm2, %%zmm2" : : "m" (ptr)); + } +#else +#error not x86 +#endif +]])], +[AC_DEFINE([HAVE_AVX512GFNI], [1], [Define to 1 if avx512gfni is supported by the assembler.]) asmavx512gfni=yes]) +AC_MSG_RESULT([$asmavx512gfni]) + dnl AS_IF(HAVE_ASSEMBLY) closed here ) diff --git a/raid/gf.h b/raid/gf.h index e1814566..67e72b9c 100644 --- a/raid/gf.h +++ b/raid/gf.h @@ -71,7 +71,39 @@ static __always_inline uint8_t A(int p, int d) #define v_64(p) (*(uint64_t *)&(p)) /* - * Multiply each byte of a uint32 by 2 in the GF(2^8). + * Polynomial-dependent XOR masks for GF(2^8) multiply and divide by 2. + * + * RAID_POLY_32 and RAID_POLY_64 are RAID_POLY repeated in every byte + * of a uint32/uint64, used as the conditional XOR in the multiply-by-2 + * path: when the MSB of a byte is set before the left shift, the + * reducing polynomial is XORed in to reduce the result back into the + * field. + * + * RAID_INV2_32 and RAID_INV2_64 are the GF(2^8) inverse of 2 (i.e. + * 2^{-1} mod RAID_POLY) repeated in every byte, used in the + * divide-by-2 path. The inverse depends on the polynomial: + * + * RAID_POLY = 0x1d -> 2^{-1} = 0x8e (standard RAID) + * RAID_POLY = 0x1b -> 2^{-1} = 0x8d (AES polynomial) + * + * Both are computed from RAID_POLY via the macro RAID_INV2_BYTE: + * in GF(2^8), 2^{-1} is the value x such that 2*x = 1, which for + * any polynomial of the form x^8 + ... + 1 resolves to: + * + * 2^{-1} = (RAID_POLY >> 1) | 0x80 + * + * since right-shifting the polynomial by 1 and setting the MSB gives + * the unique solution in both cases. + */ +#define RAID_INV2_BYTE (((RAID_POLY) >> 1) | 0x80) + +#define RAID_POLY_32 ((uint32_t)RAID_POLY * 0x01010101U) +#define RAID_POLY_64 ((uint64_t)RAID_POLY * 0x0101010101010101ULL) +#define RAID_INV2_32 ((uint32_t)RAID_INV2_BYTE * 0x01010101U) +#define RAID_INV2_64 ((uint64_t)RAID_INV2_BYTE * 0x0101010101010101ULL) + +/* + * Multiply each byte of a uint32 by 2 in GF(2^8). */ static __always_inline uint32_t x2_32(uint32_t v) { @@ -79,12 +111,12 @@ static __always_inline uint32_t x2_32(uint32_t v) mask = (mask << 1) - (mask >> 7); v = (v << 1) & 0xfefefefeU; - v ^= mask & 0x1d1d1d1dU; + v ^= mask & RAID_POLY_32; return v; } /* - * Multiply each byte of a uint64 by 2 in the GF(2^8). + * Multiply each byte of a uint64 by 2 in GF(2^8). */ static __always_inline uint64_t x2_64(uint64_t v) { @@ -92,12 +124,12 @@ static __always_inline uint64_t x2_64(uint64_t v) mask = (mask << 1) - (mask >> 7); v = (v << 1) & 0xfefefefefefefefeULL; - v ^= mask & 0x1d1d1d1d1d1d1d1dULL; + v ^= mask & RAID_POLY_64; return v; } /* - * Divide each byte of a uint32 by 2 in the GF(2^8). + * Divide each byte of a uint32 by 2 in GF(2^8). */ static __always_inline uint32_t d2_32(uint32_t v) { @@ -105,12 +137,12 @@ static __always_inline uint32_t d2_32(uint32_t v) mask = (mask << 8) - mask; v = (v >> 1) & 0x7f7f7f7fU; - v ^= mask & 0x8e8e8e8eU; + v ^= mask & RAID_INV2_32; return v; } /* - * Divide each byte of a uint64 by 2 in the GF(2^8). + * Divide each byte of a uint64 by 2 in GF(2^8). */ static __always_inline uint64_t d2_64(uint64_t v) { @@ -118,9 +150,8 @@ static __always_inline uint64_t d2_64(uint64_t v) mask = (mask << 8) - mask; v = (v >> 1) & 0x7f7f7f7f7f7f7f7fULL; - v ^= mask & 0x8e8e8e8e8e8e8e8eULL; + v ^= mask & RAID_INV2_64; return v; } #endif - diff --git a/raid/gfni.c b/raid/gfni.c new file mode 100644 index 00000000..21273ce5 --- /dev/null +++ b/raid/gfni.c @@ -0,0 +1,532 @@ +// SPDX-License-Identifier: GPL-2.0-or-later +// Copyright (C) 2026 Andrea Mazzoleni + +#include "internal.h" +#include "gf.h" +#include "cpu.h" + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * GEN2 (RAID6 with powers of 2) GFNI implementation + */ +void raid_gen2_avx512gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + int d, l = nd - 1; + size_t i; + uint8_t gf2 = 0x02; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gf2)); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0, %%zmm0" : : "m" (v[l][i])); + asm volatile ("vmovdqa64 %zmm0, %zmm1"); + + for (d = l - 1; d >= 0; --d) { + asm volatile ("vgf2p8mulb %zmm31, %zmm1, %zmm1"); + asm volatile ("vmovdqa64 %0, %%zmm2" : : "m" (v[d][i])); + asm volatile ("vpxorq %zmm2, %zmm0, %zmm0"); + asm volatile ("vpxorq %zmm2, %zmm1, %zmm1"); + } + + asm volatile ("vmovntdq %%zmm0, %0" : "=m" (p[i])); + asm volatile ("vmovntdq %%zmm1, %0" : "=m" (q[i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * GEN3 (triple parity with Cauchy matrix) GFNI implementation + */ +void raid_gen3_avx512gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 3; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l])); + + asm volatile ("vmovdqa64 %zmm6,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); + + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * GEN4 (quad parity with Cauchy matrix) GFNI implementation + */ +void raid_gen4_avx512gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + uint8_t *s = v[nd + 3]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 4; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][l])); + + asm volatile ("vmovdqa64 %zmm6,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2"); + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm3"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); + + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * GEN5 (penta parity with Cauchy matrix) GFNI implementation + */ +void raid_gen5_avx512gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + uint8_t *s = v[nd + 3]; + uint8_t *t = v[nd + 4]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 5; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][l])); + asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][l])); + + asm volatile ("vmovdqa64 %zmm6,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2"); + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm3"); + asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm4"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); + asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); + asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15"); + + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm6,%zmm4,%zmm4"); + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * GEN6 (hexa parity with Cauchy matrix) GFNI implementation + */ +void raid_gen6_avx512gfni(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p = v[nd]; + uint8_t *q = v[nd + 1]; + uint8_t *r = v[nd + 2]; + uint8_t *s = v[nd + 3]; + uint8_t *t = v[nd + 4]; + uint8_t *u = v[nd + 5]; + int d, l = nd - 1; + size_t i; + + if (l == 0) { + for (i = 0; i < 6; ++i) + memcpy(v[1 + i], v[0], size); + return; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* last disk without the by two multiplication */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][l])); + asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][l])); + asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][l])); + + asm volatile ("vmovdqa64 %zmm6,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2"); + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm3"); + asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm4"); + asm volatile ("vgf2p8mulb %zmm6,%zmm11,%zmm5"); + + /* intermediate disks */ + for (d = l - 1; d > 0; --d) { + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i])); + + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d])); + asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d])); + asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d])); + asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d])); + asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][d])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12"); + asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13"); + asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14"); + asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15"); + asm volatile ("vgf2p8mulb %zmm6,%zmm11,%zmm16"); + + asm volatile ("vpxorq %zmm12,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm13,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm14,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm15,%zmm4,%zmm4"); + asm volatile ("vpxorq %zmm16,%zmm5,%zmm5"); + } + + /* first disk with all coefficients at 1 */ + asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm6,%zmm1,%zmm1"); + asm volatile ("vpxorq %zmm6,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm6,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm6,%zmm4,%zmm4"); + asm volatile ("vpxorq %zmm6,%zmm5,%zmm5"); + + asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i])); + asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i])); + asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i])); + asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i])); + asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i])); + asm volatile ("vmovntdq %%zmm5,%0" : "=m" (u[i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * RAID recovering for one disk GFNI implementation + */ +void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p, *pa; + uint8_t G, V; + size_t i; + + (void)nr; + + if (ip[0] == 0) { + raid_rec1of1(id, nd, size, vv); + return; + } + + G = A(ip[0], id[0]); + V = inv(G); + + raid_delta_gen(1, id, ip, nd, size, vv); + + p = v[nd + ip[0]]; + pa = v[id[0]]; + + raid_avx_begin(); + + asm volatile ("vpbroadcastb %0,%%zmm4" : : "m" (V)); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vgf2p8mulb %zmm4,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * RAID recovering for two disks GFNI implementation + */ +void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + const int N = 2; + uint8_t *p[N], *pa[N]; + uint8_t G[N * N], V[N * N]; + size_t i; + int j, k; + + (void)nr; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i])); + asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i])); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + asm volatile ("vpxorq %zmm3,%zmm1,%zmm1"); + + asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + + /* + * pa[0] = V[0]*delta[0] ^ V[1]*delta[1] + */ + asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[0])); + asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + + asm volatile ("vpbroadcastb %0,%%zmm3" : : "m" (V[1])); + asm volatile ("vgf2p8mulb %zmm1,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i])); + + asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); + + /* pa[1] = V[2]*delta[0] ^ V[3]*delta[1] */ + asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[2])); + asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm6,%zmm6"); + + asm volatile ("vpbroadcastb %0,%%zmm3" : : "m" (V[3])); + asm volatile ("vgf2p8mulb %zmm1,%zmm3,%zmm3"); + asm volatile ("vpxorq %zmm3,%zmm6,%zmm6"); + + asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i])); + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +/* + * RAID recovering GFNI implementation + */ +void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + int N = nr; + uint8_t *p[RAID_PARITY_MAX]; + uint8_t *pa[RAID_PARITY_MAX]; + uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX]; + uint8_t buffer[RAID_PARITY_MAX*64+64]; + uint8_t *pd = __align_ptr(buffer, 64); + size_t i; + int j, k; + + for (j = 0; j < N; ++j) + for (k = 0; k < N; ++k) + G[j * N + k] = A(ip[j], id[k]); + + raid_invert(G, V, N); + raid_delta_gen(N, id, ip, nd, size, vv); + + for (j = 0; j < N; ++j) { + p[j] = v[nd + ip[j]]; + pa[j] = v[id[j]]; + } + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + /* delta */ + for (j = 0; j < N; ++j) { + asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i])); + asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i])); + asm volatile ("vpxorq %zmm1,%zmm0,%zmm0"); + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64])); + } + /* reconstruct */ + for (j = 0; j < N; ++j) { + asm volatile ("vpxorq %zmm0,%zmm0,%zmm0"); + for (k = 0; k < N; ++k) { + asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64])); + asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[j * N + k])); + asm volatile ("vgf2p8mulb %zmm4,%zmm2,%zmm2"); + asm volatile ("vpxorq %zmm2,%zmm0,%zmm0"); + } + asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i])); + } + } + + raid_avx_end(); +} +#endif + +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) +void raid_register_avx512gfni(void) +{ +#if defined(USE_RAID_AES) + if (raid_cpu_has_avx512gfni()) { + raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx512gfni); + raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx512gfni); + + raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx512gfni); + raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx512gfni); + } +#endif +} +#endif diff --git a/raid/internal.h b/raid/internal.h index ac11d185..06426af3 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -32,7 +32,7 @@ #endif #endif -/* Enables SSE2, SSSE3, AVX2 only if the assembler supports it */ +/* Enables SSE2, SSSE3, AVX2, AVX512BW, GFNI only if the assembler supports it */ #if HAVE_SSE2 #define CONFIG_SSE2 1 #endif @@ -42,9 +42,12 @@ #if HAVE_AVX2 #define CONFIG_AVX2 1 #endif -#if HAVE_AVX512BW /* Enables AVX2512BW only if the assembler supports it */ +#if HAVE_AVX512BW #define CONFIG_AVX512BW 1 #endif +#if HAVE_AVX512GFNI +#define CONFIG_AVX512GFNI 1 +#endif #else /* if HAVE_CONFIG_H is not defined */ @@ -67,6 +70,7 @@ #endif #ifdef CONFIG_X86_64 #define CONFIG_AVX512BW 1 +#define CONFIG_AVX512GFNI 1 #endif #endif @@ -136,6 +140,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv); void raid_gen2_avx2(int nd, size_t size, void **vv); void raid_gen2_sse2ext(int nd, size_t size, void **vv); void raid_gen2_avx512bw(int nd, size_t size, void **vv); +void raid_gen2_avx512gfni(int nd, size_t size, void **vv); void raid_genz_int32(int nd, size_t size, void **vv); void raid_genz_int64(int nd, size_t size, void **vv); void raid_genz_sse2(int nd, size_t size, void **vv); @@ -146,21 +151,25 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv); void raid_gen3_ssse3ext(int nd, size_t size, void **vv); void raid_gen3_avx2ext(int nd, size_t size, void **vv); void raid_gen3_avx512bw(int nd, size_t size, void **vv); +void raid_gen3_avx512gfni(int nd, size_t size, void **vv); void raid_gen4_int8(int nd, size_t size, void **vv); void raid_gen4_ssse3(int nd, size_t size, void **vv); void raid_gen4_ssse3ext(int nd, size_t size, void **vv); void raid_gen4_avx2ext(int nd, size_t size, void **vv); void raid_gen4_avx512bw(int nd, size_t size, void **vv); +void raid_gen4_avx512gfni(int nd, size_t size, void **vv); void raid_gen5_int8(int nd, size_t size, void **vv); void raid_gen5_ssse3(int nd, size_t size, void **vv); void raid_gen5_ssse3ext(int nd, size_t size, void **vv); void raid_gen5_avx2ext(int nd, size_t size, void **vv); void raid_gen5_avx512bw(int nd, size_t size, void **vv); +void raid_gen5_avx512gfni(int nd, size_t size, void **vv); void raid_gen6_int8(int nd, size_t size, void **vv); void raid_gen6_ssse3(int nd, size_t size, void **vv); void raid_gen6_ssse3ext(int nd, size_t size, void **vv); void raid_gen6_avx2ext(int nd, size_t size, void **vv); void raid_gen6_avx512bw(int nd, size_t size, void **vv); +void raid_gen6_avx512gfni(int nd, size_t size, void **vv); void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv); @@ -173,6 +182,9 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); +void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv); /* * Functions for parity computation. @@ -273,6 +285,11 @@ void raid_register_int(void); */ void raid_register_x86(void); +/** + * Register all the functions based on GFNI intructions. + */ +void raid_register_avx512gfni(void); + /* * Tag functions. * @@ -298,6 +315,7 @@ extern const uint8_t raid_gfvandermonde[3][256] __aligned(256); extern const uint8_t raid_gfcauchy[6][256] __aligned(256); extern const uint8_t raid_gfcauchypshufb[251][4][2][16] __aligned(256); extern const uint8_t raid_gfmulpshufb[256][2][16] __aligned(256); +extern const uint8_t raid_gfcauchycoeff[251][4][16] __aligned(256); extern const uint8_t (*raid_gfgen)[256]; #define gfmul raid_gfmul #define gfexp raid_gfexp @@ -306,6 +324,7 @@ extern const uint8_t (*raid_gfgen)[256]; #define gfcauchy raid_gfcauchy #define gfgenpshufb raid_gfcauchypshufb #define gfmulpshufb raid_gfmulpshufb +#define gfgencoeff raid_gfcauchycoeff #define gfgen raid_gfgen /* diff --git a/raid/mktables.c b/raid/mktables.c index e9126043..2264c939 100644 --- a/raid/mktables.c +++ b/raid/mktables.c @@ -4,6 +4,8 @@ #include #include +#include "raid.h" /* for RAID_POLY */ + /** * Multiplication a*b in GF(2^8). */ @@ -18,7 +20,7 @@ static uint8_t gfmul(uint8_t a, uint8_t b) if ((a & 0x80) != 0) { a <<= 1; - a ^= 0x1d; + a ^= RAID_POLY; } else { a <<= 1; } @@ -213,9 +215,7 @@ int main(void) uint8_t matrix[PARITY * 256]; printf("// SPDX-License-Identifier: GPL-2.0-or-later\n"); - printf("/*\n"); - printf(" * Copyright (C) 2013 Andrea Mazzoleni\n"); - printf(" */\n"); + printf("// Copyright (C) 2013 Andrea Mazzoleni\n"); printf("\n"); printf("#include \"internal.h\"\n"); diff --git a/raid/module.c b/raid/module.c index 9c07fac6..0b0458a9 100644 --- a/raid/module.c +++ b/raid/module.c @@ -247,6 +247,9 @@ void raid_init(void) #if defined(CONFIG_X86) && defined(CONFIG_SSE2) raid_register_x86(); #endif +#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI) + raid_register_avx512gfni(); +#endif /* set the default mode */ raid_mode(RAID_MODE_CAUCHY); diff --git a/raid/raid.h b/raid/raid.h index 517ca589..a8558a09 100644 --- a/raid/raid.h +++ b/raid/raid.h @@ -4,6 +4,47 @@ #ifndef __RAID_H #define __RAID_H +/* + * GF(2^8) reducing polynomial selection. + * + * RAID parity computation operates in GF(2^8), a finite field where + * arithmetic is performed modulo an irreducible polynomial of degree 8. + * The choice of polynomial affects both correctness and performance: + * all tables, coefficients, and parity data are tied to a specific + * polynomial and cannot be mixed. + * + * Two polynomials are supported: + * + * 0x1d (x^8 + x^4 + x^3 + x^2 + 1) -- Standard RAID polynomial + * + * The polynomial used by the original RAID-6 specification and the + * Linux kernel RAID implementation. Choosing this polynomial ensures + * on-disk compatibility with existing RAID arrays and interchange with + * other RAID implementations. Use this when reading or writing data + * that may be accessed by other software or hardware RAID controllers. + * + * 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial (USE_RAID_AES) + * + * The polynomial used by the AES encryption standard. Choosing this + * polynomial enables the use of the Intel GFNI (Galois Field New + * Instructions) extension, specifically the vgf2p8mulb instruction, + * which performs GF(2^8) multiplication natively in hardware using + * exactly this polynomial. This eliminates the nibble-split pshufb + * multiply sequence (vpsrlw + vpandq + 2x vpshufb + vpxorq) and the + * conditional multiply-by-2 sequence (vpcmpgtb + vpmovm2b + vpaddb + + * vpandq + vpxorq) used in the AVX-512BW path, replacing each with a + * single vgf2p8mulb instruction. On CPUs supporting GFNI (Ice Lake and + * later), this yields a significant throughput improvement. The + * trade-off is that parity data is not compatible with arrays using + * the standard 0x1d polynomial, so this option requires that all + * data be generated and recovered using the same polynomial choice. + */ +#ifdef USE_RAID_AES +#define RAID_POLY 0x1b +#else +#define RAID_POLY 0x1d +#endif + /** * RAID mode supporting up to 6 parities. * @@ -215,4 +256,3 @@ int raid_check(int nr, int *ir, int nd, int np, size_t size, void **v); int raid_scan(int *ir, int nd, int np, size_t size, void **v); #endif - diff --git a/raid/tables.c b/raid/tables.c index b9d50220..99aa0996 100644 --- a/raid/tables.c +++ b/raid/tables.c @@ -8749,7 +8749,7 @@ const uint8_t __aligned(256) raid_gfexp[256] = const uint8_t __aligned(256) raid_gfinv[256] = { - /* note that the first element is not significative */ + /* note that the first element is not significant */ 0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba, 0xad, 0x9d, 0xdd, 0x98, 0x3d, 0xaa, 0x5d, 0x96, 0xd8, 0x72, 0xc0, 0x58, 0xe0, 0x3e, 0x4c, 0x66, @@ -8786,7 +8786,7 @@ const uint8_t __aligned(256) raid_gfinv[256] = /** * Power matrix used to generate parity. - * This matrix is valid for up to 3 parity with 251 data disks. + * This matrix is valid for up to 3 parities with 251 data disks. * * 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 * 01 02 04 08 10 20 40 80 1d 3a 74 e8 cd 87 13 26 4c 98 2d 5a b4 75 ea c9 8f 03 06 0c 18 30 60 c0 9d 27 4e 9c 25 4a 94 35 6a d4 b5 77 ee c1 9f 23 46 8c 05 0a 14 28 50 a0 5d ba 69 d2 b9 6f de a1 5f be 61 c2 99 2f 5e bc 65 ca 89 0f 1e 3c 78 f0 fd e7 d3 bb 6b d6 b1 7f fe e1 df a3 5b b6 71 e2 d9 af 43 86 11 22 44 88 0d 1a 34 68 d0 bd 67 ce 81 1f 3e 7c f8 ed c7 93 3b 76 ec c5 97 33 66 cc 85 17 2e 5c b8 6d da a9 4f 9e 21 42 84 15 2a 54 a8 4d 9a 29 52 a4 55 aa 49 92 39 72 e4 d5 b7 73 e6 d1 bf 63 c6 91 3f 7e fc e5 d7 b3 7b f6 f1 ff e3 db ab 4b 96 31 62 c4 95 37 6e dc a5 57 ae 41 82 19 32 64 c8 8d 07 0e 1c 38 70 e0 dd a7 53 a6 51 a2 59 b2 79 f2 f9 ef c3 9b 2b 56 ac 45 8a 09 12 24 48 90 3d 7a f4 f5 f7 f3 fb eb cb 8b 0b 16 2c 58 b0 7d fa e9 cf 83 1b 36 6c diff --git a/raid/test/Makefile b/raid/test/Makefile index cbe7afb3..1c51b22d 100644 --- a/raid/test/Makefile +++ b/raid/test/Makefile @@ -24,7 +24,7 @@ CFLAGS += -O2 else CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1 endif -OBJS = raid.o check.o int.o intz.o x86.o x86z.o tables.o memory.o test.o helper.o module.o +OBJS = raid.o check.o int.o intz.o x86.o x86z.o gfni.o tables.o memory.o test.o helper.o module.o %.o: ../%.c $(CC) $(CFLAGS) -c -o $@ $< diff --git a/raid/test/fulltest.c b/raid/test/fulltest.c index b2296393..b2e5cdd3 100644 --- a/raid/test/fulltest.c +++ b/raid/test/fulltest.c @@ -39,6 +39,8 @@ int main(void) printf("Including x86 AVX2 functions\n"); if (raid_cpu_has_avx512bw()) printf("Including x86 AVX512BW functions\n"); + if (raid_cpu_has_avx512gfni()) + printf("Including x86 AVX512GFNI functions\n"); #endif #ifdef CONFIG_X86_64 printf("Including x64 extended SSE register set\n"); diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index cd0aad17..c17e70ce 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -138,6 +138,7 @@ void speed(void) #ifdef CONFIG_X86_64 printf("%8s", "avx2e"); printf("%8s", "avx512"); + printf("%8s", "gfni"); #endif #endif printf("\n"); @@ -192,10 +193,11 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_X86_64 + printf("%8s", ""); +#endif #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { - printf("%8s", ""); - SPEED_START { raid_gen1_avx512bw(nd, size, v); } SPEED_STOP @@ -263,10 +265,11 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_X86_64 + printf("%8s", ""); +#endif #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { - printf("%8s", ""); - SPEED_START { raid_gen2_avx512bw(nd, size, v); } SPEED_STOP @@ -275,6 +278,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen2_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif printf("\n"); @@ -410,6 +423,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen3_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -485,6 +508,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen4_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -560,6 +593,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen5_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -635,6 +678,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen6_avx512gfni(nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } +#endif #endif #endif printf("\n"); @@ -650,6 +703,7 @@ void speed(void) printf("%8s", "avx2"); #ifdef CONFIG_X86_64 printf("%8s", "avx512"); + printf("%8s", "gfni"); #endif #endif printf("\n"); @@ -693,7 +747,6 @@ void speed(void) } #endif #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -706,6 +759,17 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */ + /* +1 to avoid GEN1 optimized case */ + raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); @@ -752,7 +816,6 @@ void speed(void) } #endif #endif -#ifdef CONFIG_X86_64 #ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { @@ -765,6 +828,17 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen_force(2, raid_gen2_avx512gfni); + /* +1 to avoid GEN2 optimized case */ + raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); @@ -808,8 +882,7 @@ void speed(void) } #endif #endif -#ifdef CONFIG_X86_64 -#ifdef CONFIG_AVX2 +#ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { raid_gen_force(3, raid_gen3_avx512bw); @@ -820,6 +893,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen_force(3, raid_gen3_avx512gfni); + raid_recX_avx512gfni(3, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); @@ -867,8 +950,7 @@ void speed(void) } #endif #endif -#ifdef CONFIG_X86_64 -#ifdef CONFIG_AVX2 +#ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { raid_gen_force(4, raid_gen4_avx512bw); @@ -879,6 +961,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen_force(4, raid_gen4_avx512gfni); + raid_recX_avx512gfni(4, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); @@ -926,8 +1018,7 @@ void speed(void) } #endif #endif -#ifdef CONFIG_X86_64 -#ifdef CONFIG_AVX2 +#ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { raid_gen_force(5, raid_gen5_avx512bw); @@ -938,6 +1029,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen_force(5, raid_gen5_avx512gfni); + raid_recX_avx512gfni(5, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); @@ -985,8 +1086,7 @@ void speed(void) } #endif #endif -#ifdef CONFIG_X86_64 -#ifdef CONFIG_AVX2 +#ifdef CONFIG_AVX512BW if (raid_cpu_has_avx512bw()) { SPEED_START { raid_gen_force(6, raid_gen6_avx512bw); @@ -997,6 +1097,16 @@ void speed(void) fflush(stdout); } #endif +#ifdef CONFIG_AVX512GFNI + if (raid_cpu_has_avx512gfni()) { + SPEED_START { + raid_gen_force(6, raid_gen6_avx512gfni); + raid_recX_avx512gfni(6, id, ip, nd, size, v); + } SPEED_STOP + + printf("%8" PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); printf("\n"); @@ -1019,7 +1129,9 @@ int main(void) if (raid_cpu_has_avx2()) printf("Including x86 AVX2 functions\n"); if (raid_cpu_has_avx512bw()) - printf("Including x86 AVX512bw functions\n"); + printf("Including x86 AVX512BW functions\n"); + if (raid_cpu_has_avx512gfni()) + printf("Including x86 AVX512GFNI functions\n"); #endif #ifdef CONFIG_X86_64 printf("Including x64 extended SSE register set\n"); diff --git a/raid/x86.c b/raid/x86.c index dcbafeec..6a3bb5eb 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -35,6 +35,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) raid_sse_begin(); + /* additional unrolling makes it slower */ for (i = 0; i < size; i += 64) { asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i])); asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16])); @@ -76,6 +77,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv) raid_avx_begin(); + /* additional unrolling makes it slower */ for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32])); @@ -115,6 +117,7 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) raid_avx_begin(); + /* additional unrolling makes it slower */ for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[l][i])); for (d = l - 1; d >= 0; --d) { @@ -131,10 +134,10 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv) static const struct gfconst16 { uint8_t poly[16]; uint8_t low4[16]; -} gfconst16 __aligned(32) = { +} gfconst16 __aligned(16) = { { - 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, - 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY }, { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, @@ -215,7 +218,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv) raid_avx_begin(); - asm volatile ("vbroadcasti128 %0, %%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0, %%ymm7" : : "m" (gfconst16.poly[0])); asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); for (i = 0; i < size; i += 64) { @@ -268,7 +271,7 @@ void raid_gen2_avx512bw(int nd, size_t size, void **vv) raid_avx_begin(); - asm volatile ("vbroadcasti32x4 %0, %%zmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0, %%zmm14" : : "m" (gfconst16.poly[0])); asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); for (i = 0; i < size; i += 64) { @@ -641,8 +644,8 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv) raid_avx_begin(); /* generic case with at least two data disks */ - asm volatile ("vbroadcasti128 %0, %%ymm3" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti128 %0, %%ymm11" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0, %%ymm3" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* last disk without the by two multiplication */ @@ -773,8 +776,8 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv) /* generic case with at least two data disks */ asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* last disk without the by two multiplication */ @@ -1163,7 +1166,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) /* generic case with at least two data disks */ for (i = 0; i < size; i += 64) { /* last disk without the by two multiplication */ - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32])); @@ -1199,8 +1202,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) /* intermediate disks */ for (d = l - 1; d > 0; --d) { - asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32])); @@ -1251,8 +1254,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv) } /* first disk with all coefficients at 1 */ - asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i])); asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32])); @@ -1321,8 +1324,8 @@ void raid_gen4_avx512bw(int nd, size_t size, void **vv) /* generic case with at least two data disks */ asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* last disk without the by two multiplication */ @@ -1708,8 +1711,8 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv) /* generic case with at least two data disks */ asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { /* last disk without the by two multiplication */ @@ -1836,8 +1839,8 @@ void raid_gen5_avx512bw(int nd, size_t size, void **vv) /* generic case with at least two data disks */ asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* last disk without the by two multiplication */ @@ -2289,8 +2292,8 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv) /* generic case with at least two data disks */ asm volatile ("vpxor %ymm6,%ymm6,%ymm6"); - asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { /* last disk without the by two multiplication */ @@ -2445,8 +2448,8 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv) /* generic case with at least two data disks */ asm volatile ("vpxorq %zmm6,%zmm6,%zmm6"); - asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0])); - asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* last disk without the by two multiplication */ @@ -2833,7 +2836,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_avx_begin(); - asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0])); asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfmulpshufb[V][1][0])); @@ -2889,7 +2892,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_avx_begin(); - asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i])); @@ -2987,7 +2990,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) raid_avx_begin(); - asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 32) { /* delta */ @@ -3055,7 +3058,7 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv raid_avx_begin(); - asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfmulpshufb[V][0][0])); asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfmulpshufb[V][1][0])); @@ -3105,7 +3108,7 @@ void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv raid_avx_begin(); - asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i])); @@ -3201,7 +3204,7 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv raid_avx_begin(); - asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0])); + asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0])); for (i = 0; i < size; i += 64) { /* delta */ diff --git a/raid/x86z.c b/raid/x86z.c index 084479a3..29956f15 100644 --- a/raid/x86z.c +++ b/raid/x86z.c @@ -2,21 +2,22 @@ // Copyright (C) 2013 Andrea Mazzoleni #include "internal.h" +#include "gf.h" #if defined(CONFIG_X86) && defined(CONFIG_SSE2) static const struct gfzconst16 { uint8_t poly[16]; uint8_t half[16]; uint8_t low7[16]; -} gfzconst16 __aligned(64) = +} gfzconst16 __aligned(16) = { { - 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, - 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, + RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY }, { - 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, - 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e + RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, + RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE }, { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, @@ -187,9 +188,9 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv) raid_avx_begin(); - asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfzconst16.poly[0])); - asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfzconst16.half[0])); - asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfzconst16.low7[0])); + asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfzconst16.poly[0])); + asm volatile ("vpbroadcastb %0,%%ymm3" : : "m" (gfzconst16.half[0])); + asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfzconst16.low7[0])); asm volatile ("vpxor %ymm15,%ymm15,%ymm15"); for (i = 0; i < size; i += 64) {