mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-25 18:40:19 -04:00
Preliminary support for AVX512GFNI
This commit is contained in:
Andrea Mazzoleni
parent
c632bfc5af
commit
746a9b8940
15 changed files with 958 additions and 80 deletions
|
|
@ -26,6 +26,7 @@ snapraid_SOURCES = \
|
|||
raid/tables.c \
|
||||
raid/int.c \
|
||||
raid/x86.c \
|
||||
raid/gfni.c \
|
||||
raid/intz.c \
|
||||
raid/x86z.c \
|
||||
raid/helper.c \
|
||||
|
|
|
|||
132
cmdline/speed.c
132
cmdline/speed.c
|
|
@ -286,6 +286,7 @@ void speed(int period, int nd, int size)
|
|||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx2e");
|
||||
printf("%8s", "avx512");
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -342,6 +343,7 @@ void speed(int period, int nd, int size)
|
|||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -352,7 +354,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -414,6 +415,7 @@ void speed(int period, int nd, int size)
|
|||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -424,6 +426,15 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen2_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -560,6 +571,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen3_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -635,6 +656,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen4_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -710,6 +741,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen5_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -785,6 +826,16 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen6_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -800,6 +851,7 @@ void speed(int period, int nd, int size)
|
|||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx512");
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -845,7 +897,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -859,6 +910,18 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -908,7 +971,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -922,6 +984,18 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(2, raid_gen2_avx512gfni);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -972,7 +1046,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -986,6 +1059,18 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(3, raid_gen3_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -1037,7 +1122,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1051,6 +1135,18 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(4, raid_gen4_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -1101,7 +1197,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1115,6 +1210,18 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(5, raid_gen5_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -1165,7 +1272,6 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -1179,6 +1285,18 @@ void speed(int period, int nd, int size)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
/* ensure to use same hardware in the delta step */
|
||||
raid_gen_force(6, raid_gen6_avx512gfni);
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_recX_avx512gfni(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
|
|||
18
configure.ac
18
configure.ac
|
|
@ -288,7 +288,7 @@ dnl Checks for AS supporting the AVX512BW instructions.
|
|||
AC_MSG_CHECKING([for avx512bw])
|
||||
asmavx512bw=no
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
#if defined(__i386__) || defined(__x86_64__)
|
||||
#if defined(__x86_64__)
|
||||
void f(void* ptr)
|
||||
{
|
||||
asm volatile("vpshufb %0, %%zmm0, %%zmm1" : : "m" (ptr));
|
||||
|
|
@ -300,6 +300,22 @@ AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
|||
[AC_DEFINE([HAVE_AVX512BW], [1], [Define to 1 if avx512bw is supported by the assembler.]) asmavx512bw=yes])
|
||||
AC_MSG_RESULT([$asmavx512bw])
|
||||
|
||||
dnl Checks for AS supporting the AVX512GFNI instructions.
|
||||
AC_MSG_CHECKING([for avx512gfni])
|
||||
asmavx512gfni=no
|
||||
AC_COMPILE_IFELSE([AC_LANG_PROGRAM([[
|
||||
#if defined(__x86_64__)
|
||||
void f(void* ptr)
|
||||
{
|
||||
asm volatile ("vgf2p8mulb %0, %%zmm2, %%zmm2" : : "m" (ptr));
|
||||
}
|
||||
#else
|
||||
#error not x86
|
||||
#endif
|
||||
]])],
|
||||
[AC_DEFINE([HAVE_AVX512GFNI], [1], [Define to 1 if avx512gfni is supported by the assembler.]) asmavx512gfni=yes])
|
||||
AC_MSG_RESULT([$asmavx512gfni])
|
||||
|
||||
dnl AS_IF(HAVE_ASSEMBLY) closed here
|
||||
)
|
||||
|
||||
|
|
|
|||
49
raid/gf.h
49
raid/gf.h
|
|
@ -71,7 +71,39 @@ static __always_inline uint8_t A(int p, int d)
|
|||
#define v_64(p) (*(uint64_t *)&(p))
|
||||
|
||||
/*
|
||||
* Multiply each byte of a uint32 by 2 in the GF(2^8).
|
||||
* Polynomial-dependent XOR masks for GF(2^8) multiply and divide by 2.
|
||||
*
|
||||
* RAID_POLY_32 and RAID_POLY_64 are RAID_POLY repeated in every byte
|
||||
* of a uint32/uint64, used as the conditional XOR in the multiply-by-2
|
||||
* path: when the MSB of a byte is set before the left shift, the
|
||||
* reducing polynomial is XORed in to reduce the result back into the
|
||||
* field.
|
||||
*
|
||||
* RAID_INV2_32 and RAID_INV2_64 are the GF(2^8) inverse of 2 (i.e.
|
||||
* 2^{-1} mod RAID_POLY) repeated in every byte, used in the
|
||||
* divide-by-2 path. The inverse depends on the polynomial:
|
||||
*
|
||||
* RAID_POLY = 0x1d -> 2^{-1} = 0x8e (standard RAID)
|
||||
* RAID_POLY = 0x1b -> 2^{-1} = 0x8d (AES polynomial)
|
||||
*
|
||||
* Both are computed from RAID_POLY via the macro RAID_INV2_BYTE:
|
||||
* in GF(2^8), 2^{-1} is the value x such that 2*x = 1, which for
|
||||
* any polynomial of the form x^8 + ... + 1 resolves to:
|
||||
*
|
||||
* 2^{-1} = (RAID_POLY >> 1) | 0x80
|
||||
*
|
||||
* since right-shifting the polynomial by 1 and setting the MSB gives
|
||||
* the unique solution in both cases.
|
||||
*/
|
||||
#define RAID_INV2_BYTE (((RAID_POLY) >> 1) | 0x80)
|
||||
|
||||
#define RAID_POLY_32 ((uint32_t)RAID_POLY * 0x01010101U)
|
||||
#define RAID_POLY_64 ((uint64_t)RAID_POLY * 0x0101010101010101ULL)
|
||||
#define RAID_INV2_32 ((uint32_t)RAID_INV2_BYTE * 0x01010101U)
|
||||
#define RAID_INV2_64 ((uint64_t)RAID_INV2_BYTE * 0x0101010101010101ULL)
|
||||
|
||||
/*
|
||||
* Multiply each byte of a uint32 by 2 in GF(2^8).
|
||||
*/
|
||||
static __always_inline uint32_t x2_32(uint32_t v)
|
||||
{
|
||||
|
|
@ -79,12 +111,12 @@ static __always_inline uint32_t x2_32(uint32_t v)
|
|||
|
||||
mask = (mask << 1) - (mask >> 7);
|
||||
v = (v << 1) & 0xfefefefeU;
|
||||
v ^= mask & 0x1d1d1d1dU;
|
||||
v ^= mask & RAID_POLY_32;
|
||||
return v;
|
||||
}
|
||||
|
||||
/*
|
||||
* Multiply each byte of a uint64 by 2 in the GF(2^8).
|
||||
* Multiply each byte of a uint64 by 2 in GF(2^8).
|
||||
*/
|
||||
static __always_inline uint64_t x2_64(uint64_t v)
|
||||
{
|
||||
|
|
@ -92,12 +124,12 @@ static __always_inline uint64_t x2_64(uint64_t v)
|
|||
|
||||
mask = (mask << 1) - (mask >> 7);
|
||||
v = (v << 1) & 0xfefefefefefefefeULL;
|
||||
v ^= mask & 0x1d1d1d1d1d1d1d1dULL;
|
||||
v ^= mask & RAID_POLY_64;
|
||||
return v;
|
||||
}
|
||||
|
||||
/*
|
||||
* Divide each byte of a uint32 by 2 in the GF(2^8).
|
||||
* Divide each byte of a uint32 by 2 in GF(2^8).
|
||||
*/
|
||||
static __always_inline uint32_t d2_32(uint32_t v)
|
||||
{
|
||||
|
|
@ -105,12 +137,12 @@ static __always_inline uint32_t d2_32(uint32_t v)
|
|||
|
||||
mask = (mask << 8) - mask;
|
||||
v = (v >> 1) & 0x7f7f7f7fU;
|
||||
v ^= mask & 0x8e8e8e8eU;
|
||||
v ^= mask & RAID_INV2_32;
|
||||
return v;
|
||||
}
|
||||
|
||||
/*
|
||||
* Divide each byte of a uint64 by 2 in the GF(2^8).
|
||||
* Divide each byte of a uint64 by 2 in GF(2^8).
|
||||
*/
|
||||
static __always_inline uint64_t d2_64(uint64_t v)
|
||||
{
|
||||
|
|
@ -118,9 +150,8 @@ static __always_inline uint64_t d2_64(uint64_t v)
|
|||
|
||||
mask = (mask << 8) - mask;
|
||||
v = (v >> 1) & 0x7f7f7f7f7f7f7f7fULL;
|
||||
v ^= mask & 0x8e8e8e8e8e8e8e8eULL;
|
||||
v ^= mask & RAID_INV2_64;
|
||||
return v;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
|
|
|
|||
532
raid/gfni.c
Normal file
532
raid/gfni.c
Normal file
|
|
@ -0,0 +1,532 @@
|
|||
// SPDX-License-Identifier: GPL-2.0-or-later
|
||||
// Copyright (C) 2026 Andrea Mazzoleni
|
||||
|
||||
#include "internal.h"
|
||||
#include "gf.h"
|
||||
#include "cpu.h"
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN2 (RAID6 with powers of 2) GFNI implementation
|
||||
*/
|
||||
void raid_gen2_avx512gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
uint8_t gf2 = 0x02;
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm31" : : "m" (gf2));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0, %%zmm0" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa64 %zmm0, %zmm1");
|
||||
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
asm volatile ("vgf2p8mulb %zmm31, %zmm1, %zmm1");
|
||||
asm volatile ("vmovdqa64 %0, %%zmm2" : : "m" (v[d][i]));
|
||||
asm volatile ("vpxorq %zmm2, %zmm0, %zmm0");
|
||||
asm volatile ("vpxorq %zmm2, %zmm1, %zmm1");
|
||||
}
|
||||
|
||||
asm volatile ("vmovntdq %%zmm0, %0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%zmm1, %0" : "=m" (q[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen3_avx512gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 3; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l]));
|
||||
|
||||
asm volatile ("vmovdqa64 %zmm6,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13");
|
||||
|
||||
asm volatile ("vpxorq %zmm12,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm6,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm6,%zmm2,%zmm2");
|
||||
|
||||
asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen4_avx512gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
uint8_t *s = v[nd + 3];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 4; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][l]));
|
||||
|
||||
asm volatile ("vmovdqa64 %zmm6,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm3");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14");
|
||||
|
||||
asm volatile ("vpxorq %zmm12,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm14,%zmm3,%zmm3");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm6,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm6,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm6,%zmm3,%zmm3");
|
||||
|
||||
asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN5 (penta parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen5_avx512gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
uint8_t *s = v[nd + 3];
|
||||
uint8_t *t = v[nd + 4];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 5; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][l]));
|
||||
|
||||
asm volatile ("vmovdqa64 %zmm6,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm3");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm4");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15");
|
||||
|
||||
asm volatile ("vpxorq %zmm12,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm14,%zmm3,%zmm3");
|
||||
asm volatile ("vpxorq %zmm15,%zmm4,%zmm4");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm6,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm6,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm6,%zmm3,%zmm3");
|
||||
asm volatile ("vpxorq %zmm6,%zmm4,%zmm4");
|
||||
|
||||
asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i]));
|
||||
asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* GEN6 (hexa parity with Cauchy matrix) GFNI implementation
|
||||
*/
|
||||
void raid_gen6_avx512gfni(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p = v[nd];
|
||||
uint8_t *q = v[nd + 1];
|
||||
uint8_t *r = v[nd + 2];
|
||||
uint8_t *s = v[nd + 3];
|
||||
uint8_t *t = v[nd + 4];
|
||||
uint8_t *u = v[nd + 5];
|
||||
int d, l = nd - 1;
|
||||
size_t i;
|
||||
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 6; ++i)
|
||||
memcpy(v[1 + i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[l][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][l]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][l]));
|
||||
|
||||
asm volatile ("vmovdqa64 %zmm6,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm1");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm2");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm3");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm4");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm11,%zmm5");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[d][i]));
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfcauchy[1][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm8" : : "m" (gfcauchy[2][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm9" : : "m" (gfcauchy[3][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm10" : : "m" (gfcauchy[4][d]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm11" : : "m" (gfcauchy[5][d]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm7,%zmm12");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm8,%zmm13");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm9,%zmm14");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm10,%zmm15");
|
||||
asm volatile ("vgf2p8mulb %zmm6,%zmm11,%zmm16");
|
||||
|
||||
asm volatile ("vpxorq %zmm12,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm13,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm14,%zmm3,%zmm3");
|
||||
asm volatile ("vpxorq %zmm15,%zmm4,%zmm4");
|
||||
asm volatile ("vpxorq %zmm16,%zmm5,%zmm5");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vmovdqa64 %0,%%zmm6" : : "m" (v[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm6,%zmm1,%zmm1");
|
||||
asm volatile ("vpxorq %zmm6,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm6,%zmm3,%zmm3");
|
||||
asm volatile ("vpxorq %zmm6,%zmm4,%zmm4");
|
||||
asm volatile ("vpxorq %zmm6,%zmm5,%zmm5");
|
||||
|
||||
asm volatile ("vmovntdq %%zmm0,%0" : "=m" (p[i]));
|
||||
asm volatile ("vmovntdq %%zmm1,%0" : "=m" (q[i]));
|
||||
asm volatile ("vmovntdq %%zmm2,%0" : "=m" (r[i]));
|
||||
asm volatile ("vmovntdq %%zmm3,%0" : "=m" (s[i]));
|
||||
asm volatile ("vmovntdq %%zmm4,%0" : "=m" (t[i]));
|
||||
asm volatile ("vmovntdq %%zmm5,%0" : "=m" (u[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* RAID recovering for one disk GFNI implementation
|
||||
*/
|
||||
void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p, *pa;
|
||||
uint8_t G, V;
|
||||
size_t i;
|
||||
|
||||
(void)nr;
|
||||
|
||||
if (ip[0] == 0) {
|
||||
raid_rec1of1(id, nd, size, vv);
|
||||
return;
|
||||
}
|
||||
|
||||
G = A(ip[0], id[0]);
|
||||
V = inv(G);
|
||||
|
||||
raid_delta_gen(1, id, ip, nd, size, vv);
|
||||
|
||||
p = v[nd + ip[0]];
|
||||
pa = v[id[0]];
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm4" : : "m" (V));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[i]));
|
||||
asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");
|
||||
asm volatile ("vgf2p8mulb %zmm4,%zmm0,%zmm0");
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* RAID recovering for two disks GFNI implementation
|
||||
*/
|
||||
void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
const int N = 2;
|
||||
uint8_t *p[N], *pa[N];
|
||||
uint8_t G[N * N], V[N * N];
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
(void)nr;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm2" : : "m" (pa[0][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (p[1][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm3" : : "m" (pa[1][i]));
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
asm volatile ("vpxorq %zmm3,%zmm1,%zmm1");
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
/*
|
||||
* pa[0] = V[0]*delta[0] ^ V[1]*delta[1]
|
||||
*/
|
||||
asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[0]));
|
||||
asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm3" : : "m" (V[1]));
|
||||
asm volatile ("vgf2p8mulb %zmm1,%zmm3,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[0][i]));
|
||||
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
/* pa[1] = V[2]*delta[0] ^ V[3]*delta[1] */
|
||||
asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[2]));
|
||||
asm volatile ("vgf2p8mulb %zmm0,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vpbroadcastb %0,%%zmm3" : : "m" (V[3]));
|
||||
asm volatile ("vgf2p8mulb %zmm1,%zmm3,%zmm3");
|
||||
asm volatile ("vpxorq %zmm3,%zmm6,%zmm6");
|
||||
|
||||
asm volatile ("vmovdqa64 %%zmm6,%0" : "=m" (pa[1][i]));
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
/*
|
||||
* RAID recovering GFNI implementation
|
||||
*/
|
||||
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
int N = nr;
|
||||
uint8_t *p[RAID_PARITY_MAX];
|
||||
uint8_t *pa[RAID_PARITY_MAX];
|
||||
uint8_t G[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t V[RAID_PARITY_MAX * RAID_PARITY_MAX];
|
||||
uint8_t buffer[RAID_PARITY_MAX*64+64];
|
||||
uint8_t *pd = __align_ptr(buffer, 64);
|
||||
size_t i;
|
||||
int j, k;
|
||||
|
||||
for (j = 0; j < N; ++j)
|
||||
for (k = 0; k < N; ++k)
|
||||
G[j * N + k] = A(ip[j], id[k]);
|
||||
|
||||
raid_invert(G, V, N);
|
||||
raid_delta_gen(N, id, ip, nd, size, vv);
|
||||
|
||||
for (j = 0; j < N; ++j) {
|
||||
p[j] = v[nd + ip[j]];
|
||||
pa[j] = v[id[j]];
|
||||
}
|
||||
|
||||
raid_avx_begin();
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* delta */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[j][i]));
|
||||
asm volatile ("vmovdqa64 %0,%%zmm1" : : "m" (pa[j][i]));
|
||||
asm volatile ("vpxorq %zmm1,%zmm0,%zmm0");
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pd[j*64]));
|
||||
}
|
||||
/* reconstruct */
|
||||
for (j = 0; j < N; ++j) {
|
||||
asm volatile ("vpxorq %zmm0,%zmm0,%zmm0");
|
||||
for (k = 0; k < N; ++k) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm4" : : "m" (pd[k*64]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm2" : : "m" (V[j * N + k]));
|
||||
asm volatile ("vgf2p8mulb %zmm4,%zmm2,%zmm2");
|
||||
asm volatile ("vpxorq %zmm2,%zmm0,%zmm0");
|
||||
}
|
||||
asm volatile ("vmovdqa64 %%zmm0,%0" : "=m" (pa[j][i]));
|
||||
}
|
||||
}
|
||||
|
||||
raid_avx_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
void raid_register_avx512gfni(void)
|
||||
{
|
||||
#if defined(USE_RAID_AES)
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_gen2_avx512gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_gen3_avx512gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_gen4_avx512gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_gen5_avx512gfni);
|
||||
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_gen6_avx512gfni);
|
||||
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "gfni", raid_rec1_avx512gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "gfni", raid_rec2_avx512gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR3, "gfni", raid_recX_avx512gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR4, "gfni", raid_recX_avx512gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR5, "gfni", raid_recX_avx512gfni);
|
||||
raid_rec_register(RAID_ALGO_CAUCHY_PAR6, "gfni", raid_recX_avx512gfni);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
|
@ -32,7 +32,7 @@
|
|||
#endif
|
||||
#endif
|
||||
|
||||
/* Enables SSE2, SSSE3, AVX2 only if the assembler supports it */
|
||||
/* Enables SSE2, SSSE3, AVX2, AVX512BW, GFNI only if the assembler supports it */
|
||||
#if HAVE_SSE2
|
||||
#define CONFIG_SSE2 1
|
||||
#endif
|
||||
|
|
@ -42,9 +42,12 @@
|
|||
#if HAVE_AVX2
|
||||
#define CONFIG_AVX2 1
|
||||
#endif
|
||||
#if HAVE_AVX512BW /* Enables AVX2512BW only if the assembler supports it */
|
||||
#if HAVE_AVX512BW
|
||||
#define CONFIG_AVX512BW 1
|
||||
#endif
|
||||
#if HAVE_AVX512GFNI
|
||||
#define CONFIG_AVX512GFNI 1
|
||||
#endif
|
||||
|
||||
#else /* if HAVE_CONFIG_H is not defined */
|
||||
|
||||
|
|
@ -67,6 +70,7 @@
|
|||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#define CONFIG_AVX512BW 1
|
||||
#define CONFIG_AVX512GFNI 1
|
||||
#endif
|
||||
#endif
|
||||
|
||||
|
|
@ -136,6 +140,7 @@ void raid_gen2_sse2(int nd, size_t size, void **vv);
|
|||
void raid_gen2_avx2(int nd, size_t size, void **vv);
|
||||
void raid_gen2_sse2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen2_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_genz_int32(int nd, size_t size, void **vv);
|
||||
void raid_genz_int64(int nd, size_t size, void **vv);
|
||||
void raid_genz_sse2(int nd, size_t size, void **vv);
|
||||
|
|
@ -146,21 +151,25 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv);
|
|||
void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen4_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen4_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen4_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen5_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen5_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen5_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_gen6_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen6_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512bw(int nd, size_t size, void **vv);
|
||||
void raid_gen6_avx512gfni(int nd, size_t size, void **vv);
|
||||
void raid_rec1_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_int8(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
|
|
@ -173,6 +182,9 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
|||
void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec1_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
|
||||
|
||||
/*
|
||||
* Functions for parity computation.
|
||||
|
|
@ -273,6 +285,11 @@ void raid_register_int(void);
|
|||
*/
|
||||
void raid_register_x86(void);
|
||||
|
||||
/**
|
||||
* Register all the functions based on GFNI intructions.
|
||||
*/
|
||||
void raid_register_avx512gfni(void);
|
||||
|
||||
/*
|
||||
* Tag functions.
|
||||
*
|
||||
|
|
@ -298,6 +315,7 @@ extern const uint8_t raid_gfvandermonde[3][256] __aligned(256);
|
|||
extern const uint8_t raid_gfcauchy[6][256] __aligned(256);
|
||||
extern const uint8_t raid_gfcauchypshufb[251][4][2][16] __aligned(256);
|
||||
extern const uint8_t raid_gfmulpshufb[256][2][16] __aligned(256);
|
||||
extern const uint8_t raid_gfcauchycoeff[251][4][16] __aligned(256);
|
||||
extern const uint8_t (*raid_gfgen)[256];
|
||||
#define gfmul raid_gfmul
|
||||
#define gfexp raid_gfexp
|
||||
|
|
@ -306,6 +324,7 @@ extern const uint8_t (*raid_gfgen)[256];
|
|||
#define gfcauchy raid_gfcauchy
|
||||
#define gfgenpshufb raid_gfcauchypshufb
|
||||
#define gfmulpshufb raid_gfmulpshufb
|
||||
#define gfgencoeff raid_gfcauchycoeff
|
||||
#define gfgen raid_gfgen
|
||||
|
||||
/*
|
||||
|
|
|
|||
|
|
@ -4,6 +4,8 @@
|
|||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
|
||||
#include "raid.h" /* for RAID_POLY */
|
||||
|
||||
/**
|
||||
* Multiplication a*b in GF(2^8).
|
||||
*/
|
||||
|
|
@ -18,7 +20,7 @@ static uint8_t gfmul(uint8_t a, uint8_t b)
|
|||
|
||||
if ((a & 0x80) != 0) {
|
||||
a <<= 1;
|
||||
a ^= 0x1d;
|
||||
a ^= RAID_POLY;
|
||||
} else {
|
||||
a <<= 1;
|
||||
}
|
||||
|
|
@ -213,9 +215,7 @@ int main(void)
|
|||
uint8_t matrix[PARITY * 256];
|
||||
|
||||
printf("// SPDX-License-Identifier: GPL-2.0-or-later\n");
|
||||
printf("/*\n");
|
||||
printf(" * Copyright (C) 2013 Andrea Mazzoleni\n");
|
||||
printf(" */\n");
|
||||
printf("// Copyright (C) 2013 Andrea Mazzoleni\n");
|
||||
printf("\n");
|
||||
|
||||
printf("#include \"internal.h\"\n");
|
||||
|
|
|
|||
|
|
@ -247,6 +247,9 @@ void raid_init(void)
|
|||
#if defined(CONFIG_X86) && defined(CONFIG_SSE2)
|
||||
raid_register_x86();
|
||||
#endif
|
||||
#if defined(CONFIG_X86_64) && defined(CONFIG_AVX512GFNI)
|
||||
raid_register_avx512gfni();
|
||||
#endif
|
||||
|
||||
/* set the default mode */
|
||||
raid_mode(RAID_MODE_CAUCHY);
|
||||
|
|
|
|||
42
raid/raid.h
42
raid/raid.h
|
|
@ -4,6 +4,47 @@
|
|||
#ifndef __RAID_H
|
||||
#define __RAID_H
|
||||
|
||||
/*
|
||||
* GF(2^8) reducing polynomial selection.
|
||||
*
|
||||
* RAID parity computation operates in GF(2^8), a finite field where
|
||||
* arithmetic is performed modulo an irreducible polynomial of degree 8.
|
||||
* The choice of polynomial affects both correctness and performance:
|
||||
* all tables, coefficients, and parity data are tied to a specific
|
||||
* polynomial and cannot be mixed.
|
||||
*
|
||||
* Two polynomials are supported:
|
||||
*
|
||||
* 0x1d (x^8 + x^4 + x^3 + x^2 + 1) -- Standard RAID polynomial
|
||||
*
|
||||
* The polynomial used by the original RAID-6 specification and the
|
||||
* Linux kernel RAID implementation. Choosing this polynomial ensures
|
||||
* on-disk compatibility with existing RAID arrays and interchange with
|
||||
* other RAID implementations. Use this when reading or writing data
|
||||
* that may be accessed by other software or hardware RAID controllers.
|
||||
*
|
||||
* 0x1b (x^8 + x^4 + x^3 + x + 1) -- AES polynomial (USE_RAID_AES)
|
||||
*
|
||||
* The polynomial used by the AES encryption standard. Choosing this
|
||||
* polynomial enables the use of the Intel GFNI (Galois Field New
|
||||
* Instructions) extension, specifically the vgf2p8mulb instruction,
|
||||
* which performs GF(2^8) multiplication natively in hardware using
|
||||
* exactly this polynomial. This eliminates the nibble-split pshufb
|
||||
* multiply sequence (vpsrlw + vpandq + 2x vpshufb + vpxorq) and the
|
||||
* conditional multiply-by-2 sequence (vpcmpgtb + vpmovm2b + vpaddb +
|
||||
* vpandq + vpxorq) used in the AVX-512BW path, replacing each with a
|
||||
* single vgf2p8mulb instruction. On CPUs supporting GFNI (Ice Lake and
|
||||
* later), this yields a significant throughput improvement. The
|
||||
* trade-off is that parity data is not compatible with arrays using
|
||||
* the standard 0x1d polynomial, so this option requires that all
|
||||
* data be generated and recovered using the same polynomial choice.
|
||||
*/
|
||||
#ifdef USE_RAID_AES
|
||||
#define RAID_POLY 0x1b
|
||||
#else
|
||||
#define RAID_POLY 0x1d
|
||||
#endif
|
||||
|
||||
/**
|
||||
* RAID mode supporting up to 6 parities.
|
||||
*
|
||||
|
|
@ -215,4 +256,3 @@ int raid_check(int nr, int *ir, int nd, int np, size_t size, void **v);
|
|||
int raid_scan(int *ir, int nd, int np, size_t size, void **v);
|
||||
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -8749,7 +8749,7 @@ const uint8_t __aligned(256) raid_gfexp[256] =
|
|||
|
||||
const uint8_t __aligned(256) raid_gfinv[256] =
|
||||
{
|
||||
/* note that the first element is not significative */
|
||||
/* note that the first element is not significant */
|
||||
0x00, 0x01, 0x8e, 0xf4, 0x47, 0xa7, 0x7a, 0xba,
|
||||
0xad, 0x9d, 0xdd, 0x98, 0x3d, 0xaa, 0x5d, 0x96,
|
||||
0xd8, 0x72, 0xc0, 0x58, 0xe0, 0x3e, 0x4c, 0x66,
|
||||
|
|
@ -8786,7 +8786,7 @@ const uint8_t __aligned(256) raid_gfinv[256] =
|
|||
|
||||
/**
|
||||
* Power matrix used to generate parity.
|
||||
* This matrix is valid for up to 3 parity with 251 data disks.
|
||||
* This matrix is valid for up to 3 parities with 251 data disks.
|
||||
*
|
||||
* 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01 01
|
||||
* 01 02 04 08 10 20 40 80 1d 3a 74 e8 cd 87 13 26 4c 98 2d 5a b4 75 ea c9 8f 03 06 0c 18 30 60 c0 9d 27 4e 9c 25 4a 94 35 6a d4 b5 77 ee c1 9f 23 46 8c 05 0a 14 28 50 a0 5d ba 69 d2 b9 6f de a1 5f be 61 c2 99 2f 5e bc 65 ca 89 0f 1e 3c 78 f0 fd e7 d3 bb 6b d6 b1 7f fe e1 df a3 5b b6 71 e2 d9 af 43 86 11 22 44 88 0d 1a 34 68 d0 bd 67 ce 81 1f 3e 7c f8 ed c7 93 3b 76 ec c5 97 33 66 cc 85 17 2e 5c b8 6d da a9 4f 9e 21 42 84 15 2a 54 a8 4d 9a 29 52 a4 55 aa 49 92 39 72 e4 d5 b7 73 e6 d1 bf 63 c6 91 3f 7e fc e5 d7 b3 7b f6 f1 ff e3 db ab 4b 96 31 62 c4 95 37 6e dc a5 57 ae 41 82 19 32 64 c8 8d 07 0e 1c 38 70 e0 dd a7 53 a6 51 a2 59 b2 79 f2 f9 ef c3 9b 2b 56 ac 45 8a 09 12 24 48 90 3d 7a f4 f5 f7 f3 fb eb cb 8b 0b 16 2c 58 b0 7d fa e9 cf 83 1b 36 6c
|
||||
|
|
|
|||
|
|
@ -24,7 +24,7 @@ CFLAGS += -O2
|
|||
else
|
||||
CFLAGS += -O0 --coverage -DCOVERAGE=1 -DNDEBUG=1
|
||||
endif
|
||||
OBJS = raid.o check.o int.o intz.o x86.o x86z.o tables.o memory.o test.o helper.o module.o
|
||||
OBJS = raid.o check.o int.o intz.o x86.o x86z.o gfni.o tables.o memory.o test.o helper.o module.o
|
||||
|
||||
%.o: ../%.c
|
||||
$(CC) $(CFLAGS) -c -o $@ $<
|
||||
|
|
|
|||
|
|
@ -39,6 +39,8 @@ int main(void)
|
|||
printf("Including x86 AVX2 functions\n");
|
||||
if (raid_cpu_has_avx512bw())
|
||||
printf("Including x86 AVX512BW functions\n");
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
printf("Including x86 AVX512GFNI functions\n");
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("Including x64 extended SSE register set\n");
|
||||
|
|
|
|||
|
|
@ -138,6 +138,7 @@ void speed(void)
|
|||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx2e");
|
||||
printf("%8s", "avx512");
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -192,10 +193,11 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
printf("%8s", "");
|
||||
|
||||
SPEED_START {
|
||||
raid_gen1_avx512bw(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -263,10 +265,11 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
printf("%8s", "");
|
||||
|
||||
SPEED_START {
|
||||
raid_gen2_avx512bw(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
|
@ -275,6 +278,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen2_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -410,6 +423,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen3_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -485,6 +508,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen4_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -560,6 +593,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen5_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -635,6 +678,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen6_avx512gfni(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -650,6 +703,7 @@ void speed(void)
|
|||
printf("%8s", "avx2");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "avx512");
|
||||
printf("%8s", "gfni");
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
|
@ -693,7 +747,6 @@ void speed(void)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -706,6 +759,17 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(1, raid_gen1_avx512bw); /* there is no raid_gen1_avx512gfni */
|
||||
/* +1 to avoid GEN1 optimized case */
|
||||
raid_rec1_avx512gfni(1, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -752,7 +816,6 @@ void speed(void)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
|
|
@ -765,6 +828,17 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(2, raid_gen2_avx512gfni);
|
||||
/* +1 to avoid GEN2 optimized case */
|
||||
raid_rec2_avx512gfni(2, id, ip + 1, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -808,8 +882,7 @@ void speed(void)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX2
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx512bw);
|
||||
|
|
@ -820,6 +893,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(3, raid_gen3_avx512gfni);
|
||||
raid_recX_avx512gfni(3, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -867,8 +950,7 @@ void speed(void)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX2
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx512bw);
|
||||
|
|
@ -879,6 +961,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(4, raid_gen4_avx512gfni);
|
||||
raid_recX_avx512gfni(4, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -926,8 +1018,7 @@ void speed(void)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX2
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx512bw);
|
||||
|
|
@ -938,6 +1029,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(5, raid_gen5_avx512gfni);
|
||||
raid_recX_avx512gfni(5, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -985,8 +1086,7 @@ void speed(void)
|
|||
}
|
||||
#endif
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
#ifdef CONFIG_AVX2
|
||||
#ifdef CONFIG_AVX512BW
|
||||
if (raid_cpu_has_avx512bw()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx512bw);
|
||||
|
|
@ -997,6 +1097,16 @@ void speed(void)
|
|||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#ifdef CONFIG_AVX512GFNI
|
||||
if (raid_cpu_has_avx512gfni()) {
|
||||
SPEED_START {
|
||||
raid_gen_force(6, raid_gen6_avx512gfni);
|
||||
raid_recX_avx512gfni(6, id, ip, nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8" PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
printf("\n");
|
||||
printf("\n");
|
||||
|
|
@ -1019,7 +1129,9 @@ int main(void)
|
|||
if (raid_cpu_has_avx2())
|
||||
printf("Including x86 AVX2 functions\n");
|
||||
if (raid_cpu_has_avx512bw())
|
||||
printf("Including x86 AVX512bw functions\n");
|
||||
printf("Including x86 AVX512BW functions\n");
|
||||
if (raid_cpu_has_avx512gfni())
|
||||
printf("Including x86 AVX512GFNI functions\n");
|
||||
#endif
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("Including x64 extended SSE register set\n");
|
||||
|
|
|
|||
63
raid/x86.c
63
raid/x86.c
|
|
@ -35,6 +35,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
|
|||
|
||||
raid_sse_begin();
|
||||
|
||||
/* additional unrolling makes it slower */
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("movdqa %0,%%xmm0" : : "m" (v[l][i]));
|
||||
asm volatile ("movdqa %0,%%xmm1" : : "m" (v[l][i + 16]));
|
||||
|
|
@ -76,6 +77,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
/* additional unrolling makes it slower */
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (v[l][i + 32]));
|
||||
|
|
@ -115,6 +117,7 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
/* additional unrolling makes it slower */
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (v[l][i]));
|
||||
for (d = l - 1; d >= 0; --d) {
|
||||
|
|
@ -131,10 +134,10 @@ void raid_gen1_avx512bw(int nd, size_t size, void **vv)
|
|||
static const struct gfconst16 {
|
||||
uint8_t poly[16];
|
||||
uint8_t low4[16];
|
||||
} gfconst16 __aligned(32) = {
|
||||
} gfconst16 __aligned(16) = {
|
||||
{
|
||||
0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d,
|
||||
0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d
|
||||
RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY,
|
||||
RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY
|
||||
},
|
||||
{
|
||||
0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f,
|
||||
|
|
@ -215,7 +218,7 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0, %%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0, %%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
|
|
@ -268,7 +271,7 @@ void raid_gen2_avx512bw(int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti32x4 %0, %%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0, %%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
|
|
@ -641,8 +644,8 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
|
|||
raid_avx_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vbroadcasti128 %0, %%ymm3" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0, %%ymm11" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0, %%ymm3" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0, %%ymm11" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -773,8 +776,8 @@ void raid_gen3_avx512bw(int nd, size_t size, void **vv)
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -1163,7 +1166,7 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
|
|||
/* generic case with at least two data disks */
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[l][i + 32]));
|
||||
|
||||
|
|
@ -1199,8 +1202,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
|
|||
|
||||
/* intermediate disks */
|
||||
for (d = l - 1; d > 0; --d) {
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
|
||||
|
||||
|
|
@ -1251,8 +1254,8 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
|
|||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
|
||||
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
|
||||
|
||||
|
|
@ -1321,8 +1324,8 @@ void raid_gen4_avx512bw(int nd, size_t size, void **vv)
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -1708,8 +1711,8 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -1836,8 +1839,8 @@ void raid_gen5_avx512bw(int nd, size_t size, void **vv)
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -2289,8 +2292,8 @@ void raid_gen6_avx2ext(int nd, size_t size, void **vv)
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxor %ymm6,%ymm6,%ymm6");
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -2445,8 +2448,8 @@ void raid_gen6_avx512bw(int nd, size_t size, void **vv)
|
|||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile ("vpxorq %zmm6,%zmm6,%zmm6");
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm14" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm15" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
|
|
@ -2833,7 +2836,7 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm4" : : "m" (gfmulpshufb[V][0][0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm5" : : "m" (gfmulpshufb[V][1][0]));
|
||||
|
||||
|
|
@ -2889,7 +2892,7 @@ void raid_rec2_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[0][i]));
|
||||
|
|
@ -2987,7 +2990,7 @@ void raid_recX_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 32) {
|
||||
/* delta */
|
||||
|
|
@ -3055,7 +3058,7 @@ void raid_rec1_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm4" : : "m" (gfmulpshufb[V][0][0]));
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm5" : : "m" (gfmulpshufb[V][1][0]));
|
||||
|
||||
|
|
@ -3105,7 +3108,7 @@ void raid_rec2_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile ("vmovdqa64 %0,%%zmm0" : : "m" (p[0][i]));
|
||||
|
|
@ -3201,7 +3204,7 @@ void raid_recX_avx512bw(int nr, int *id, int *ip, int nd, size_t size, void **vv
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti32x4 %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%zmm7" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* delta */
|
||||
|
|
|
|||
17
raid/x86z.c
17
raid/x86z.c
|
|
@ -2,21 +2,22 @@
|
|||
// Copyright (C) 2013 Andrea Mazzoleni
|
||||
|
||||
#include "internal.h"
|
||||
#include "gf.h"
|
||||
|
||||
#if defined(CONFIG_X86) && defined(CONFIG_SSE2)
|
||||
static const struct gfzconst16 {
|
||||
uint8_t poly[16];
|
||||
uint8_t half[16];
|
||||
uint8_t low7[16];
|
||||
} gfzconst16 __aligned(64) =
|
||||
} gfzconst16 __aligned(16) =
|
||||
{
|
||||
{
|
||||
0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d,
|
||||
0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d
|
||||
RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY,
|
||||
RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY, RAID_POLY
|
||||
},
|
||||
{
|
||||
0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e,
|
||||
0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e
|
||||
RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE,
|
||||
RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE, RAID_INV2_BYTE
|
||||
},
|
||||
{
|
||||
0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
|
||||
|
|
@ -187,9 +188,9 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv)
|
|||
|
||||
raid_avx_begin();
|
||||
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm7" : : "m" (gfzconst16.poly[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm3" : : "m" (gfzconst16.half[0]));
|
||||
asm volatile ("vbroadcasti128 %0,%%ymm11" : : "m" (gfzconst16.low7[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm7" : : "m" (gfzconst16.poly[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm3" : : "m" (gfzconst16.half[0]));
|
||||
asm volatile ("vpbroadcastb %0,%%ymm11" : : "m" (gfzconst16.low7[0]));
|
||||
asm volatile ("vpxor %ymm15,%ymm15,%ymm15");
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue