rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 19:01:57 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use primitive generator 3 for AES Cauchy mode
Some checks are pending
Build / build-mac (push) Waiting to run
Details
Build / build-linux (push) Waiting to run
Details
Build / build-win (push) Waiting to run
Details
CodeQL Advanced / Analyze (c-cpp) (push) Waiting to run
Details
CodeQL Advanced / Analyze (actions) (push) Waiting to run
Details

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-19 01:36:18 +02:00
parent 15831064a1
commit d997fe43a0
11 changed files with 5454 additions and 5034 deletions
Download patch file Download diff file Expand all files Collapse all files

139
raid/avx2.c
View file

@ -41,7 +41,7 @@ void raid_gen1_avx2(int nd, size_t size, void **vv)
/*
* GEN2 (RAID6 with powers of 2) AVX2 implementation
*/
void raid_gen2_avx2(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_avx2_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -66,14 +66,30 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
for (d = l - 1; d >= 0; --d) {
/* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */
asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4");
asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
if (generator == 3) {
/* ymm4 preserves one Q lane while ymm5 forms its reduction mask. */
asm volatile ("vmovdqa %ymm2,%ymm4");
asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm5");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm5,%ymm2,%ymm2");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vmovdqa %ymm3,%ymm4");
asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
asm volatile ("vpxor %ymm4,%ymm3,%ymm3");
} else {
asm volatile ("vpcmpgtb %ymm2,%ymm6,%ymm4");
asm volatile ("vpcmpgtb %ymm3,%ymm6,%ymm5");
asm volatile ("vpaddb %ymm2,%ymm2,%ymm2");
asm volatile ("vpaddb %ymm3,%ymm3,%ymm3");
asm volatile ("vpand %ymm7,%ymm4,%ymm4");
asm volatile ("vpand %ymm7,%ymm5,%ymm5");
asm volatile ("vpxor %ymm4,%ymm2,%ymm2");
asm volatile ("vpxor %ymm5,%ymm3,%ymm3");
}
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm5" : : "m" (v[d][i + 32]));
@ -92,6 +108,16 @@ void raid_gen2_avx2(int nd, size_t size, void **vv)
raid_avx_end();
}
void raid_gen2_avx2(int nd, size_t size, void **vv)
{
raid_gen2_avx2_gen(nd, size, vv, 2);
}
void raid_gen2_avx2aes(int nd, size_t size, void **vv)
{
raid_gen2_avx2_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GENz (triple parity with powers of 2^-1) AVX2 implementation
@ -175,7 +201,7 @@ void raid_genz_avx2ext(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen3_avx2ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen3_avx2ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -233,6 +259,10 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
if (generator == 3) {
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vmovdqa %ymm9,%ymm14");
}
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
@ -243,6 +273,10 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3) {
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
}
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
@ -272,6 +306,10 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
if (generator == 3) {
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vmovdqa %ymm9,%ymm14");
}
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
@ -282,6 +320,10 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm3,%ymm13,%ymm13");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3) {
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
}
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
@ -300,6 +342,16 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
raid_avx_end();
}
void raid_gen3_avx2ext(int nd, size_t size, void **vv)
{
raid_gen3_avx2ext_gen(nd, size, vv, 2);
}
void raid_gen3_avx2extaes(int nd, size_t size, void **vv)
{
raid_gen3_avx2ext_gen(nd, size, vv, 3);
}
#endif
#ifdef CONFIG_X86_64
@ -308,7 +360,7 @@ void raid_gen3_avx2ext(int nd, size_t size, void **vv)
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen4_avx2ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen4_avx2ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -377,6 +429,10 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[d][i + 32]));
if (generator == 3) {
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vmovdqa %ymm9,%ymm14");
}
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
@ -387,6 +443,10 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3) {
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
}
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
@ -429,6 +489,10 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
asm volatile ("vmovdqa %0,%%ymm12" : : "m" (v[0][i + 32]));
if (generator == 3) {
asm volatile ("vmovdqa %ymm1,%ymm6");
asm volatile ("vmovdqa %ymm9,%ymm14");
}
asm volatile ("vpxor %ymm5,%ymm5,%ymm5");
asm volatile ("vpxor %ymm13,%ymm13,%ymm13");
asm volatile ("vpcmpgtb %ymm1,%ymm5,%ymm5");
@ -439,6 +503,10 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
asm volatile ("vpand %ymm7,%ymm13,%ymm13");
asm volatile ("vpxor %ymm5,%ymm1,%ymm1");
asm volatile ("vpxor %ymm13,%ymm9,%ymm9");
if (generator == 3) {
asm volatile ("vpxor %ymm6,%ymm1,%ymm1");
asm volatile ("vpxor %ymm14,%ymm9,%ymm9");
}
asm volatile ("vpxor %ymm4,%ymm0,%ymm0");
asm volatile ("vpxor %ymm4,%ymm1,%ymm1");
@ -461,13 +529,23 @@ void raid_gen4_avx2ext(int nd, size_t size, void **vv)
raid_avx_end();
}
void raid_gen4_avx2ext(int nd, size_t size, void **vv)
{
raid_gen4_avx2ext_gen(nd, size, vv, 2);
}
void raid_gen4_avx2extaes(int nd, size_t size, void **vv)
{
raid_gen4_avx2ext_gen(nd, size, vv, 3);
}
#endif
#ifdef CONFIG_X86_64
/*
* GENX AVX2EXT implementation
*/
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -536,10 +614,14 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
for (d = l - 1; d > 0; --d) {
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm12");
asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpand %ymm14,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm1,%ymm1");
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
asm volatile ("vpxor %ymm10,%ymm1,%ymm1");
@ -588,10 +670,14 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
/* first disk with all coefficients at 1 */
asm volatile ("vmovdqa %0,%%ymm10" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("vmovdqa %ymm1,%ymm12");
asm volatile ("vpcmpgtb %ymm1,%ymm6,%ymm11");
asm volatile ("vpaddb %ymm1,%ymm1,%ymm1");
asm volatile ("vpand %ymm14,%ymm11,%ymm11");
asm volatile ("vpxor %ymm11,%ymm1,%ymm1");
if (generator == 3)
asm volatile ("vpxor %ymm12,%ymm1,%ymm1");
asm volatile ("vpxor %ymm10,%ymm0,%ymm0");
asm volatile ("vpxor %ymm10,%ymm1,%ymm1");
@ -628,7 +714,12 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
*/
void raid_gen5_avx2ext(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 5);
raid_genX_avx2ext(nd, size, vv, 5, 2);
}
void raid_gen5_avx2extaes(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 5, 3);
}
#endif
@ -640,7 +731,12 @@ void raid_gen5_avx2ext(int nd, size_t size, void **vv)
*/
void raid_gen6_avx2ext(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 6);
raid_genX_avx2ext(nd, size, vv, 6, 2);
}
void raid_gen6_avx2extaes(int nd, size_t size, void **vv)
{
raid_genX_avx2ext(nd, size, vv, 6, 3);
}
#endif
@ -871,12 +967,17 @@ void raid_register_avx2(void)
{
if (raid_cpu_has_avx2()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "avx2", raid_gen1_avx2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "avx2", raid_gen2_avx2aes, RAID_POLY_AES);
#ifdef CONFIG_X86_64
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "avx2e", raid_gen3_avx2extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "avx2e", raid_gen4_avx2extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "avx2e", raid_gen5_avx2extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "avx2e", raid_gen6_avx2extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "avx2e", raid_genz_avx2ext, RAID_POLY_ANY);
#endif

4
raid/gf.h
View file

@ -30,9 +30,9 @@ static __always_inline uint8_t inv(uint8_t v)
}
/*
* GF 2^a.
* GF generator^a for the active Cauchy matrix.
*/
static __always_inline uint8_t pow2(int v)
static __always_inline uint8_t powgen(int v)
{
BUG_ON(v < 0 || v > 254); /* invalid exponent */

17
raid/int.c
View file

@ -144,6 +144,18 @@ void raid_gen2_int64(int nd, size_t size, void **vv)
}
}
/*
* GEN2 AES scalar fallback.
*
* Keep the unrolled RAID scalar loops unchanged: retaining the old Q for
* generator 3 makes the 64-bit version spill in the hot loop. The reference
* generator is table-driven and therefore handles the active AES matrix.
*/
void raid_gen2_int8aes(int nd, size_t size, void **vv)
{
raid_gen_ref(nd, 2, size, vv);
}
/*
* GENz (triple parity with powers of 2^-1) 32bit C implementation
*/
@ -655,13 +667,14 @@ void raid_register_int(void)
{
if (sizeof(void *) == 4) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int32", raid_gen1_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int32", raid_gen2_int32, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int32", raid_genz_int32, RAID_POLY_ANY);
} else {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "int64", raid_gen1_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int64", raid_gen2_int64, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "int64", raid_genz_int64, RAID_POLY_ANY);
}
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "int8", raid_gen2_int8aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "int8", raid_gen3_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "int8", raid_gen4_int8, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "int8", raid_gen5_int8, RAID_POLY_ANY);

19
raid/internal.h
View file

@ -92,9 +92,13 @@ void raid_gen1_avx2(int nd, size_t size, void **vv);
void raid_gen1_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_int32(int nd, size_t size, void **vv);
void raid_gen2_int64(int nd, size_t size, void **vv);
void raid_gen2_int8aes(int nd, size_t size, void **vv);
void raid_gen2_sse2(int nd, size_t size, void **vv);
void raid_gen2_sse2aes(int nd, size_t size, void **vv);
void raid_gen2_avx2(int nd, size_t size, void **vv);
void raid_gen2_avx2aes(int nd, size_t size, void **vv);
void raid_gen2_sse2ext(int nd, size_t size, void **vv);
void raid_gen2_sse2extaes(int nd, size_t size, void **vv);
void raid_gen2_avx512bw(int nd, size_t size, void **vv);
void raid_gen2_avx2gfni(int nd, size_t size, void **vv);
void raid_gen2_avx512gfni(int nd, size_t size, void **vv);
@ -105,29 +109,39 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv);
void raid_genz_avx2ext(int nd, size_t size, void **vv);
void raid_gen3_int8(int nd, size_t size, void **vv);
void raid_gen3_ssse3(int nd, size_t size, void **vv);
void raid_gen3_ssse3aes(int nd, size_t size, void **vv);
void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
void raid_gen3_ssse3extaes(int nd, size_t size, void **vv);
void raid_gen3_avx2ext(int nd, size_t size, void **vv);
void raid_gen3_avx2extaes(int nd, size_t size, void **vv);
void raid_gen3_avx512bw(int nd, size_t size, void **vv);
void raid_gen3_avx2gfni(int nd, size_t size, void **vv);
void raid_gen3_avx512gfni(int nd, size_t size, void **vv);
void raid_gen4_int8(int nd, size_t size, void **vv);
void raid_gen4_ssse3(int nd, size_t size, void **vv);
void raid_gen4_ssse3aes(int nd, size_t size, void **vv);
void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
void raid_gen4_ssse3extaes(int nd, size_t size, void **vv);
void raid_gen4_avx2ext(int nd, size_t size, void **vv);
void raid_gen4_avx2extaes(int nd, size_t size, void **vv);
void raid_gen4_avx512bw(int nd, size_t size, void **vv);
void raid_gen4_avx2gfni(int nd, size_t size, void **vv);
void raid_gen4_avx512gfni(int nd, size_t size, void **vv);
void raid_gen5_int8(int nd, size_t size, void **vv);
void raid_gen5_ssse3(int nd, size_t size, void **vv);
void raid_gen5_ssse3ext(int nd, size_t size, void **vv);
void raid_gen5_ssse3extaes(int nd, size_t size, void **vv);
void raid_gen5_avx2ext(int nd, size_t size, void **vv);
void raid_gen5_avx2extaes(int nd, size_t size, void **vv);
void raid_gen5_avx512bw(int nd, size_t size, void **vv);
void raid_gen5_avx2gfni(int nd, size_t size, void **vv);
void raid_gen5_avx512gfni(int nd, size_t size, void **vv);
void raid_gen6_int8(int nd, size_t size, void **vv);
void raid_gen6_ssse3(int nd, size_t size, void **vv);
void raid_gen6_ssse3ext(int nd, size_t size, void **vv);
void raid_gen6_ssse3extaes(int nd, size_t size, void **vv);
void raid_gen6_avx2ext(int nd, size_t size, void **vv);
void raid_gen6_avx2extaes(int nd, size_t size, void **vv);
void raid_gen6_avx512bw(int nd, size_t size, void **vv);
void raid_gen6_avx2gfni(int nd, size_t size, void **vv);
void raid_gen6_avx512gfni(int nd, size_t size, void **vv);
@ -151,11 +165,16 @@ void raid_rec2_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **
void raid_recX_avx512gfni(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_gen1_neon(int nd, size_t size, void **vv);
void raid_gen2_neon(int nd, size_t size, void **vv);
void raid_gen2_neonaes(int nd, size_t size, void **vv);
void raid_genz_neon(int nd, size_t size, void **vv);
void raid_gen3_neon(int nd, size_t size, void **vv);
void raid_gen3_neonaes(int nd, size_t size, void **vv);
void raid_gen4_neon(int nd, size_t size, void **vv);
void raid_gen4_neonaes(int nd, size_t size, void **vv);
void raid_gen5_neon(int nd, size_t size, void **vv);
void raid_gen5_neonaes(int nd, size_t size, void **vv);
void raid_gen6_neon(int nd, size_t size, void **vv);
void raid_gen6_neonaes(int nd, size_t size, void **vv);
void raid_rec1_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_rec2_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);
void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv);

36
raid/mktables.c
View file

@ -58,7 +58,7 @@ uint8_t raid_gfinv[256];
/**
* Setup the Cauchy matrix used to generate the parity.
*/
static void set_cauchy(uint8_t poly, uint8_t *matrix)
static void set_cauchy(uint8_t poly, uint8_t generator, uint8_t *matrix)
{
int i, j;
uint8_t inv_x, y;
@ -83,18 +83,18 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
matrix[0 * DISK + i] = 1;
/*
* The second row is formed with powers 2^i, and it's the first
* The second row is formed with powers g^i, and it's the first
* row of the Cauchy matrix.
*
* Each element of the Cauchy matrix is in the form 1/(x_i + y_j)
* where all x_i and y_j must be different for any i and j.
*
* For the first row with j=0, we choose x_i = 2^-i and y_0 = 0
* For the first row with j=0, we choose x_i = g^-i and y_0 = 0
* and we obtain a first row formed as:
*
* 1/(x_i + y_0) = 1/(2^-i + 0) = 2^i
* 1/(x_i + y_0) = 1/(g^-i + 0) = g^i
*
* with 2^-i != 0 for any i
* with g^-i != 0 for any i
*
* In the example we get:
*
@ -116,17 +116,17 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
inv_x = 1;
for (i = 0; i < DISK; ++i) {
matrix[1 * DISK + i] = inv_x;
inv_x = raid_gfmul(poly, 2, inv_x);
inv_x = raid_gfmul(poly, generator, inv_x);
}
/*
* The rest of the Cauchy matrix is formed by choosing for each row j
* a new y_j = 2^j and reusing the x_i already assigned in the first
* a new y_j = g^j and reusing the x_i already assigned in the first
* row obtaining :
*
* 1/(x_i + y_j) = 1/(2^-i + 2^j)
* 1/(x_i + y_j) = 1/(g^-i + g^j)
*
* with 2^-i + 2^j != 0 for any i,j with i>=0,j>=1,i+j<255
* with g^-i + g^j != 0 for any i,j with i>=0,j>=1,i+j<255
*
* In the example we get:
*
@ -140,17 +140,17 @@ static void set_cauchy(uint8_t poly, uint8_t *matrix)
* 244 83 78 183 118 47
* 167 39 213 59 153 82
*/
y = 2;
y = generator;
for (j = 0; j < PARITY - 2; ++j) {
inv_x = 1;
for (i = 0; i < DISK; ++i) {
uint8_t x = raid_gfinv[inv_x];
matrix[(j + 2) * DISK + i] = raid_gfinv[y ^ x];
inv_x = raid_gfmul(poly, 2, inv_x);
inv_x = raid_gfmul(poly, generator, inv_x);
}
y = raid_gfmul(poly, 2, y);
y = raid_gfmul(poly, generator, y);
}
/*
@ -200,7 +200,7 @@ static void set_power(uint8_t poly, uint8_t *matrix)
}
}
void tables(uint8_t poly, const char *tag)
void tables(uint8_t poly, uint8_t generator, const char *tag)
{
uint8_t v;
int i, j, k, p;
@ -227,7 +227,7 @@ void tables(uint8_t poly, const char *tag)
}
printf("};\n\n");
/* 2^a */
/* generator^a */
printf("const uint8_t __aligned(256) raid_gfexp_%s[256] =\n", tag);
printf("{\n");
v = 1;
@ -235,7 +235,7 @@ void tables(uint8_t poly, const char *tag)
if (i % 8 == 0)
printf("\t");
printf("0x%02x,", v);
v = raid_gfmul(poly, v, 2);
v = raid_gfmul(poly, v, generator);
if (i % 8 == 7)
printf("\n");
else
@ -263,7 +263,7 @@ void tables(uint8_t poly, const char *tag)
printf("};\n\n");
/* cauchy matrix */
set_cauchy(poly, matrix);
set_cauchy(poly, generator, matrix);
printf("/**\n");
printf(" * Cauchy matrix used to generate parity.\n");
@ -403,10 +403,10 @@ int main(void)
printf("\n");
printf("/* Tables with the RAID Polynomial 0x1d */\n");
tables(RAID_POLY_RAID, "raid");
tables(RAID_POLY_RAID, 2, "raid");
printf("/* Tables with the AES Polynomial 0x1b */\n");
tables(RAID_POLY_AES, "aes");
tables(RAID_POLY_AES, 3, "aes");
return 0;
}

94
raid/neon.c
View file

@ -62,7 +62,7 @@ void raid_gen1_neon(int nd, size_t size, void **vv)
/*
* GEN2 (RAID6 with powers of 2) NEON implementation
*/
void raid_gen2_neon(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_neon_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -93,6 +93,13 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
);
for (d = l - 1; d >= 0; --d) {
if (generator == 3) {
/* v14-v15 are otherwise unused and preserve Q across the AES xtime. */
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v16.16b, v2.16b, #7\n"
@ -115,6 +122,12 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
}
asm volatile (
@ -130,6 +143,16 @@ void raid_gen2_neon(int nd, size_t size, void **vv)
raid_neon_end();
}
void raid_gen2_neon(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 2);
}
void raid_gen2_neonaes(int nd, size_t size, void **vv)
{
raid_gen2_neon_gen(nd, size, vv, 3);
}
/*
* GENz (triple parity with powers of 2^-1) NEON implementation
*/
@ -227,7 +250,7 @@ void raid_genz_neon(int nd, size_t size, void **vv)
/*
* GENX NEON implementation
*/
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
size_t i;
@ -336,6 +359,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
/* intermediate disks */
for (d = l - 1; d > 0; --d) {
if (generator == 3) {
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v20.16b, v2.16b, #7\n"
@ -358,6 +387,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
:
: "m" (v[d][i]), "m" (v[d][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
if (np >= 3) {
asm volatile (
@ -440,6 +475,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
}
/* first disk with all coefficients at 1 */
if (generator == 3) {
asm volatile (
"mov v14.16b, v2.16b\n"
"mov v15.16b, v3.16b\n"
);
}
asm volatile (
/* double Q0-Q1 */
"sshr v20.16b, v2.16b, #7\n"
@ -462,6 +503,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
:
: "m" (v[0][i]), "m" (v[0][i + 16])
);
if (generator == 3) {
asm volatile (
"eor v2.16b, v2.16b, v14.16b\n"
"eor v3.16b, v3.16b, v15.16b\n"
);
}
if (np >= 3) {
asm volatile (
@ -535,7 +582,12 @@ static __always_inline void raid_genX_neon(int nd, size_t size, void **vv, int n
*/
void raid_gen3_neon(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 3);
raid_genX_neon(nd, size, vv, 3, 2);
}
void raid_gen3_neonaes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 3, 3);
}
/*
@ -543,7 +595,12 @@ void raid_gen3_neon(int nd, size_t size, void **vv)
*/
void raid_gen4_neon(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 4);
raid_genX_neon(nd, size, vv, 4, 2);
}
void raid_gen4_neonaes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 4, 3);
}
/*
@ -551,7 +608,12 @@ void raid_gen4_neon(int nd, size_t size, void **vv)
*/
void raid_gen5_neon(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 5);
raid_genX_neon(nd, size, vv, 5, 2);
}
void raid_gen5_neonaes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 5, 3);
}
/*
@ -559,7 +621,12 @@ void raid_gen5_neon(int nd, size_t size, void **vv)
*/
void raid_gen6_neon(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 6);
raid_genX_neon(nd, size, vv, 6, 2);
}
void raid_gen6_neonaes(int nd, size_t size, void **vv)
{
raid_genX_neon(nd, size, vv, 6, 3);
}
/*
@ -811,12 +878,17 @@ void raid_recX_neon(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_neon(void)
{
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_gen1_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neon, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_gen2_neonaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "neon", raid_genz_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neon, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "neon", raid_gen3_neonaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neon, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "neon", raid_gen4_neonaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neon, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "neon", raid_gen5_neonaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neon, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "neon", raid_gen6_neonaes, RAID_POLY_AES);
raid_rec_register(RAID_ALGO_CAUCHY_PAR1, "neon", raid_rec1_neon, RAID_POLY_ANY);
raid_rec_register(RAID_ALGO_CAUCHY_PAR2, "neon", raid_rec2_neon, RAID_POLY_ANY);

14
raid/raid.c
View file

@ -398,20 +398,20 @@ void raid_rec1of1(int *id, int nd, size_t size, void **v)
* Starting from the equations:
*
* Pd = Dx + Dy
* Qd = 2^id[0] * Dx + 2^id[1] * Dy
* Qd = g^id[0] * Dx + g^id[1] * Dy
*
* and solving we get:
*
* 1 2^(-id[0])
* 1 g^(-id[0])
* Dy = ------------------- * Pd + ------------------- * Qd
* 2^(id[1]-id[0]) + 1 2^(id[1]-id[0]) + 1
* g^(id[1]-id[0]) + 1 g^(id[1]-id[0]) + 1
*
* Dx = Dy + Pd
*
* with conditions:
*
* 2^id[0] != 0
* 2^(id[1]-id[0]) + 1 != 0
* g^id[0] != 0
* g^(id[1]-id[0]) + 1 != 0
*
* That are always satisfied for any 0<=id[0]<id[1]<255.
*/
@ -426,8 +426,8 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv)
const uint8_t *T[2];
/* get multiplication tables */
T[0] = table(inv(pow2(id[1] - id[0]) ^ 1));
T[1] = table(inv(pow2(id[0]) ^ pow2(id[1])));
T[0] = table(inv(powgen(id[1] - id[0]) ^ 1));
T[1] = table(inv(powgen(id[0]) ^ powgen(id[1])));
/* compute delta parity */
raid_delta_gen(2, id, ip, nd, size, vv);

131
raid/sse2.c
View file

@ -138,7 +138,7 @@ void raid_gen1_sse2(int nd, size_t size, void **vv)
/*
* GEN2 (RAID6 with powers of 2) SSE2 implementation
*/
void raid_gen2_sse2(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_sse2_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -162,16 +162,34 @@ void raid_gen2_sse2(int nd, size_t size, void **vv)
for (d = l - 1; d >= 0; --d) {
/* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
if (generator == 3) {
/* xmm4-xmm5 preserve Q while the otherwise free xmm6 forms each reduction mask. */
asm volatile ("movdqa %xmm2,%xmm4");
asm volatile ("movdqa %xmm3,%xmm5");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm2,%xmm6");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pxor %xmm6,%xmm2");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm6,%xmm6");
asm volatile ("pcmpgtb %xmm3,%xmm6");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm6");
asm volatile ("pxor %xmm6,%xmm3");
asm volatile ("pxor %xmm5,%xmm3");
} else {
asm volatile ("pxor %xmm4,%xmm4");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm2,%xmm4");
asm volatile ("pcmpgtb %xmm3,%xmm5");
asm volatile ("paddb %xmm2,%xmm2");
asm volatile ("paddb %xmm3,%xmm3");
asm volatile ("pand %xmm7,%xmm4");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm4,%xmm2");
asm volatile ("pxor %xmm5,%xmm3");
}
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm5" : : "m" (v[d][i + 16]));
@ -190,13 +208,23 @@ void raid_gen2_sse2(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen2_sse2(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 2);
}
void raid_gen2_sse2aes(int nd, size_t size, void **vv)
{
raid_gen2_sse2_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN2 (RAID6 with powers of 2) SSE2 implementation
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen2_sse2ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen2_sse2ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -224,26 +252,46 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv)
for (d = l - 1; d >= 0; --d) {
/* scale by 2 before adding the current disk so the first (last processed) disk with factor 1 avoids doubling */
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
if (generator == 3) {
/* xmm8-xmm11 preserve Q; xmm12 is dead until the following data load. */
asm volatile ("movdqa %xmm4,%xmm8");
asm volatile ("movdqa %xmm5,%xmm9");
asm volatile ("movdqa %xmm6,%xmm10");
asm volatile ("movdqa %xmm7,%xmm11");
#define RAID_SSE2_X3(q, old) \
asm volatile ("pxor %xmm12,%xmm12"); \
asm volatile ("pcmpgtb %" q ",%xmm12"); \
asm volatile ("paddb %" q ",%" q); \
asm volatile ("pand %xmm15,%xmm12"); \
asm volatile ("pxor %xmm12,%" q); \
asm volatile ("pxor %" old ",%" q)
RAID_SSE2_X3("xmm4", "xmm8");
RAID_SSE2_X3("xmm5", "xmm9");
RAID_SSE2_X3("xmm6", "xmm10");
RAID_SSE2_X3("xmm7", "xmm11");
#undef RAID_SSE2_X3
} else {
asm volatile ("pxor %xmm8,%xmm8");
asm volatile ("pxor %xmm9,%xmm9");
asm volatile ("pxor %xmm10,%xmm10");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm4,%xmm8");
asm volatile ("pcmpgtb %xmm5,%xmm9");
asm volatile ("pcmpgtb %xmm6,%xmm10");
asm volatile ("pcmpgtb %xmm7,%xmm11");
asm volatile ("paddb %xmm4,%xmm4");
asm volatile ("paddb %xmm5,%xmm5");
asm volatile ("paddb %xmm6,%xmm6");
asm volatile ("paddb %xmm7,%xmm7");
asm volatile ("pand %xmm15,%xmm8");
asm volatile ("pand %xmm15,%xmm9");
asm volatile ("pand %xmm15,%xmm10");
asm volatile ("pand %xmm15,%xmm11");
asm volatile ("pxor %xmm8,%xmm4");
asm volatile ("pxor %xmm9,%xmm5");
asm volatile ("pxor %xmm10,%xmm6");
asm volatile ("pxor %xmm11,%xmm7");
}
asm volatile ("movdqa %0,%%xmm8" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm9" : : "m" (v[d][i + 16]));
@ -271,6 +319,16 @@ void raid_gen2_sse2ext(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen2_sse2ext(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 2);
}
void raid_gen2_sse2extaes(int nd, size_t size, void **vv)
{
raid_gen2_sse2ext_gen(nd, size, vv, 3);
}
#endif
/*
@ -415,10 +473,13 @@ void raid_register_sse2(void)
{
if (raid_cpu_has_sse2()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR1, "sse2", raid_gen1_sse2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2", raid_gen2_sse2aes, RAID_POLY_AES);
#ifdef CONFIG_X86_64
if (!raid_cpu_has_slow_extendedreg())
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext, RAID_POLY_ANY);
if (!raid_cpu_has_slow_extendedreg()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR2, "sse2e", raid_gen2_sse2extaes, RAID_POLY_AES);
}
/* note that raid_cpu_has_slow_extendedreg() doesn't affect vandermonde */
raid_gen_register(RAID_ALGO_VANDERMONDE_PAR3, "sse2e", raid_genz_sse2ext, RAID_POLY_ANY);
#else

142
raid/ssse3.c
View file

@ -9,7 +9,7 @@
/*
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
*/
void raid_gen3_ssse3(int nd, size_t size, void **vv)
static __always_inline void raid_gen3_ssse3_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -58,11 +58,15 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm3,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -83,11 +87,15 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm3,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -101,13 +109,23 @@ void raid_gen3_ssse3(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen3_ssse3(int nd, size_t size, void **vv)
{
raid_gen3_ssse3_gen(nd, size, vv, 2);
}
void raid_gen3_ssse3aes(int nd, size_t size, void **vv)
{
raid_gen3_ssse3_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN3 (triple parity with Cauchy matrix) SSSE3 implementation
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen3_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -169,6 +187,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -179,6 +201,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm3,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -212,6 +238,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -222,6 +252,10 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm3,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -240,12 +274,22 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
{
raid_gen3_ssse3ext_gen(nd, size, vv, 2);
}
void raid_gen3_ssse3extaes(int nd, size_t size, void **vv)
{
raid_gen3_ssse3ext_gen(nd, size, vv, 3);
}
#endif
/*
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
*/
void raid_gen4_ssse3(int nd, size_t size, void **vv)
static __always_inline void raid_gen4_ssse3_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -301,11 +345,15 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.low4[0]));
@ -336,11 +384,15 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm7" : : "m" (gfconst16.poly[0]));
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pcmpgtb %xmm1,%xmm5");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm7,%xmm5");
asm volatile ("pxor %xmm5,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -356,13 +408,23 @@ void raid_gen4_ssse3(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen4_ssse3(int nd, size_t size, void **vv)
{
raid_gen4_ssse3_gen(nd, size, vv, 2);
}
void raid_gen4_ssse3aes(int nd, size_t size, void **vv)
{
raid_gen4_ssse3_gen(nd, size, vv, 3);
}
#ifdef CONFIG_X86_64
/*
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
*
* Note that it uses 16 registers, meaning that x64 is required.
*/
void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
static __always_inline void raid_gen4_ssse3ext_gen(int nd, size_t size, void **vv, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -437,6 +499,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[d][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[d][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -447,6 +513,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -495,6 +565,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("movdqa %0,%%xmm4" : : "m" (v[0][i]));
asm volatile ("movdqa %0,%%xmm12" : : "m" (v[0][i + 16]));
if (generator == 3) {
asm volatile ("movdqa %xmm1,%xmm6");
asm volatile ("movdqa %xmm9,%xmm14");
}
asm volatile ("pxor %xmm5,%xmm5");
asm volatile ("pxor %xmm13,%xmm13");
asm volatile ("pcmpgtb %xmm1,%xmm5");
@ -505,6 +579,10 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
asm volatile ("pand %xmm7,%xmm13");
asm volatile ("pxor %xmm5,%xmm1");
asm volatile ("pxor %xmm13,%xmm9");
if (generator == 3) {
asm volatile ("pxor %xmm6,%xmm1");
asm volatile ("pxor %xmm14,%xmm9");
}
asm volatile ("pxor %xmm4,%xmm0");
asm volatile ("pxor %xmm4,%xmm1");
@ -527,6 +605,16 @@ void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
raid_sse_end();
}
void raid_gen4_ssse3ext(int nd, size_t size, void **vv)
{
raid_gen4_ssse3ext_gen(nd, size, vv, 2);
}
void raid_gen4_ssse3extaes(int nd, size_t size, void **vv)
{
raid_gen4_ssse3ext_gen(nd, size, vv, 3);
}
#endif
/*
@ -668,7 +756,7 @@ void raid_gen5_ssse3(int nd, size_t size, void **vv)
/*
* GENX SSSE3ext implementation
*/
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np)
static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, int np, int generator)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *p;
@ -750,11 +838,15 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
for (d = l - 1; d > 0; --d) {
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[d][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm12");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm1,%xmm11");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm14,%xmm11");
asm volatile ("pxor %xmm11,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm12,%xmm1");
asm volatile ("pxor %xmm10,%xmm0");
asm volatile ("pxor %xmm10,%xmm1");
@ -804,11 +896,15 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
/* first disk with all coefficients at 1 */
asm volatile ("movdqa %0,%%xmm10" : : "m" (v[0][i]));
if (generator == 3)
asm volatile ("movdqa %xmm1,%xmm12");
asm volatile ("pxor %xmm11,%xmm11");
asm volatile ("pcmpgtb %xmm1,%xmm11");
asm volatile ("paddb %xmm1,%xmm1");
asm volatile ("pand %xmm14,%xmm11");
asm volatile ("pxor %xmm11,%xmm1");
if (generator == 3)
asm volatile ("pxor %xmm12,%xmm1");
asm volatile ("pxor %xmm10,%xmm0");
asm volatile ("pxor %xmm10,%xmm1");
@ -843,7 +939,12 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
*/
void raid_gen5_ssse3ext(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 5);
raid_genX_ssse3ext(nd, size, vv, 5, 2);
}
void raid_gen5_ssse3extaes(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 5, 3);
}
#endif
@ -1011,7 +1112,12 @@ void raid_gen6_ssse3(int nd, size_t size, void **vv)
*/
void raid_gen6_ssse3ext(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 6);
raid_genX_ssse3ext(nd, size, vv, 6, 2);
}
void raid_gen6_ssse3extaes(int nd, size_t size, void **vv)
{
raid_genX_ssse3ext(nd, size, vv, 6, 3);
}
#endif
@ -1253,16 +1359,22 @@ void raid_recX_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
void raid_register_ssse3(void)
{
if (raid_cpu_has_ssse3()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3", raid_gen3_ssse3aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3", raid_gen4_ssse3aes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3", raid_gen5_ssse3, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3", raid_gen6_ssse3, RAID_POLY_RAID);
#ifdef CONFIG_X86_64
if (!raid_cpu_has_slow_extendedreg()) {
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext, RAID_POLY_ANY);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR3, "ssse3e", raid_gen3_ssse3extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR4, "ssse3e", raid_gen4_ssse3extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR5, "ssse3e", raid_gen5_ssse3extaes, RAID_POLY_AES);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3ext, RAID_POLY_RAID);
raid_gen_register(RAID_ALGO_CAUCHY_PAR6, "ssse3e", raid_gen6_ssse3extaes, RAID_POLY_AES);
}
#endif

9806
raid/tables.c
View file

File diff suppressed because it is too large Load diff

86
raid/test.c
View file

@ -374,25 +374,41 @@ int raid_test_par(int mode, int nd, size_t size)
/* setup all the available functions */
test_setup(1) = raid_gen1_int32;
test_setup(1) = raid_gen1_int64;
test_setup(2) = raid_gen2_int32;
test_setup(2) = raid_gen2_int64;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_int8aes;
else {
test_setup(2) = raid_gen2_int32;
test_setup(2) = raid_gen2_int64;
}
#ifdef CONFIG_NEON
test_setup(1) = raid_gen1_neon;
test_setup(2) = raid_gen2_neon;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_neonaes;
else
test_setup(2) = raid_gen2_neon;
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_sse2()) {
test_setup(1) = raid_gen1_sse2;
test_setup(2) = raid_gen2_sse2;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_sse2aes;
else
test_setup(2) = raid_gen2_sse2;
#ifdef CONFIG_X86_64
test_setup(2) = raid_gen2_sse2ext;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_sse2extaes;
else
test_setup(2) = raid_gen2_sse2ext;
#endif
}
if (raid_cpu_has_avx2()) {
test_setup(1) = raid_gen1_avx2;
test_setup(2) = raid_gen2_avx2;
if (mode == RAID_MODE_CAUCHY_AES)
test_setup(2) = raid_gen2_avx2aes;
else
test_setup(2) = raid_gen2_avx2;
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx512bw()) {
@ -417,31 +433,57 @@ int raid_test_par(int mode, int nd, size_t size)
test_setup(6) = raid_gen6_int8;
#ifdef CONFIG_NEON
test_setup(3) = raid_gen3_neon;
test_setup(4) = raid_gen4_neon;
test_setup(5) = raid_gen5_neon;
test_setup(6) = raid_gen6_neon;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_neonaes;
test_setup(4) = raid_gen4_neonaes;
test_setup(5) = raid_gen5_neonaes;
test_setup(6) = raid_gen6_neonaes;
} else {
test_setup(3) = raid_gen3_neon;
test_setup(4) = raid_gen4_neon;
test_setup(5) = raid_gen5_neon;
test_setup(6) = raid_gen6_neon;
}
#endif
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
test_setup(3) = raid_gen3_ssse3;
test_setup(4) = raid_gen4_ssse3;
test_setup(5) = raid_gen5_ssse3;
test_setup(6) = raid_gen6_ssse3;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_ssse3aes;
test_setup(4) = raid_gen4_ssse3aes;
} else {
test_setup(3) = raid_gen3_ssse3;
test_setup(4) = raid_gen4_ssse3;
test_setup(5) = raid_gen5_ssse3;
test_setup(6) = raid_gen6_ssse3;
}
#ifdef CONFIG_X86_64
test_setup(3) = raid_gen3_ssse3ext;
test_setup(4) = raid_gen4_ssse3ext;
test_setup(5) = raid_gen5_ssse3ext;
test_setup(6) = raid_gen6_ssse3ext;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_ssse3extaes;
test_setup(4) = raid_gen4_ssse3extaes;
test_setup(5) = raid_gen5_ssse3extaes;
test_setup(6) = raid_gen6_ssse3extaes;
} else {
test_setup(3) = raid_gen3_ssse3ext;
test_setup(4) = raid_gen4_ssse3ext;
test_setup(5) = raid_gen5_ssse3ext;
test_setup(6) = raid_gen6_ssse3ext;
}
#endif
}
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
test_setup(3) = raid_gen3_avx2ext;
test_setup(4) = raid_gen4_avx2ext;
test_setup(5) = raid_gen5_avx2ext;
test_setup(6) = raid_gen6_avx2ext;
if (mode == RAID_MODE_CAUCHY_AES) {
test_setup(3) = raid_gen3_avx2extaes;
test_setup(4) = raid_gen4_avx2extaes;
test_setup(5) = raid_gen5_avx2extaes;
test_setup(6) = raid_gen6_avx2extaes;
} else {
test_setup(3) = raid_gen3_avx2ext;
test_setup(4) = raid_gen4_avx2ext;
test_setup(5) = raid_gen5_avx2ext;
test_setup(6) = raid_gen6_avx2ext;
}
}
if (raid_cpu_has_avx512bw()) {
test_setup(3) = raid_gen3_avx512bw;