mirror of
https://github.com/amadvance/snapraid.git
synced 2026-10-26 10:51:54 -04:00
Adds gen3/z support for AVX2.
This commit is contained in:
Andrea Mazzoleni
parent
8e97b7a248
commit
0e3901ca3e
7 changed files with 270 additions and 0 deletions
|
|
@ -297,9 +297,13 @@ void speed(int period)
|
|||
fflush(stdout);
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
printf("%8s", "");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
|
|
@ -355,7 +359,9 @@ void speed(int period)
|
|||
}
|
||||
|
||||
printf("%8s", "");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
|
|
@ -407,6 +413,23 @@ void speed(int period)
|
|||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
|
||||
printf("%8s", "");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
printf("%8s", "");
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_genz_avx2ext(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8"PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
@ -453,6 +476,23 @@ void speed(int period)
|
|||
fflush(stdout);
|
||||
#endif
|
||||
}
|
||||
|
||||
printf("%8s", "");
|
||||
#ifdef CONFIG_X86_64
|
||||
printf("%8s", "");
|
||||
#endif
|
||||
printf("%8s", "");
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
SPEED_START {
|
||||
raid_gen3_avx2ext(nd, size, v);
|
||||
} SPEED_STOP
|
||||
|
||||
printf("%8"PRIu64, ds / dt);
|
||||
fflush(stdout);
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
printf("\n");
|
||||
|
||||
|
|
|
|||
|
|
@ -77,9 +77,11 @@ void raid_genz_int32(int nd, size_t size, void **vv);
|
|||
void raid_genz_int64(int nd, size_t size, void **vv);
|
||||
void raid_genz_sse2(int nd, size_t size, void **vv);
|
||||
void raid_genz_sse2ext(int nd, size_t size, void **vv);
|
||||
void raid_genz_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen3_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen3_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen3_ssse3ext(int nd, size_t size, void **vv);
|
||||
void raid_gen3_avx2ext(int nd, size_t size, void **vv);
|
||||
void raid_gen4_int8(int nd, size_t size, void **vv);
|
||||
void raid_gen4_ssse3(int nd, size_t size, void **vv);
|
||||
void raid_gen4_ssse3ext(int nd, size_t size, void **vv);
|
||||
|
|
|
|||
|
|
@ -83,6 +83,10 @@ void raid_init(void)
|
|||
if (raid_cpu_has_avx2()) {
|
||||
raid_gen_ptr[0] = raid_gen1_avx2;
|
||||
raid_gen_ptr[1] = raid_gen2_avx2;
|
||||
#ifdef CONFIG_X86_64
|
||||
raid_gen3_ptr = raid_gen3_avx2ext;
|
||||
raid_genz_ptr = raid_genz_avx2ext;
|
||||
#endif
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
|
|
@ -54,6 +54,8 @@ static struct raid_func {
|
|||
{ "ssse3e", raid_gen4_ssse3ext },
|
||||
{ "ssse3e", raid_gen5_ssse3ext },
|
||||
{ "ssse3e", raid_gen6_ssse3ext },
|
||||
{ "avx2e", raid_gen3_avx2ext },
|
||||
{ "avx2e", raid_genz_avx2ext },
|
||||
#endif
|
||||
{ 0, 0 }
|
||||
};
|
||||
|
|
|
|||
12
raid/test.c
12
raid/test.c
|
|
@ -335,6 +335,12 @@ int raid_test_par(int mode, int nd, size_t size)
|
|||
f[nf++] = raid_gen6_ssse3ext;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
f[nf++] = raid_gen3_avx2ext;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
} else {
|
||||
f[nf++] = raid_genz_int32;
|
||||
|
|
@ -347,6 +353,12 @@ int raid_test_par(int mode, int nd, size_t size)
|
|||
f[nf++] = raid_genz_sse2ext;
|
||||
#endif
|
||||
}
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
if (raid_cpu_has_avx2()) {
|
||||
f[nf++] = raid_genz_avx2ext;
|
||||
}
|
||||
#endif
|
||||
#endif
|
||||
}
|
||||
|
||||
|
|
|
|||
133
raid/x86.c
133
raid/x86.c
|
|
@ -532,6 +532,139 @@ void raid_gen3_ssse3ext(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GEN3 (triple parity with Cauchy matrix) AVX2 implementation
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
void raid_gen3_avx2ext(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *q;
|
||||
uint8_t *r;
|
||||
int d, l;
|
||||
size_t i;
|
||||
|
||||
l = nd - 1;
|
||||
p = v[nd];
|
||||
q = v[nd+1];
|
||||
r = v[nd+2];
|
||||
|
||||
/* special case with only one data disk */
|
||||
if (l == 0) {
|
||||
for (i = 0; i < 3; ++i)
|
||||
memcpy(v[1+i], v[0], size);
|
||||
return;
|
||||
}
|
||||
|
||||
raid_asm_begin();
|
||||
|
||||
/* generic case with at least two data disks */
|
||||
asm volatile("vbroadcasti128 %0, %%ymm3" : : "m" (gfconst16.poly[0]));
|
||||
asm volatile("vbroadcasti128 %0, %%ymm11" : : "m" (gfconst16.low4[0]));
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
/* last disk without the by two multiplication */
|
||||
asm volatile("vmovdqa %0,%%ymm4" : : "m" (v[l][i]));
|
||||
asm volatile("vmovdqa %0,%%ymm12" : : "m" (v[l][i+32]));
|
||||
|
||||
asm volatile("vmovdqa %ymm4,%ymm0");
|
||||
asm volatile("vmovdqa %ymm4,%ymm1");
|
||||
asm volatile("vmovdqa %ymm12,%ymm8");
|
||||
asm volatile("vmovdqa %ymm12,%ymm9");
|
||||
|
||||
asm volatile("vpsrlw $4,%ymm4,%ymm5");
|
||||
asm volatile("vpsrlw $4,%ymm12,%ymm13");
|
||||
asm volatile("vpand %ymm11,%ymm4,%ymm4");
|
||||
asm volatile("vpand %ymm11,%ymm12,%ymm12");
|
||||
asm volatile("vpand %ymm11,%ymm5,%ymm5");
|
||||
asm volatile("vpand %ymm11,%ymm13,%ymm13");
|
||||
|
||||
asm volatile("vbroadcasti128 %0,%%ymm10" : : "m" (gfgenpshufb[l][0][0][0]));
|
||||
asm volatile("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[l][0][1][0]));
|
||||
asm volatile("vpshufb %ymm4,%ymm10,%ymm2");
|
||||
asm volatile("vpshufb %ymm12,%ymm10,%ymm10");
|
||||
asm volatile("vpshufb %ymm5,%ymm15,%ymm7");
|
||||
asm volatile("vpshufb %ymm13,%ymm15,%ymm15");
|
||||
asm volatile("vpxor %ymm7,%ymm2,%ymm2");
|
||||
asm volatile("vpxor %ymm15,%ymm10,%ymm10");
|
||||
|
||||
/* intermediate disks */
|
||||
for (d = l-1; d > 0; --d) {
|
||||
asm volatile("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile("vmovdqa %0,%%ymm12" : : "m" (v[d][i+32]));
|
||||
|
||||
asm volatile("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile("vpand %ymm3,%ymm5,%ymm5");
|
||||
asm volatile("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile("vpxor %ymm5,%ymm1,%ymm1");
|
||||
asm volatile("vpxor %ymm13,%ymm9,%ymm9");
|
||||
|
||||
asm volatile("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile("vpxor %ymm12,%ymm8,%ymm8");
|
||||
asm volatile("vpxor %ymm12,%ymm9,%ymm9");
|
||||
|
||||
asm volatile("vpsrlw $4,%ymm4,%ymm5");
|
||||
asm volatile("vpsrlw $4,%ymm12,%ymm13");
|
||||
asm volatile("vpand %ymm11,%ymm4,%ymm4");
|
||||
asm volatile("vpand %ymm11,%ymm12,%ymm12");
|
||||
asm volatile("vpand %ymm11,%ymm5,%ymm5");
|
||||
asm volatile("vpand %ymm11,%ymm13,%ymm13");
|
||||
|
||||
asm volatile("vbroadcasti128 %0,%%ymm14" : : "m" (gfgenpshufb[d][0][0][0]));
|
||||
asm volatile("vbroadcasti128 %0,%%ymm15" : : "m" (gfgenpshufb[d][0][1][0]));
|
||||
asm volatile("vpshufb %ymm4,%ymm14,%ymm6");
|
||||
asm volatile("vpshufb %ymm12,%ymm14,%ymm14");
|
||||
asm volatile("vpshufb %ymm5,%ymm15,%ymm7");
|
||||
asm volatile("vpshufb %ymm13,%ymm15,%ymm15");
|
||||
asm volatile("vpxor %ymm6,%ymm2,%ymm2");
|
||||
asm volatile("vpxor %ymm14,%ymm10,%ymm10");
|
||||
asm volatile("vpxor %ymm7,%ymm2,%ymm2");
|
||||
asm volatile("vpxor %ymm15,%ymm10,%ymm10");
|
||||
}
|
||||
|
||||
/* first disk with all coefficients at 1 */
|
||||
asm volatile("vmovdqa %0,%%ymm4" : : "m" (v[0][i]));
|
||||
asm volatile("vmovdqa %0,%%ymm12" : : "m" (v[0][i+32]));
|
||||
|
||||
asm volatile("vpxor %ymm5,%ymm5,%ymm5");
|
||||
asm volatile("vpxor %ymm13,%ymm13,%ymm13");
|
||||
asm volatile("vpcmpgtb %ymm1,%ymm5,%ymm5");
|
||||
asm volatile("vpcmpgtb %ymm9,%ymm13,%ymm13");
|
||||
asm volatile("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile("vpand %ymm3,%ymm5,%ymm5");
|
||||
asm volatile("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile("vpxor %ymm5,%ymm1,%ymm1");
|
||||
asm volatile("vpxor %ymm13,%ymm9,%ymm9");
|
||||
|
||||
asm volatile("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile("vpxor %ymm4,%ymm2,%ymm2");
|
||||
asm volatile("vpxor %ymm12,%ymm8,%ymm8");
|
||||
asm volatile("vpxor %ymm12,%ymm9,%ymm9");
|
||||
asm volatile("vpxor %ymm12,%ymm10,%ymm10");
|
||||
|
||||
asm volatile("vmovntdq %%ymm0,%0" : "=m" (p[i]));
|
||||
asm volatile("vmovntdq %%ymm8,%0" : "=m" (p[i+32]));
|
||||
asm volatile("vmovntdq %%ymm1,%0" : "=m" (q[i]));
|
||||
asm volatile("vmovntdq %%ymm9,%0" : "=m" (q[i+32]));
|
||||
asm volatile("vmovntdq %%ymm2,%0" : "=m" (r[i]));
|
||||
asm volatile("vmovntdq %%ymm10,%0" : "=m" (r[i+32]));
|
||||
}
|
||||
|
||||
raid_asm_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86
|
||||
/*
|
||||
* GEN4 (quad parity with Cauchy matrix) SSSE3 implementation
|
||||
|
|
|
|||
77
raid/x86z.c
77
raid/x86z.c
|
|
@ -175,3 +175,80 @@ void raid_genz_sse2ext(int nd, size_t size, void **vv)
|
|||
}
|
||||
#endif
|
||||
|
||||
#ifdef CONFIG_X86_64
|
||||
/*
|
||||
* GENz (triple parity with powers of 2^-1) AVX2 implementation
|
||||
*
|
||||
* Note that it uses 16 registers, meaning that x64 is required.
|
||||
*/
|
||||
void raid_genz_avx2ext(int nd, size_t size, void **vv)
|
||||
{
|
||||
uint8_t **v = (uint8_t **)vv;
|
||||
uint8_t *p;
|
||||
uint8_t *q;
|
||||
uint8_t *r;
|
||||
int d, l;
|
||||
size_t i;
|
||||
|
||||
l = nd - 1;
|
||||
p = v[nd];
|
||||
q = v[nd+1];
|
||||
r = v[nd+2];
|
||||
|
||||
raid_asm_begin();
|
||||
|
||||
asm volatile("vbroadcasti128 %0,%%ymm7" : : "m" (gfzconst16.poly[0]));
|
||||
asm volatile("vbroadcasti128 %0,%%ymm3" : : "m" (gfzconst16.half[0]));
|
||||
asm volatile("vbroadcasti128 %0,%%ymm11" : : "m" (gfzconst16.low7[0]));
|
||||
asm volatile("vpxor %ymm15,%ymm15,%ymm15");
|
||||
|
||||
for (i = 0; i < size; i += 64) {
|
||||
asm volatile("vmovdqa %0,%%ymm0" : : "m" (v[l][i]));
|
||||
asm volatile("vmovdqa %0,%%ymm8" : : "m" (v[l][i+32]));
|
||||
asm volatile("vmovdqa %ymm0,%ymm1");
|
||||
asm volatile("vmovdqa %ymm8,%ymm9");
|
||||
asm volatile("vmovdqa %ymm0,%ymm2");
|
||||
asm volatile("vmovdqa %ymm8,%ymm10");
|
||||
for (d = l-1; d >= 0; --d) {
|
||||
asm volatile("vpsllw $7,%ymm2,%ymm6");
|
||||
asm volatile("vpsllw $7,%ymm10,%ymm14");
|
||||
asm volatile("vpsrlw $1,%ymm2,%ymm2");
|
||||
asm volatile("vpsrlw $1,%ymm10,%ymm10");
|
||||
asm volatile("vpcmpgtb %ymm1,%ymm15,%ymm4");
|
||||
asm volatile("vpcmpgtb %ymm9,%ymm15,%ymm12");
|
||||
asm volatile("vpcmpgtb %ymm6,%ymm15,%ymm5");
|
||||
asm volatile("vpcmpgtb %ymm14,%ymm15,%ymm13");
|
||||
asm volatile("vpaddb %ymm1,%ymm1,%ymm1");
|
||||
asm volatile("vpaddb %ymm9,%ymm9,%ymm9");
|
||||
asm volatile("vpand %ymm11,%ymm2,%ymm2");
|
||||
asm volatile("vpand %ymm11,%ymm10,%ymm10");
|
||||
asm volatile("vpand %ymm7,%ymm4,%ymm4");
|
||||
asm volatile("vpand %ymm7,%ymm12,%ymm12");
|
||||
asm volatile("vpand %ymm3,%ymm5,%ymm5");
|
||||
asm volatile("vpand %ymm3,%ymm13,%ymm13");
|
||||
asm volatile("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile("vpxor %ymm12,%ymm9,%ymm9");
|
||||
asm volatile("vpxor %ymm5,%ymm2,%ymm2");
|
||||
asm volatile("vpxor %ymm13,%ymm10,%ymm10");
|
||||
|
||||
asm volatile("vmovdqa %0,%%ymm4" : : "m" (v[d][i]));
|
||||
asm volatile("vmovdqa %0,%%ymm12" : : "m" (v[d][i+32]));
|
||||
asm volatile("vpxor %ymm4,%ymm0,%ymm0");
|
||||
asm volatile("vpxor %ymm4,%ymm1,%ymm1");
|
||||
asm volatile("vpxor %ymm4,%ymm2,%ymm2");
|
||||
asm volatile("vpxor %ymm12,%ymm8,%ymm8");
|
||||
asm volatile("vpxor %ymm12,%ymm9,%ymm9");
|
||||
asm volatile("vpxor %ymm12,%ymm10,%ymm10");
|
||||
}
|
||||
asm volatile("vmovntdq %%ymm0,%0" : "=m" (p[i]));
|
||||
asm volatile("vmovntdq %%ymm8,%0" : "=m" (p[i+32]));
|
||||
asm volatile("vmovntdq %%ymm1,%0" : "=m" (q[i]));
|
||||
asm volatile("vmovntdq %%ymm9,%0" : "=m" (q[i+32]));
|
||||
asm volatile("vmovntdq %%ymm2,%0" : "=m" (r[i]));
|
||||
asm volatile("vmovntdq %%ymm10,%0" : "=m" (r[i+32]));
|
||||
}
|
||||
|
||||
raid_asm_end();
|
||||
}
|
||||
#endif
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue