rt/snapraid
1
0
Fork 0
mirror of https://github.com/amadvance/snapraid.git synced 2026-10-26 02:49:02 -04:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use temporal stores for recovered data in SSSE3 and AVX2 REC1OF1 paths

Browse source
This commit is contained in:
Andrea Mazzoleni 2026-08-31 21:33:45 +02:00
parent 6cd217ec2b
commit b991d5e6c8
4 changed files with 110 additions and 24 deletions
Download patch file Download diff file Expand all files Collapse all files

56
raid/avx2.c
View file

@ -891,6 +891,54 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in
}
#endif
/*
* Recover one data failure using P with AVX2.
*
* The missing block is:
*
* Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn
*
* Use temporal stores because recovered data is consumed immediately by
* validation and parity regeneration.
*/
static __always_inline void raid_rec1of1_avx2(int *id, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *src[RAID_DATA_MAX];
uint8_t *p;
uint8_t *pa;
size_t i;
int d, s, ns;
p = v[nd];
pa = v[id[0]];
ns = 0;
for (d = 0; d < nd; ++d) {
if (d != id[0])
src[ns++] = v[d];
}
BUG_ON(ns != nd - 1);
raid_avx_begin();
for (i = 0; i < size; i += 64) {
asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i]));
asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32]));
for (s = 0; s < ns; ++s) {
asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (src[s][i]));
asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (src[s][i + 32]));
}
asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i]));
asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32]));
}
raid_avx_end();
}
/*
* Recover one data failure using selected parity with AVX2.
*
@ -2695,9 +2743,9 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
/* if recovering with P uses the delta function */
/* if recovering with P, use a custom XOR-only path with temporal stores */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
raid_rec1of1_avx2(id, nd, size, vv);
return;
}
@ -2760,9 +2808,9 @@ void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
/* if recovering with P uses the delta function */
/* if recovering with P, use a custom XOR-only path with temporal stores */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
raid_rec1of1_avx2(id, nd, size, vv);
return;
}

62
raid/ssse3.c
View file

@ -1166,6 +1166,60 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i
#endif
/*
* Recover one data failure using P with SSSE3.
*
* The missing block is:
*
* Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn
*
* Use temporal stores because recovered data is consumed immediately by
* validation and parity regeneration.
*/
static __always_inline void raid_rec1of1_ssse3(int *id, int nd, size_t size, void **vv)
{
uint8_t **v = (uint8_t **)vv;
uint8_t *src[RAID_DATA_MAX];
uint8_t *p;
uint8_t *pa;
size_t i;
int d, s, ns;
p = v[nd];
pa = v[id[0]];
ns = 0;
for (d = 0; d < nd; ++d) {
if (d != id[0])
src[ns++] = v[d];
}
BUG_ON(ns != nd - 1);
raid_sse_begin();
for (i = 0; i < size; i += 64) {
asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i]));
asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16]));
asm volatile ("movdqa %0,%%xmm2" : : "m" (p[i + 32]));
asm volatile ("movdqa %0,%%xmm3" : : "m" (p[i + 48]));
for (s = 0; s < ns; ++s) {
asm volatile ("pxor %0,%%xmm0" : : "m" (src[s][i]));
asm volatile ("pxor %0,%%xmm1" : : "m" (src[s][i + 16]));
asm volatile ("pxor %0,%%xmm2" : : "m" (src[s][i + 32]));
asm volatile ("pxor %0,%%xmm3" : : "m" (src[s][i + 48]));
}
asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i]));
asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16]));
asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32]));
asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48]));
}
raid_sse_end();
}
/*
* Recover failure of one data block using selected parity with SSSE3, optimized for one failure.
*
@ -4052,9 +4106,9 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv)
{
BUG_ON(nr != 1);
/* if recovering with P uses the delta function */
/* if recovering with P, use a custom XOR-only path with temporal stores */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
raid_rec1of1_ssse3(id, nd, size, vv);
return;
}
@ -4124,9 +4178,9 @@ void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv
{
BUG_ON(nr != 1);
/* if recovering with P uses the delta function */
/* if recovering with P, use a custom XOR-only path with temporal stores */
if (ip[0] == 0) {
raid_rec1of1(id, nd, size, vv);
raid_rec1of1_ssse3(id, nd, size, vv);
return;
}

8
raid/test/speedtest.c
View file

@ -1183,8 +1183,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86
if (raid_cpu_has_ssse3()) {
SPEED_START {
/* force the parity generator used by raid_rec1of1() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1193,8 +1191,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86_64
SPEED_START {
/* force the parity generator used by raid_rec1of1() */
raid_gen_force(1, raid_gen1_sse2);
raid_rec1_ssse3ext(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1204,8 +1200,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
}
if (raid_cpu_has_avx2()) {
SPEED_START {
/* force the parity generator used by raid_rec1of1() */
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2(1, id, ip, nd, size, v);
} SPEED_STOP
@ -1215,8 +1209,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period)
#ifdef CONFIG_X86_64
if (raid_cpu_has_avx2()) {
SPEED_START {
/* force the parity generator used by raid_rec1of1() */
raid_gen_force(1, raid_gen1_avx2);
raid_rec1_avx2ext(1, id, ip, nd, size, v);
} SPEED_STOP