diff --git a/cmdline/speed.c b/cmdline/speed.c index 94eaf560..2dd3446e 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -1287,8 +1287,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3(1, id, ip, nd, size, v); } SPEED_STOP @@ -1297,8 +1295,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3ext(1, id, ip, nd, size, v); } SPEED_STOP @@ -1308,8 +1304,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) } if (raid_cpu_has_avx2()) { SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2(1, id, ip, nd, size, v); } SPEED_STOP @@ -1319,8 +1313,6 @@ void speed_rec(int nd, void** v, int size, int delta, int period) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx2()) { SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2ext(1, id, ip, nd, size, v); } SPEED_STOP diff --git a/raid/avx2.c b/raid/avx2.c index f3447268..0e675f30 100644 --- a/raid/avx2.c +++ b/raid/avx2.c @@ -891,6 +891,54 @@ static __always_inline void raid_genX_avx2ext(int nd, size_t size, void **vv, in } #endif +/* + * Recover one data failure using P with AVX2. + * + * The missing block is: + * + * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn + * + * Use temporal stores because recovered data is consumed immediately by + * validation and parity regeneration. + */ +static __always_inline void raid_rec1of1_avx2(int *id, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *src[RAID_DATA_MAX]; + uint8_t *p; + uint8_t *pa; + size_t i; + int d, s, ns; + + p = v[nd]; + pa = v[id[0]]; + + ns = 0; + for (d = 0; d < nd; ++d) { + if (d != id[0]) + src[ns++] = v[d]; + } + + BUG_ON(ns != nd - 1); + + raid_avx_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("vmovdqa %0,%%ymm0" : : "m" (p[i])); + asm volatile ("vmovdqa %0,%%ymm1" : : "m" (p[i + 32])); + + for (s = 0; s < ns; ++s) { + asm volatile ("vpxor %0,%%ymm0,%%ymm0" : : "m" (src[s][i])); + asm volatile ("vpxor %0,%%ymm1,%%ymm1" : : "m" (src[s][i + 32])); + } + + asm volatile ("vmovdqa %%ymm0,%0" : "=m" (pa[i])); + asm volatile ("vmovdqa %%ymm1,%0" : "=m" (pa[i + 32])); + } + + raid_avx_end(); +} + /* * Recover one data failure using selected parity with AVX2. * @@ -2695,9 +2743,9 @@ void raid_rec1_avx2(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - /* if recovering with P uses the delta function */ + /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); + raid_rec1of1_avx2(id, nd, size, vv); return; } @@ -2760,9 +2808,9 @@ void raid_rec1_avx2ext(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - /* if recovering with P uses the delta function */ + /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); + raid_rec1of1_avx2(id, nd, size, vv); return; } diff --git a/raid/ssse3.c b/raid/ssse3.c index 9416565f..f792becd 100644 --- a/raid/ssse3.c +++ b/raid/ssse3.c @@ -1166,6 +1166,60 @@ static __always_inline void raid_genX_ssse3ext(int nd, size_t size, void **vv, i #endif +/* + * Recover one data failure using P with SSSE3. + * + * The missing block is: + * + * Dx = P ^ D0 ^ ... ^ D(x-1) ^ D(x+1) ^ ... ^ Dn + * + * Use temporal stores because recovered data is consumed immediately by + * validation and parity regeneration. + */ +static __always_inline void raid_rec1of1_ssse3(int *id, int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *src[RAID_DATA_MAX]; + uint8_t *p; + uint8_t *pa; + size_t i; + int d, s, ns; + + p = v[nd]; + pa = v[id[0]]; + + ns = 0; + for (d = 0; d < nd; ++d) { + if (d != id[0]) + src[ns++] = v[d]; + } + + BUG_ON(ns != nd - 1); + + raid_sse_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile ("movdqa %0,%%xmm0" : : "m" (p[i])); + asm volatile ("movdqa %0,%%xmm1" : : "m" (p[i + 16])); + asm volatile ("movdqa %0,%%xmm2" : : "m" (p[i + 32])); + asm volatile ("movdqa %0,%%xmm3" : : "m" (p[i + 48])); + + for (s = 0; s < ns; ++s) { + asm volatile ("pxor %0,%%xmm0" : : "m" (src[s][i])); + asm volatile ("pxor %0,%%xmm1" : : "m" (src[s][i + 16])); + asm volatile ("pxor %0,%%xmm2" : : "m" (src[s][i + 32])); + asm volatile ("pxor %0,%%xmm3" : : "m" (src[s][i + 48])); + } + + asm volatile ("movdqa %%xmm0,%0" : "=m" (pa[i])); + asm volatile ("movdqa %%xmm1,%0" : "=m" (pa[i + 16])); + asm volatile ("movdqa %%xmm2,%0" : "=m" (pa[i + 32])); + asm volatile ("movdqa %%xmm3,%0" : "=m" (pa[i + 48])); + } + + raid_sse_end(); +} + /* * Recover failure of one data block using selected parity with SSSE3, optimized for one failure. * @@ -4052,9 +4106,9 @@ void raid_rec1_ssse3(int nr, int *id, int *ip, int nd, size_t size, void **vv) { BUG_ON(nr != 1); - /* if recovering with P uses the delta function */ + /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); + raid_rec1of1_ssse3(id, nd, size, vv); return; } @@ -4124,9 +4178,9 @@ void raid_rec1_ssse3ext(int nr, int *id, int *ip, int nd, size_t size, void **vv { BUG_ON(nr != 1); - /* if recovering with P uses the delta function */ + /* if recovering with P, use a custom XOR-only path with temporal stores */ if (ip[0] == 0) { - raid_rec1of1(id, nd, size, vv); + raid_rec1of1_ssse3(id, nd, size, vv); return; } diff --git a/raid/test/speedtest.c b/raid/test/speedtest.c index 8f575d08..93dda9fd 100644 --- a/raid/test/speedtest.c +++ b/raid/test/speedtest.c @@ -1183,8 +1183,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86 if (raid_cpu_has_ssse3()) { SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3(1, id, ip, nd, size, v); } SPEED_STOP @@ -1193,8 +1191,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86_64 SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_sse2); raid_rec1_ssse3ext(1, id, ip, nd, size, v); } SPEED_STOP @@ -1204,8 +1200,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) } if (raid_cpu_has_avx2()) { SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2(1, id, ip, nd, size, v); } SPEED_STOP @@ -1215,8 +1209,6 @@ void speed_rec(int nd, void **v, int size, int delta, int period) #ifdef CONFIG_X86_64 if (raid_cpu_has_avx2()) { SPEED_START { - /* force the parity generator used by raid_rec1of1() */ - raid_gen_force(1, raid_gen1_avx2); raid_rec1_avx2ext(1, id, ip, nd, size, v); } SPEED_STOP