From 957db46f5ba45fe66c633c9c3bd5c63af05f3c84 Mon Sep 17 00:00:00 2001 From: Andrea Mazzoleni Date: Sun, 14 Sep 2014 18:03:40 +0200 Subject: [PATCH] Preliminary AVX2 support. --- cmdline/speed.c | 31 +++++++++++++++ raid/internal.h | 2 + raid/module.c | 5 +++ raid/tag.c | 2 + raid/test.c | 5 +++ raid/x86.c | 102 +++++++++++++++++++++++++++++++++++++++++++++--- raid/x86z.c | 18 ++++++--- 7 files changed, 154 insertions(+), 11 deletions(-) diff --git a/cmdline/speed.c b/cmdline/speed.c index c2374f20..e4867669 100644 --- a/cmdline/speed.c +++ b/cmdline/speed.c @@ -259,6 +259,10 @@ void speed(int period) printf("%8s", "ssse3"); #ifdef CONFIG_X86_64 printf("%8s", "ssse3e"); +#endif + printf("%8s", "avx2"); +#ifdef CONFIG_X86_64 + printf("%8s", "avx2e"); #endif #endif printf("\n"); @@ -293,6 +297,19 @@ void speed(int period) printf("%8"PRIu64, ds / dt); fflush(stdout); } + + printf("%8s", ""); + printf("%8s", ""); + printf("%8s", ""); + + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_gen1_avx2(nd, size, v); + } SPEED_STOP + + printf("%8"PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); @@ -333,8 +350,22 @@ void speed(int period) printf("%8"PRIu64, ds / dt); fflush(stdout); +#else + printf("%8s", ""); #endif } + + printf("%8s", ""); + printf("%8s", ""); + + if (raid_cpu_has_avx2()) { + SPEED_START { + raid_gen2_avx2(nd, size, v); + } SPEED_STOP + + printf("%8"PRIu64, ds / dt); + fflush(stdout); + } #endif printf("\n"); diff --git a/raid/internal.h b/raid/internal.h index fab6e0cb..71b2bcf9 100644 --- a/raid/internal.h +++ b/raid/internal.h @@ -67,9 +67,11 @@ void raid_rec2of2_int8(int *id, int *ip, int nd, size_t size, void **vv); void raid_gen1_int32(int nd, size_t size, void **vv); void raid_gen1_int64(int nd, size_t size, void **vv); void raid_gen1_sse2(int nd, size_t size, void **vv); +void raid_gen1_avx2(int nd, size_t size, void **vv); void raid_gen2_int32(int nd, size_t size, void **vv); void raid_gen2_int64(int nd, size_t size, void **vv); void raid_gen2_sse2(int nd, size_t size, void **vv); +void raid_gen2_avx2(int nd, size_t size, void **vv); void raid_gen2_sse2ext(int nd, size_t size, void **vv); void raid_genz_int32(int nd, size_t size, void **vv); void raid_genz_int64(int nd, size_t size, void **vv); diff --git a/raid/module.c b/raid/module.c index 7a72c986..2d4b56ba 100644 --- a/raid/module.c +++ b/raid/module.c @@ -79,6 +79,11 @@ void raid_init(void) raid_rec_ptr[4] = raid_recX_ssse3; raid_rec_ptr[5] = raid_recX_ssse3; } + + if (raid_cpu_has_avx2()) { + raid_gen_ptr[0] = raid_gen1_avx2; + raid_gen_ptr[1] = raid_gen2_avx2; + } #endif /* set the default mode */ diff --git a/raid/tag.c b/raid/tag.c index e3042c88..2007660d 100644 --- a/raid/tag.c +++ b/raid/tag.c @@ -43,6 +43,8 @@ static struct raid_func { { "ssse3", raid_rec1_ssse3 }, { "ssse3", raid_rec2_ssse3 }, { "ssse3", raid_recX_ssse3 }, + { "avx2", raid_gen1_avx2 }, + { "avx2", raid_gen2_avx2 }, #endif #ifdef CONFIG_X86_64 diff --git a/raid/test.c b/raid/test.c index a7189623..bcdf60db 100644 --- a/raid/test.c +++ b/raid/test.c @@ -309,6 +309,11 @@ int raid_test_par(int mode, int nd, size_t size) f[nf++] = raid_gen2_sse2ext; #endif } + + if (raid_cpu_has_avx2()) { + f[nf++] = raid_gen1_avx2; + f[nf++] = raid_gen2_avx2; + } #endif if (mode == RAID_MODE_CAUCHY) { diff --git a/raid/x86.c b/raid/x86.c index 77badc48..d4e4ff25 100644 --- a/raid/x86.c +++ b/raid/x86.c @@ -56,15 +56,54 @@ void raid_gen1_sse2(int nd, size_t size, void **vv) } #endif +#ifdef CONFIG_X86 +/* + * GEN1 (RAID5 with xor) AVX2 implementation + * + * Intentionally don't process more than 64 bytes because 64 is the typical + * cache block, and processing 128 bytes doesn't increase performance, and in + * some cases it even decreases it. + */ +void raid_gen1_avx2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + + raid_asm_begin(); + + for (i = 0; i < size; i += 64) { + asm volatile("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); + asm volatile("vmovdqa %0,%%ymm2" : : "m" (v[l][i+32])); + for (d = l-1; d >= 0; --d) { + asm volatile("vpxor %0,%%ymm0,%%ymm0" : : "m" (v[d][i])); + asm volatile("vpxor %0,%%ymm2,%%ymm2" : : "m" (v[d][i+32])); + } + asm volatile("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile("vmovntdq %%ymm2,%0" : "=m" (p[i+32])); + } + + raid_asm_end(); +} +#endif + #ifdef CONFIG_X86 static const struct gfconst16 { uint8_t poly[16]; uint8_t low4[16]; -} gfconst16 __aligned(32) = { - { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, - 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, - { 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, - 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f }, +} gfconst16 __aligned(32) = { + { + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d + }, + { + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, + 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f, 0x0f + }, }; #endif @@ -122,6 +161,59 @@ void raid_gen2_sse2(int nd, size_t size, void **vv) } #endif +#ifdef CONFIG_X86 +/* + * GEN2 (RAID6 with powers of 2) AVX2 implementation + */ +void raid_gen2_avx2(int nd, size_t size, void **vv) +{ + uint8_t **v = (uint8_t **)vv; + uint8_t *p; + uint8_t *q; + int d, l; + size_t i; + + l = nd - 1; + p = v[nd]; + q = v[nd+1]; + + raid_asm_begin(); + + asm volatile("vbroadcasti128 %0, %%ymm7" : : "m" (gfconst16.poly[0])); + asm volatile("vpxor %ymm6,%ymm6,%ymm6"); + + for (i = 0; i < size; i += 64) { + asm volatile("vmovdqa %0,%%ymm0" : : "m" (v[l][i])); + asm volatile("vmovdqa %0,%%ymm1" : : "m" (v[l][i+32])); + asm volatile("vmovdqa %ymm0,%ymm2"); + asm volatile("vmovdqa %ymm1,%ymm3"); + for (d = l-1; d >= 0; --d) { + asm volatile("vpcmpgtb %ymm2,%ymm6,%ymm4"); + asm volatile("vpcmpgtb %ymm3,%ymm6,%ymm5"); + asm volatile("vpaddb %ymm2,%ymm2,%ymm2"); + asm volatile("vpaddb %ymm3,%ymm3,%ymm3"); + asm volatile("vpand %ymm7,%ymm4,%ymm4"); + asm volatile("vpand %ymm7,%ymm5,%ymm5"); + asm volatile("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile("vpxor %ymm5,%ymm3,%ymm3"); + + asm volatile("vmovdqa %0,%%ymm4" : : "m" (v[d][i])); + asm volatile("vmovdqa %0,%%ymm5" : : "m" (v[d][i+32])); + asm volatile("vpxor %ymm4,%ymm0,%ymm0"); + asm volatile("vpxor %ymm5,%ymm1,%ymm1"); + asm volatile("vpxor %ymm4,%ymm2,%ymm2"); + asm volatile("vpxor %ymm5,%ymm3,%ymm3"); + } + asm volatile("vmovntdq %%ymm0,%0" : "=m" (p[i])); + asm volatile("vmovntdq %%ymm1,%0" : "=m" (p[i+32])); + asm volatile("vmovntdq %%ymm2,%0" : "=m" (q[i])); + asm volatile("vmovntdq %%ymm3,%0" : "=m" (q[i+32])); + } + + raid_asm_end(); +} +#endif + #ifdef CONFIG_X86_64 /* * GEN2 (RAID6 with powers of 2) SSE2 implementation diff --git a/raid/x86z.c b/raid/x86z.c index 8aa9567a..28e41f55 100644 --- a/raid/x86z.c +++ b/raid/x86z.c @@ -20,12 +20,18 @@ static const struct gfzconst16 { uint8_t half[16]; uint8_t low7[16]; } gfzconst16 __aligned(64) = { - { 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, - 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d }, - { 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, - 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e }, - { 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f } + { + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, + 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d, 0x1d + }, + { + 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, + 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e, 0x8e + }, + { + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, + 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f + } }; #endif